Refined polly retry logic to try and tighten down transaction failures.

Ed-Fi-Alliance-OSS · Oct 26, 2024 · dd53890 · dd53890
1 parent 60578f8
commit dd53890
Show file tree

Hide file tree

Showing 5 changed files with 19 additions and 78 deletions.
diff --git a/Application/EdFi.Ods.Common/EdFi.Ods.Common.csproj b/Application/EdFi.Ods.Common/EdFi.Ods.Common.csproj
@@ -39,6 +39,7 @@
     <PackageReference Include="NHibernate" Version="5.5.2" />
     <PackageReference Include="Npgsql" Version="8.0.3" />
     <PackageReference Include="Polly" Version="8.4.1" />
+    <PackageReference Include="Polly.Contrib.WaitAndRetry" Version="1.1.1" />
     <PackageReference Include="Sandwych.QuickGraph.Core" Version="1.0.0" />
     <PackageReference Include="Standart.Hash.xxHash" Version="4.0.5" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />

diff --git a/Application/EdFi.Ods.Common/Infrastructure/Interceptors/EdFiOdsInterceptor.cs b/Application/EdFi.Ods.Common/Infrastructure/Interceptors/EdFiOdsInterceptor.cs
@@ -4,31 +4,15 @@
 // See the LICENSE and NOTICES files in the project root for more information.
 
 using System;
-using System.Text.RegularExpressions;
-using EdFi.Ods.Common.Conventions;
-using EdFi.Ods.Common.Security.CustomViewBased;
-using log4net;
 using NHibernate;
-using NHibernate.SqlCommand;
 
 namespace EdFi.Ods.Common.Infrastructure.Interceptors
 {
     public class EdFiOdsInterceptor : EmptyInterceptor
     {
-        /*
-        private readonly ILog _logger = LogManager.GetLogger(typeof(EdFiOdsInterceptor));
-
-        private const string SpaceLiteral = " ";
-
-        private static readonly Regex regex = new(
-            @$"(?<join>inner join\s+)\(?(?<basisEntity>\w+\.\w+) (?<alias>(?<aliasPrefix>{CustomViewHelpers.CustomViewAliasPrefixBase}[a-f\d]{{4}}_).*?\d_).*?(?<onClause>on[\s\(]+this_)",
-            RegexOptions.Compiled);
-        */
-
         public override bool? IsTransient(object entity)
         {
-            var property = entity.GetType()
-                                 .GetProperty("CreateDate");
+            var property = entity.GetType().GetProperty("CreateDate");
 
             if (property != null)
             {
@@ -39,43 +23,5 @@ public class EdFiOdsInterceptor : EmptyInterceptor
 
             return base.IsTransient(entity);
         }
-
-        /*
-        public override SqlString OnPrepareStatement(SqlString sql)
-        {
-            // Determine if we need to perform custom authorization view SQL manipulation
-            if (sql.IndexOf(CustomViewHelpers.CustomViewAliasPrefixBase, 0, sql.Length, StringComparison.Ordinal) >= 0)
-            {
-                var matches = regex.Matches(sql.ToString());
-
-                for (int i = matches.Count - 1; i >= 0; i--)
-                {
-                    var match = matches[i];
-                    
-                    var preamble = sql.Substring(0, match.Groups["join"].Index + match.Groups["join"].Length);
-                    var viewName = CustomViewHelpers.GetViewName(match.Groups["aliasPrefix"].Value);
-
-                    var final = new SqlString(
-                        preamble,
-                        SystemConventions.AuthSchema,
-                        ".",
-                        viewName,
-                        SpaceLiteral,
-                        match.Groups["alias"].Value,
-                        SpaceLiteral,
-                        sql.Substring(match.Groups["onClause"].Index));
-                    
-                    sql = final;
-                }
-            }
-
-            if (_logger.IsDebugEnabled)
-            {
-                _logger.Debug($"Prepared SQL: {sql}");
-            }
-
-            return sql;
-        }
-        */
     }
 }
diff --git a/Application/EdFi.Ods.Common/Infrastructure/Repositories/CreateEntity.cs b/Application/EdFi.Ods.Common/Infrastructure/Repositories/CreateEntity.cs
@@ -116,25 +116,19 @@ await DeadlockPolicyHelper.RetryPolicy.ExecuteAsync(
                         try
                         {
                             await Session.SaveAsync(entity, cancellationToken);
+                            await trans.CommitAsync(cancellationToken);
                         }
                         catch (Exception)
                         {
                             await trans.RollbackAsync(cancellationToken);
                             throw;
                         }
-                        finally
-                        {
-                            if (!trans.WasRolledBack)
-                            {
-                                await trans.CommitAsync(cancellationToken);
-                            }
-                        }
                     },
                     _retryPolicyContextData);
 
                 bool IdHasValue()
                 {
-                    return !entity.Id.Equals(default(Guid));
+                    return !entity.Id.Equals(default);
                 }
             }
         }

diff --git a/Application/EdFi.Ods.Common/Infrastructure/Repositories/DeadlockPolicyHelper.cs b/Application/EdFi.Ods.Common/Infrastructure/Repositories/DeadlockPolicyHelper.cs
@@ -4,10 +4,12 @@
 // See the LICENSE and NOTICES files in the project root for more information.
 
 using System;
+using System.Threading;
 using log4net;
 using Microsoft.Data.SqlClient;
 using Npgsql;
 using Polly;
+using Polly.Contrib.WaitAndRetry;
 using Polly.Retry;
 
 namespace EdFi.Ods.Common.Infrastructure.Repositories;
@@ -19,21 +21,25 @@ public static class DeadlockPolicyHelper
     public static int RetryCount = 5;
     public static int RetryStartingDelayMilliseconds = 100;
 
+    private static int _totalRetries = 0;
+
     static DeadlockPolicyHelper()
     {
         RetryPolicy = Policy.Handle<Exception>(ShouldRetry)
             .WaitAndRetryAsync(
-                RetryCount,
-                (retryNumber, context) =>
+                Backoff.ExponentialBackoff(TimeSpan.FromMilliseconds(RetryStartingDelayMilliseconds), RetryCount),
+                onRetry: (result, ts, retryAttempt, context) =>
                 {
-                    var waitDuration = TimeSpan.FromMilliseconds(RetryStartingDelayMilliseconds * (Math.Pow(2, retryNumber)));
+                    Interlocked.Increment(ref _totalRetries);
 
-                    (context["Logger"] as Lazy<ILog>)?.Value.Warn(
-                        $"Deadlock exception encountered during '{context["EntityTypeName"]}' entity persistence. Retrying transaction (retry #{retryNumber} of {RetryCount} after {waitDuration.TotalMilliseconds:N0}ms))...");
+                    var logger = context["Logger"] as ILog;
 
-                    return waitDuration;
-                },
-                onRetry: (res, ts, context) => { });
+                    if (logger?.IsWarnEnabled == true)
+                    {
+                        logger.Warn(
+                            $"Deadlock exception encountered during '{context["EntityTypeName"]}' entity persistence. Retrying transaction (attempt #{retryAttempt + 1} of {RetryCount + 1} after {ts.TotalMilliseconds:N0}ms). {_totalRetries} total retries...");
+                    }
+                });
     }
 
     /// <summary>

diff --git a/Application/EdFi.Ods.Common/Infrastructure/Repositories/UpdateEntity.cs b/Application/EdFi.Ods.Common/Infrastructure/Repositories/UpdateEntity.cs
@@ -42,19 +42,13 @@ await DeadlockPolicyHelper.RetryPolicy.ExecuteAsync(
                         try
                         {
                             await Session.UpdateAsync(persistentEntity, cancellationToken);
+                            await trans.CommitAsync(cancellationToken);
                         }
                         catch (Exception)
                         {
                             await trans.RollbackAsync(cancellationToken);
                             throw;
                         }
-                        finally
-                        {
-                            if (!trans.WasRolledBack)
-                            {
-                                await trans.CommitAsync(cancellationToken);
-                            }
-                        }
                     },
                     _retryPolicyContextData);
             }