Fixed Polly retry logic for deadlocks.

Ed-Fi-Alliance-OSS · Nov 7, 2024 · 77fd9de · 77fd9de
1 parent 8703fa8
commit 77fd9de
Show file tree

Hide file tree

Showing 5 changed files with 18 additions and 28 deletions.
diff --git a/Application/EdFi.Ods.Common/EdFi.Ods.Common.csproj b/Application/EdFi.Ods.Common/EdFi.Ods.Common.csproj
@@ -38,6 +38,7 @@
     <PackageReference Include="NHibernate" Version="5.5.2" />
     <PackageReference Include="Npgsql" Version="8.0.3" />
     <PackageReference Include="Polly" Version="8.4.1" />
+    <PackageReference Include="Polly.Contrib.WaitAndRetry" Version="1.1.1" />
     <PackageReference Include="Sandwych.QuickGraph.Core" Version="1.0.0" />
     <PackageReference Include="Standart.Hash.xxHash" Version="4.0.5" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />

diff --git a/Application/EdFi.Ods.Common/Infrastructure/Interceptors/EdFiOdsInterceptor.cs b/Application/EdFi.Ods.Common/Infrastructure/Interceptors/EdFiOdsInterceptor.cs
@@ -4,12 +4,7 @@
 // See the LICENSE and NOTICES files in the project root for more information.
 
 using System;
-using System.Text.RegularExpressions;
-using EdFi.Ods.Common.Conventions;
-using EdFi.Ods.Common.Security.CustomViewBased;
-using log4net;
 using NHibernate;
-using NHibernate.SqlCommand;
 
 namespace EdFi.Ods.Common.Infrastructure.Interceptors
 {

diff --git a/Application/EdFi.Ods.Common/Infrastructure/Repositories/CreateEntity.cs b/Application/EdFi.Ods.Common/Infrastructure/Repositories/CreateEntity.cs
@@ -116,25 +116,19 @@ await DeadlockPolicyHelper.RetryPolicy.ExecuteAsync(
                         try
                         {
                             await Session.SaveAsync(entity, cancellationToken);
+                            await trans.CommitAsync(cancellationToken);
                         }
                         catch (Exception)
                         {
                             await trans.RollbackAsync(cancellationToken);
                             throw;
                         }
-                        finally
-                        {
-                            if (!trans.WasRolledBack)
-                            {
-                                await trans.CommitAsync(cancellationToken);
-                            }
-                        }
                     },
                     _retryPolicyContextData);
 
                 bool IdHasValue()
                 {
-                    return !entity.Id.Equals(default(Guid));
+                    return !entity.Id.Equals(default);
                 }
             }
         }

diff --git a/Application/EdFi.Ods.Common/Infrastructure/Repositories/DeadlockPolicyHelper.cs b/Application/EdFi.Ods.Common/Infrastructure/Repositories/DeadlockPolicyHelper.cs
@@ -4,10 +4,12 @@
 // See the LICENSE and NOTICES files in the project root for more information.
 
 using System;
+using System.Threading;
 using log4net;
 using Microsoft.Data.SqlClient;
 using Npgsql;
 using Polly;
+using Polly.Contrib.WaitAndRetry;
 using Polly.Retry;
 
 namespace EdFi.Ods.Common.Infrastructure.Repositories;
@@ -19,21 +21,25 @@ public static class DeadlockPolicyHelper
     public static int RetryCount = 5;
     public static int RetryStartingDelayMilliseconds = 100;
 
+    private static int _totalRetries = 0;
+
     static DeadlockPolicyHelper()
     {
         RetryPolicy = Policy.Handle<Exception>(ShouldRetry)
             .WaitAndRetryAsync(
-                RetryCount,
-                (retryNumber, context) =>
+                Backoff.ExponentialBackoff(TimeSpan.FromMilliseconds(RetryStartingDelayMilliseconds), RetryCount),
+                onRetry: (result, ts, retryAttempt, context) =>
                 {
-                    var waitDuration = TimeSpan.FromMilliseconds(RetryStartingDelayMilliseconds * (Math.Pow(2, retryNumber)));
+                    Interlocked.Increment(ref _totalRetries);
 
-                    (context["Logger"] as Lazy<ILog>)?.Value.Warn(
-                        $"Deadlock exception encountered during '{context["EntityTypeName"]}' entity persistence. Retrying transaction (retry #{retryNumber} of {RetryCount} after {waitDuration.TotalMilliseconds:N0}ms))...");
+                    var logger = context["Logger"] as ILog;
 
-                    return waitDuration;
-                },
-                onRetry: (res, ts, context) => { });
+                    if (logger?.IsWarnEnabled == true)
+                    {
+                        logger.Warn(
+                            $"Deadlock exception encountered during '{context["EntityTypeName"]}' entity persistence. Retrying transaction (attempt #{retryAttempt + 1} of {RetryCount + 1} after {ts.TotalMilliseconds:N0}ms). {_totalRetries} total retries...");
+                    }
+                });
     }
 
     /// <summary>

diff --git a/Application/EdFi.Ods.Common/Infrastructure/Repositories/UpdateEntity.cs b/Application/EdFi.Ods.Common/Infrastructure/Repositories/UpdateEntity.cs
@@ -48,19 +48,13 @@ await DeadlockPolicyHelper.RetryPolicy.ExecuteAsync(
                         try
                         {
                             await Session.UpdateAsync(persistentEntity, cancellationToken);
+                            await trans.CommitAsync(cancellationToken);
                         }
                         catch (Exception)
                         {
                             await trans.RollbackAsync(cancellationToken);
                             throw;
                         }
-                        finally
-                        {
-                            if (!trans.WasRolledBack)
-                            {
-                                await trans.CommitAsync(cancellationToken);
-                            }
-                        }
                     },
                     _retryPolicyContextData);
             }