What about 6 retries and not 3?

sillsdev · Oct 30, 2024 · 9b5d2c9 · 9b5d2c9
1 parent b6b9ae4
commit 9b5d2c9
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 3 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -28,6 +28,7 @@
     "ptcc",
     "Rebinder",
     "stylesheet",
+    "timespan",
     "upserted",
     "USFM"
   ],

diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs
@@ -158,9 +158,26 @@ public static IMachineBuilder AddClearMLService(this IMachineBuilder builder, st
         builder
             .Services.AddHttpClient("ClearML")
             .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString!))
-            // Add retry policy; fail after approx. 2 + 4 + 8 = 14 seconds
             .AddTransientHttpErrorPolicy(b =>
-                b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)))
+                b.WaitAndRetryAsync(
+                    6,
+                    retryAttempt => TimeSpan.FromSeconds(new[] { 2, 4, 8, 12, 14, 16 }[retryAttempt]), // total 56 seconds, under the 1 minute https limit
+                    onRetryAsync: (outcome, timespan, retryAttempt, context) =>
+                    {
+                        if (retryAttempt < 3)
+                            return Task.CompletedTask;
+                        // Log the retry attempt
+                        var serviceProvider = builder.Services.BuildServiceProvider();
+                        var logger = serviceProvider.GetService<ILogger<ClearMLService>>();
+                        logger?.LogInformation(
+                            "Retry {RetryAttempt} encountered an error. Waiting {Timespan} before next retry. Error: {ErrorMessage}",
+                            retryAttempt,
+                            timespan,
+                            outcome.Exception?.Message
+                        );
+                        return Task.CompletedTask;
+                    }
+                )
             );
 
         builder.Services.AddSingleton<IClearMLService, ClearMLService>();

diff --git a/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs b/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs
@@ -6,8 +6,12 @@ public static IServalBuilder AddWebhooks(this IServalBuilder builder)
     {
         builder
             .Services.AddHttpClient<WebhookJob>()
+            // Add retry policy; fail after approx. 1.5 + 2.25 ... 1.5^6 ~= 31 seconds
             .AddTransientHttpErrorPolicy(b =>
-                b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)))
+                b.WaitAndRetryAsync(
+                    6,
+                    retryAttempt => TimeSpan.FromSeconds(new[] { 2, 4, 8, 12, 14, 16 }[retryAttempt]) // total 56 seconds, under the 1 minute https limit
+                )
             );
         builder.Services.AddScoped<IWebhookService, WebhookService>();
         return builder;