What about 6 retries and not 3?

.vscode/settings.json

@@ -28,6 +28,7 @@
     "ptcc",
     "Rebinder",
     "stylesheet",
+    "timespan",
     "upserted",
     "USFM"
   ],

src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs

@@ -158,9 +158,26 @@ public static IMachineBuilder AddClearMLService(this IMachineBuilder builder, st
         builder
             .Services.AddHttpClient("ClearML")
             .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString!))
-            // Add retry policy; fail after approx. 2 + 4 + 8 = 14 seconds
             .AddTransientHttpErrorPolicy(b =>
-                b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)))
+                b.WaitAndRetryAsync(
+                    7,
+                    retryAttempt => TimeSpan.FromSeconds(2 * retryAttempt), // total 56, less than the 1 minute limit
+                    onRetryAsync: (outcome, timespan, retryAttempt, context) =>
+                    {
+                        if (retryAttempt < 3)
+                            return Task.CompletedTask;
+                        // Log the retry attempt
+                        var serviceProvider = builder.Services.BuildServiceProvider();
+                        var logger = serviceProvider.GetService<ILogger<ClearMLService>>();
+                        logger?.LogInformation(
+                            "Retry {RetryAttempt} encountered an error. Waiting {Timespan} before next retry. Error: {ErrorMessage}",
+                            retryAttempt,
+                            timespan,
+                            outcome.Exception?.Message
+                        );
+                        return Task.CompletedTask;
+                    }
+                )
             );
 
         builder.Services.AddSingleton<IClearMLService, ClearMLService>();

src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs

@@ -7,7 +7,10 @@ public static IServalBuilder AddWebhooks(this IServalBuilder builder)
         builder
             .Services.AddHttpClient<WebhookJob>()
             .AddTransientHttpErrorPolicy(b =>
-                b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)))
+                b.WaitAndRetryAsync(
+                    7,
+                    retryAttempt => TimeSpan.FromSeconds(2 * retryAttempt) // total 56, less than the 1 minute limit
+                )
             );
         builder.Services.AddScoped<IWebhookService, WebhookService>();
         return builder;