diff --git a/.vscode/settings.json b/.vscode/settings.json index 4c5aadb3..cbe0a073 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -28,6 +28,7 @@ "ptcc", "Rebinder", "stylesheet", + "timespan", "upserted", "USFM" ], diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs index 5a577cb5..19f72185 100644 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs @@ -158,9 +158,26 @@ public static IMachineBuilder AddClearMLService(this IMachineBuilder builder, st builder .Services.AddHttpClient("ClearML") .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString!)) - // Add retry policy; fail after approx. 2 + 4 + 8 = 14 seconds .AddTransientHttpErrorPolicy(b => - b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))) + b.WaitAndRetryAsync( + 7, + retryAttempt => TimeSpan.FromSeconds(2 * retryAttempt), // total 56, less than the 1 minute limit + onRetryAsync: (outcome, timespan, retryAttempt, context) => + { + if (retryAttempt < 3) + return Task.CompletedTask; + // Log the retry attempt + var serviceProvider = builder.Services.BuildServiceProvider(); + var logger = serviceProvider.GetService>(); + logger?.LogInformation( + "Retry {RetryAttempt} encountered an error. Waiting {Timespan} before next retry. Error: {ErrorMessage}", + retryAttempt, + timespan, + outcome.Exception?.Message + ); + return Task.CompletedTask; + } + ) ); builder.Services.AddSingleton(); diff --git a/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs b/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs index 129804e3..2c2f8503 100644 --- a/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs +++ b/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs @@ -7,7 +7,10 @@ public static IServalBuilder AddWebhooks(this IServalBuilder builder) builder .Services.AddHttpClient() .AddTransientHttpErrorPolicy(b => - b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))) + b.WaitAndRetryAsync( + 7, + retryAttempt => TimeSpan.FromSeconds(2 * retryAttempt) // total 56, less than the 1 minute limit + ) ); builder.Services.AddScoped(); return builder;