From 9b5d2c9f7eddcd41b178a3adc9974da3a39e1061 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Tue, 29 Oct 2024 17:51:06 -0400 Subject: [PATCH 1/3] What about 6 retries and not 3? --- .vscode/settings.json | 1 + .../IMachineBuilderExtensions.cs | 21 +++++++++++++++++-- .../Configuration/IServalBuilderExtensions.cs | 6 +++++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 4c5aadb3..cbe0a073 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -28,6 +28,7 @@ "ptcc", "Rebinder", "stylesheet", + "timespan", "upserted", "USFM" ], diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs index 5a577cb5..ca96c40b 100644 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs @@ -158,9 +158,26 @@ public static IMachineBuilder AddClearMLService(this IMachineBuilder builder, st builder .Services.AddHttpClient("ClearML") .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString!)) - // Add retry policy; fail after approx. 2 + 4 + 8 = 14 seconds .AddTransientHttpErrorPolicy(b => - b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))) + b.WaitAndRetryAsync( + 6, + retryAttempt => TimeSpan.FromSeconds(new[] { 2, 4, 8, 12, 14, 16 }[retryAttempt]), // total 56 seconds, under the 1 minute https limit + onRetryAsync: (outcome, timespan, retryAttempt, context) => + { + if (retryAttempt < 3) + return Task.CompletedTask; + // Log the retry attempt + var serviceProvider = builder.Services.BuildServiceProvider(); + var logger = serviceProvider.GetService>(); + logger?.LogInformation( + "Retry {RetryAttempt} encountered an error. Waiting {Timespan} before next retry. Error: {ErrorMessage}", + retryAttempt, + timespan, + outcome.Exception?.Message + ); + return Task.CompletedTask; + } + ) ); builder.Services.AddSingleton(); diff --git a/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs b/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs index 129804e3..20c728c4 100644 --- a/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs +++ b/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs @@ -6,8 +6,12 @@ public static IServalBuilder AddWebhooks(this IServalBuilder builder) { builder .Services.AddHttpClient() + // Add retry policy; fail after approx. 1.5 + 2.25 ... 1.5^6 ~= 31 seconds .AddTransientHttpErrorPolicy(b => - b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))) + b.WaitAndRetryAsync( + 6, + retryAttempt => TimeSpan.FromSeconds(new[] { 2, 4, 8, 12, 14, 16 }[retryAttempt]) // total 56 seconds, under the 1 minute https limit + ) ); builder.Services.AddScoped(); return builder; From 43a5d558df5227fd1858273a63395665ee9a0741 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Wed, 30 Oct 2024 14:11:53 -0400 Subject: [PATCH 2/3] comment should have been removed. --- .../Serval.Webhooks/Configuration/IServalBuilderExtensions.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs b/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs index 20c728c4..b42a66d4 100644 --- a/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs +++ b/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs @@ -6,7 +6,6 @@ public static IServalBuilder AddWebhooks(this IServalBuilder builder) { builder .Services.AddHttpClient() - // Add retry policy; fail after approx. 1.5 + 2.25 ... 1.5^6 ~= 31 seconds .AddTransientHttpErrorPolicy(b => b.WaitAndRetryAsync( 6, From d992401928761e9234190d9cdd924385d1feb3cc Mon Sep 17 00:00:00 2001 From: John Lambert Date: Wed, 30 Oct 2024 14:20:23 -0400 Subject: [PATCH 3/3] linear backoff --- .../Configuration/IMachineBuilderExtensions.cs | 4 ++-- .../Serval.Webhooks/Configuration/IServalBuilderExtensions.cs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs index ca96c40b..19f72185 100644 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs @@ -160,8 +160,8 @@ public static IMachineBuilder AddClearMLService(this IMachineBuilder builder, st .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString!)) .AddTransientHttpErrorPolicy(b => b.WaitAndRetryAsync( - 6, - retryAttempt => TimeSpan.FromSeconds(new[] { 2, 4, 8, 12, 14, 16 }[retryAttempt]), // total 56 seconds, under the 1 minute https limit + 7, + retryAttempt => TimeSpan.FromSeconds(2 * retryAttempt), // total 56, less than the 1 minute limit onRetryAsync: (outcome, timespan, retryAttempt, context) => { if (retryAttempt < 3) diff --git a/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs b/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs index b42a66d4..2c2f8503 100644 --- a/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs +++ b/src/Serval/src/Serval.Webhooks/Configuration/IServalBuilderExtensions.cs @@ -8,8 +8,8 @@ public static IServalBuilder AddWebhooks(this IServalBuilder builder) .Services.AddHttpClient() .AddTransientHttpErrorPolicy(b => b.WaitAndRetryAsync( - 6, - retryAttempt => TimeSpan.FromSeconds(new[] { 2, 4, 8, 12, 14, 16 }[retryAttempt]) // total 56 seconds, under the 1 minute https limit + 7, + retryAttempt => TimeSpan.FromSeconds(2 * retryAttempt) // total 56, less than the 1 minute limit ) ); builder.Services.AddScoped();