From 6362191a6a622c74632d791b63020941e0070f89 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 20 Jun 2024 11:56:42 -0400 Subject: [PATCH 1/9] retry hangfire jobs - https://github.com/sillsdev/machine/issues/158 * Fixed auto-retry as per this forum post: https://discuss.hangfire.io/t/recurring-jobs-do-not-automatically-get-retried-after-application-crash-net-core-service/9160 * MongoDB can't handle documents greater than 16MB * Treat messages from one id as a group * Kill failing messages over 4 days old * Make outbox truly generic, handling multiple queues * Ensure globally ordered outbox messages * Add "MoveAsync" to SharedStorage * Refactor saving pretranslations file --- .../IMachineBuilderExtensions.cs | 25 ++ .../IServiceCollectionExtensions.cs | 2 + .../Configuration/MessageOutboxOptions.cs | 8 + src/SIL.Machine.AspNetCore/Models/Outbox.cs | 28 ++ .../Models/OutboxMessage.cs | 14 + .../Services/ClearMLMonitorService.cs | 55 ++-- .../Services/HangfireBuildJob.cs | 51 +++- .../Services/IFileStorage.cs | 1 + .../Services/IMessageOutboxService.cs | 12 + .../Services/IOutboxMessageHandler.cs | 9 + .../Services/IPlatformService.cs | 6 +- .../Services/ISharedFileService.cs | 1 + .../Services/InMemoryStorage.cs | 9 + .../Services/LocalStorage.cs | 9 + .../Services/MessageOutboxDeliveryService.cs | 142 +++++++++ .../Services/MessageOutboxService.cs | 52 ++++ .../Services/NmtEngineService.cs | 14 +- .../Services/NmtPreprocessBuildJob.cs | 13 +- .../Services/PostprocessBuildJob.cs | 60 ++-- .../Services/PreprocessBuildJob.cs | 3 +- .../Services/S3FileStorage.cs | 21 ++ .../Services/ServalPlatformOutboxHandler.cs | 119 ++++++++ .../Services/ServalPlatformService.cs | 84 +++--- .../Services/SharedFileService.cs | 5 + .../Services/SmtTransferBuildJob.cs | 157 ++++++++++ .../Services/SmtTransferEngineService.cs | 40 +-- .../SmtTransferPostprocessBuildJob.cs | 12 +- .../Services/SmtTransferTrainBuildJob.cs | 3 +- .../Program.cs | 2 +- src/SIL.Machine.Serval.JobServer/Program.cs | 2 +- .../SIL.Machine.AspNetCore.Tests.csproj | 1 + .../Services/InMemoryStorageTests.cs | 16 + .../Services/LocalStorageTests.cs | 17 ++ .../MessageOutboxDeliveryServiceTests.cs | 277 ++++++++++++++++++ .../Services/NmtEngineServiceTests.cs | 3 + .../Services/PreprocessBuildJobTests.cs | 2 + .../Services/SmtTransferEngineServiceTests.cs | 4 + 37 files changed, 1142 insertions(+), 137 deletions(-) create mode 100644 src/SIL.Machine.AspNetCore/Configuration/MessageOutboxOptions.cs create mode 100644 src/SIL.Machine.AspNetCore/Models/Outbox.cs create mode 100644 src/SIL.Machine.AspNetCore/Models/OutboxMessage.cs create mode 100644 src/SIL.Machine.AspNetCore/Services/IMessageOutboxService.cs create mode 100644 src/SIL.Machine.AspNetCore/Services/IOutboxMessageHandler.cs create mode 100644 src/SIL.Machine.AspNetCore/Services/MessageOutboxDeliveryService.cs create mode 100644 src/SIL.Machine.AspNetCore/Services/MessageOutboxService.cs create mode 100644 src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxHandler.cs create mode 100644 src/SIL.Machine.AspNetCore/Services/SmtTransferBuildJob.cs create mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxDeliveryServiceTests.cs diff --git a/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions.cs b/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions.cs index 6183c476d..b1f3212bc 100644 --- a/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions.cs +++ b/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions.cs @@ -49,6 +49,21 @@ public static IMachineBuilder AddClearMLOptions(this IMachineBuilder builder, IC return builder; } + public static IMachineBuilder AddMessageOutboxOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddMessageOutboxOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + public static IMachineBuilder AddSharedFileOptions( this IMachineBuilder builder, Action configureOptions @@ -263,6 +278,8 @@ await c.Indexes.CreateOrUpdateAsync( ) ) ); + o.AddRepository("outbox_messages"); + o.AddRepository("outboxes"); } ); builder.Services.AddHealthChecks().AddMongoDb(connectionString!, name: "Mongo"); @@ -280,6 +297,11 @@ public static IMachineBuilder AddServalPlatformService( throw new InvalidOperationException("Serval connection string is required"); builder.Services.AddScoped(); + + builder.Services.AddSingleton(); + + builder.Services.AddSingleton(); + builder .Services.AddGrpcClient(o => { @@ -334,6 +356,9 @@ public static IMachineBuilder AddServalTranslationEngineService( options.Interceptors.Add(); }); builder.AddServalPlatformService(connectionString); + + builder.Services.AddHostedService(); + engineTypes ??= builder.Configuration?.GetSection("TranslationEngines").Get() ?? [TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt]; diff --git a/src/SIL.Machine.AspNetCore/Configuration/IServiceCollectionExtensions.cs b/src/SIL.Machine.AspNetCore/Configuration/IServiceCollectionExtensions.cs index 21642d82d..0dd26f291 100644 --- a/src/SIL.Machine.AspNetCore/Configuration/IServiceCollectionExtensions.cs +++ b/src/SIL.Machine.AspNetCore/Configuration/IServiceCollectionExtensions.cs @@ -28,6 +28,7 @@ public static IMachineBuilder AddMachine(this IServiceCollection services, IConf builder.AddSmtTransferEngineOptions(o => { }); builder.AddClearMLOptions(o => { }); builder.AddBuildJobOptions(o => { }); + builder.AddMessageOutboxOptions(o => { }); } else { @@ -36,6 +37,7 @@ public static IMachineBuilder AddMachine(this IServiceCollection services, IConf builder.AddSmtTransferEngineOptions(configuration.GetSection(SmtTransferEngineOptions.Key)); builder.AddClearMLOptions(configuration.GetSection(ClearMLOptions.Key)); builder.AddBuildJobOptions(configuration.GetSection(BuildJobOptions.Key)); + builder.AddMessageOutboxOptions(configuration.GetSection(MessageOutboxOptions.Key)); } return builder; } diff --git a/src/SIL.Machine.AspNetCore/Configuration/MessageOutboxOptions.cs b/src/SIL.Machine.AspNetCore/Configuration/MessageOutboxOptions.cs new file mode 100644 index 000000000..0b306a900 --- /dev/null +++ b/src/SIL.Machine.AspNetCore/Configuration/MessageOutboxOptions.cs @@ -0,0 +1,8 @@ +namespace SIL.Machine.AspNetCore.Configuration; + +public class MessageOutboxOptions +{ + public const string Key = "MessageOutbox"; + + public int MessageExpirationInHours { get; set; } = 48; +} diff --git a/src/SIL.Machine.AspNetCore/Models/Outbox.cs b/src/SIL.Machine.AspNetCore/Models/Outbox.cs new file mode 100644 index 000000000..8b7acdfb3 --- /dev/null +++ b/src/SIL.Machine.AspNetCore/Models/Outbox.cs @@ -0,0 +1,28 @@ +namespace SIL.Machine.AspNetCore.Models; + +public record Outbox : IEntity +{ + public string Id { get; set; } = ""; + + public int Revision { get; set; } + + public string Name { get; set; } = null!; + public int CurrentIndex { get; set; } + + public static async Task GetOutboxNextIndexAsync( + IRepository indexRepository, + string outboxName, + CancellationToken cancellationToken + ) + { + Outbox outbox = ( + await indexRepository.UpdateAsync( + i => i.Name == outboxName, + i => i.Inc(b => b.CurrentIndex, 1), + upsert: true, + cancellationToken: cancellationToken + ) + )!; + return outbox; + } +} diff --git a/src/SIL.Machine.AspNetCore/Models/OutboxMessage.cs b/src/SIL.Machine.AspNetCore/Models/OutboxMessage.cs new file mode 100644 index 000000000..f13d3a082 --- /dev/null +++ b/src/SIL.Machine.AspNetCore/Models/OutboxMessage.cs @@ -0,0 +1,14 @@ +namespace SIL.Machine.AspNetCore.Models; + +public record OutboxMessage : IEntity +{ + public string Id { get; set; } = ""; + public int Revision { get; set; } = 1; + public required int Index { get; set; } + public required string OutboxName { get; set; } + public required string Method { get; set; } + public required string GroupId { get; set; } + public required string? RequestContent { get; set; } + public DateTimeOffset Created { get; set; } = DateTimeOffset.UtcNow; + public int Attempts { get; set; } = 0; +} diff --git a/src/SIL.Machine.AspNetCore/Services/ClearMLMonitorService.cs b/src/SIL.Machine.AspNetCore/Services/ClearMLMonitorService.cs index 217a398e1..952a41254 100644 --- a/src/SIL.Machine.AspNetCore/Services/ClearMLMonitorService.cs +++ b/src/SIL.Machine.AspNetCore/Services/ClearMLMonitorService.cs @@ -4,6 +4,7 @@ public class ClearMLMonitorService( IServiceProvider services, IClearMLService clearMLService, ISharedFileService sharedFileService, + IDataAccessContext dataAccessContext, IOptionsMonitor clearMLOptions, IOptionsMonitor buildJobOptions, ILogger logger @@ -23,6 +24,7 @@ ILogger logger private readonly IClearMLService _clearMLService = clearMLService; private readonly ISharedFileService _sharedFileService = sharedFileService; + private readonly IDataAccessContext _dataAccessContext = dataAccessContext; private readonly ILogger _logger = logger; private readonly Dictionary _curBuildStatus = new(); @@ -225,17 +227,24 @@ private async Task TrainJobStartedAsync( CancellationToken cancellationToken = default ) { + bool success; IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(engineId, cancellationToken); await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) { - if (!await buildJobService.BuildJobStartedAsync(engineId, buildId, cancellationToken)) - return false; + success = await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + if (!await buildJobService.BuildJobStartedAsync(engineId, buildId, ct)) + return false; + await platformService.BuildStartedAsync(buildId, CancellationToken.None); + return true; + }, + cancellationToken: cancellationToken + ); } - await platformService.BuildStartedAsync(buildId, CancellationToken.None); - await UpdateTrainJobStatus(platformService, buildId, new ProgressStatus(0), 0, cancellationToken); _logger.LogInformation("Build started ({BuildId})", buildId); - return true; + return success; } private async Task TrainJobCompletedAsync( @@ -286,12 +295,18 @@ CancellationToken cancellationToken IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(engineId, cancellationToken); await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) { - await platformService.BuildFaultedAsync(buildId, message, cancellationToken); - await buildJobService.BuildJobFinishedAsync( - engineId, - buildId, - buildComplete: false, - CancellationToken.None + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + await platformService.BuildFaultedAsync(buildId, message, ct); + await buildJobService.BuildJobFinishedAsync( + engineId, + buildId, + buildComplete: false, + CancellationToken.None + ); + }, + cancellationToken: cancellationToken ); } _logger.LogError("Build faulted ({BuildId}). Error: {ErrorMessage}", buildId, message); @@ -316,12 +331,18 @@ CancellationToken cancellationToken IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(engineId, cancellationToken); await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) { - await platformService.BuildCanceledAsync(buildId, cancellationToken); - await buildJobService.BuildJobFinishedAsync( - engineId, - buildId, - buildComplete: false, - CancellationToken.None + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + await platformService.BuildCanceledAsync(buildId, ct); + await buildJobService.BuildJobFinishedAsync( + engineId, + buildId, + buildComplete: false, + CancellationToken.None + ); + }, + cancellationToken: cancellationToken ); } _logger.LogInformation("Build canceled ({BuildId})", buildId); diff --git a/src/SIL.Machine.AspNetCore/Services/HangfireBuildJob.cs b/src/SIL.Machine.AspNetCore/Services/HangfireBuildJob.cs index deb9219b6..515f88732 100644 --- a/src/SIL.Machine.AspNetCore/Services/HangfireBuildJob.cs +++ b/src/SIL.Machine.AspNetCore/Services/HangfireBuildJob.cs @@ -4,9 +4,10 @@ public abstract class HangfireBuildJob( IPlatformService platformService, IRepository engines, IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, IBuildJobService buildJobService, ILogger logger -) : HangfireBuildJob(platformService, engines, lockFactory, buildJobService, logger) +) : HangfireBuildJob(platformService, engines, lockFactory, dataAccessContext, buildJobService, logger) { public virtual Task RunAsync( string engineId, @@ -23,6 +24,7 @@ public abstract class HangfireBuildJob( IPlatformService platformService, IRepository engines, IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, IBuildJobService buildJobService, ILogger> logger ) @@ -30,6 +32,7 @@ ILogger> logger protected IPlatformService PlatformService { get; } = platformService; protected IRepository Engines { get; } = engines; protected IDistributedReaderWriterLockFactory LockFactory { get; } = lockFactory; + protected IDataAccessContext DataAccessContext { get; } = dataAccessContext; protected IBuildJobService BuildJobService { get; } = buildJobService; protected ILogger> Logger { get; } = logger; @@ -69,12 +72,18 @@ CancellationToken cancellationToken completionStatus = JobCompletionStatus.Canceled; await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) { - await PlatformService.BuildCanceledAsync(buildId, CancellationToken.None); - await BuildJobService.BuildJobFinishedAsync( - engineId, - buildId, - buildComplete: false, - CancellationToken.None + await DataAccessContext.WithTransactionAsync( + async (ct) => + { + await PlatformService.BuildCanceledAsync(buildId, CancellationToken.None); + await BuildJobService.BuildJobFinishedAsync( + engineId, + buildId, + buildComplete: false, + CancellationToken.None + ); + }, + cancellationToken: CancellationToken.None ); } Logger.LogInformation("Build canceled ({0})", buildId); @@ -86,8 +95,14 @@ await BuildJobService.BuildJobFinishedAsync( completionStatus = JobCompletionStatus.Restarting; await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) { - await PlatformService.BuildRestartingAsync(buildId, CancellationToken.None); - await BuildJobService.BuildJobRestartingAsync(engineId, buildId, CancellationToken.None); + await DataAccessContext.WithTransactionAsync( + async (ct) => + { + await PlatformService.BuildRestartingAsync(buildId, CancellationToken.None); + await BuildJobService.BuildJobRestartingAsync(engineId, buildId, CancellationToken.None); + }, + cancellationToken: CancellationToken.None + ); } throw; } @@ -101,12 +116,18 @@ await BuildJobService.BuildJobFinishedAsync( completionStatus = JobCompletionStatus.Faulted; await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) { - await PlatformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - await BuildJobService.BuildJobFinishedAsync( - engineId, - buildId, - buildComplete: false, - CancellationToken.None + await DataAccessContext.WithTransactionAsync( + async (ct) => + { + await PlatformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + await BuildJobService.BuildJobFinishedAsync( + engineId, + buildId, + buildComplete: false, + CancellationToken.None + ); + }, + cancellationToken: CancellationToken.None ); } Logger.LogError(0, e, "Build faulted ({0})", buildId); diff --git a/src/SIL.Machine.AspNetCore/Services/IFileStorage.cs b/src/SIL.Machine.AspNetCore/Services/IFileStorage.cs index 3417cffae..e910a426f 100644 --- a/src/SIL.Machine.AspNetCore/Services/IFileStorage.cs +++ b/src/SIL.Machine.AspNetCore/Services/IFileStorage.cs @@ -16,5 +16,6 @@ Task> ListFilesAsync( Task GetDownloadUrlAsync(string path, DateTime expiresAt, CancellationToken cancellationToken = default); + Task MoveAsync(string sourcePath, string destPath, CancellationToken cancellationToken = default); Task DeleteAsync(string path, bool recurse = false, CancellationToken cancellationToken = default); } diff --git a/src/SIL.Machine.AspNetCore/Services/IMessageOutboxService.cs b/src/SIL.Machine.AspNetCore/Services/IMessageOutboxService.cs new file mode 100644 index 000000000..997f1efd5 --- /dev/null +++ b/src/SIL.Machine.AspNetCore/Services/IMessageOutboxService.cs @@ -0,0 +1,12 @@ +namespace SIL.Machine.AspNetCore.Services; + +public interface IMessageOutboxService +{ + public Task EnqueueMessageAsync( + T method, + string groupId, + string? requestContent = null, + string? requestContentPath = null, + CancellationToken cancellationToken = default + ); +} diff --git a/src/SIL.Machine.AspNetCore/Services/IOutboxMessageHandler.cs b/src/SIL.Machine.AspNetCore/Services/IOutboxMessageHandler.cs new file mode 100644 index 000000000..a27ade201 --- /dev/null +++ b/src/SIL.Machine.AspNetCore/Services/IOutboxMessageHandler.cs @@ -0,0 +1,9 @@ +namespace SIL.Machine.AspNetCore.Services; + +public interface IOutboxMessageHandler +{ + public string Name { get; } + + public Task SendMessageAsync(OutboxMessage message, CancellationToken cancellationToken = default); + public Task CleanupMessageAsync(OutboxMessage message, CancellationToken cancellationToken = default); +} diff --git a/src/SIL.Machine.AspNetCore/Services/IPlatformService.cs b/src/SIL.Machine.AspNetCore/Services/IPlatformService.cs index 163d11517..c27188597 100644 --- a/src/SIL.Machine.AspNetCore/Services/IPlatformService.cs +++ b/src/SIL.Machine.AspNetCore/Services/IPlatformService.cs @@ -22,9 +22,5 @@ Task BuildCompletedAsync( Task BuildFaultedAsync(string buildId, string message, CancellationToken cancellationToken = default); Task BuildRestartingAsync(string buildId, CancellationToken cancellationToken = default); - Task InsertPretranslationsAsync( - string engineId, - IAsyncEnumerable pretranslations, - CancellationToken cancellationToken = default - ); + Task InsertPretranslationsAsync(string engineId, string path, CancellationToken cancellationToken = default); } diff --git a/src/SIL.Machine.AspNetCore/Services/ISharedFileService.cs b/src/SIL.Machine.AspNetCore/Services/ISharedFileService.cs index f082a79c2..ea17ad113 100644 --- a/src/SIL.Machine.AspNetCore/Services/ISharedFileService.cs +++ b/src/SIL.Machine.AspNetCore/Services/ISharedFileService.cs @@ -19,6 +19,7 @@ Task> ListFilesAsync( Task OpenWriteAsync(string path, CancellationToken cancellationToken = default); Task ExistsAsync(string path, CancellationToken cancellationToken = default); + Task MoveAsync(string sourcePath, string destPath, CancellationToken cancellationToken = default); Task DeleteAsync(string path, CancellationToken cancellationToken = default); } diff --git a/src/SIL.Machine.AspNetCore/Services/InMemoryStorage.cs b/src/SIL.Machine.AspNetCore/Services/InMemoryStorage.cs index 7deccb6e9..3824bfe0e 100644 --- a/src/SIL.Machine.AspNetCore/Services/InMemoryStorage.cs +++ b/src/SIL.Machine.AspNetCore/Services/InMemoryStorage.cs @@ -133,6 +133,15 @@ public async Task DeleteAsync(string path, bool recurse, CancellationToken cance } } + public Task MoveAsync(string sourcePath, string destPath, CancellationToken cancellationToken = default) + { + if (!_memoryStreams.TryGetValue(Normalize(sourcePath), out Entry? entry)) + throw new FileNotFoundException($"Unable to find file {sourcePath}"); + _memoryStreams[Normalize(destPath)] = entry; + _memoryStreams.Remove(Normalize(sourcePath), out _); + return Task.CompletedTask; + } + protected override void DisposeManagedResources() { foreach (Entry stream in _memoryStreams.Values) diff --git a/src/SIL.Machine.AspNetCore/Services/LocalStorage.cs b/src/SIL.Machine.AspNetCore/Services/LocalStorage.cs index 38e9049bd..42c573c38 100644 --- a/src/SIL.Machine.AspNetCore/Services/LocalStorage.cs +++ b/src/SIL.Machine.AspNetCore/Services/LocalStorage.cs @@ -59,6 +59,15 @@ public Task OpenWriteAsync(string path, CancellationToken cancellationTo return Task.FromResult(File.OpenWrite(pathUri.LocalPath)); } + public Task MoveAsync(string sourcePath, string destPath, CancellationToken cancellationToken = default) + { + Uri sourcePathUri = new(_basePath, Normalize(sourcePath)); + Uri destPathUri = new(_basePath, Normalize(destPath)); + Directory.CreateDirectory(Path.GetDirectoryName(destPathUri.LocalPath)!); + File.Move(sourcePathUri.LocalPath, destPathUri.LocalPath); + return Task.CompletedTask; + } + public async Task DeleteAsync(string path, bool recurse, CancellationToken cancellationToken = default) { Uri pathUri = new(_basePath, Normalize(path)); diff --git a/src/SIL.Machine.AspNetCore/Services/MessageOutboxDeliveryService.cs b/src/SIL.Machine.AspNetCore/Services/MessageOutboxDeliveryService.cs new file mode 100644 index 000000000..08cd891b3 --- /dev/null +++ b/src/SIL.Machine.AspNetCore/Services/MessageOutboxDeliveryService.cs @@ -0,0 +1,142 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class MessageOutboxDeliveryService( + IRepository messages, + IEnumerable outboxMessageHandlers, + MessageOutboxOptions options, + ILogger logger +) : BackgroundService +{ + private readonly IRepository _messages = messages; + private readonly Dictionary _outboxMessageHandlers = + outboxMessageHandlers.ToDictionary(o => o.Name); + + private readonly ILogger _logger = logger; + protected TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(10); + protected TimeSpan MessageExpiration { get; set; } = TimeSpan.FromHours(options.MessageExpirationInHours); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + using ISubscription subscription = await _messages.SubscribeAsync(e => true); + while (true) + { + await subscription.WaitForChangeAsync(timeout: Timeout, cancellationToken: stoppingToken); + if (stoppingToken.IsCancellationRequested) + break; + await ProcessMessagesAsync(); + } + } + + protected async Task ProcessMessagesAsync(CancellationToken cancellationToken = default) + { + bool anyMessages = await _messages.ExistsAsync(m => true); + if (!anyMessages) + return; + + IReadOnlyList messages = await _messages.GetAllAsync(); + + IEnumerable> messageGroups = messages.GroupBy( + m => new { m.GroupId, m.OutboxName }, + m => m, + (key, element) => element.OrderBy(m => m.Index).ToList() + ); + + foreach (List messageGroup in messageGroups) + { + bool abortMessageGroup = false; + var outboxMessageHandler = _outboxMessageHandlers[messageGroup.First().OutboxName]; + foreach (OutboxMessage message in messageGroup) + { + try + { + await ProcessGroupMessagesAsync(message, outboxMessageHandler, cancellationToken); + } + catch (RpcException e) + { + switch (e.StatusCode) + { + case StatusCode.Unavailable: + case StatusCode.Unauthenticated: + case StatusCode.PermissionDenied: + case StatusCode.Cancelled: + _logger.LogWarning(e, "Platform Message sending failure: {statusCode}", e.StatusCode); + return; + case StatusCode.Aborted: + case StatusCode.DeadlineExceeded: + case StatusCode.Internal: + case StatusCode.ResourceExhausted: + case StatusCode.Unknown: + abortMessageGroup = !await CheckIfFinalMessageAttempt(message, e); + break; + case StatusCode.InvalidArgument: + default: + // log error + await PermanentlyFailedMessage(message, e); + break; + } + } + catch (Exception e) + { + await PermanentlyFailedMessage(message, e); + break; + } + if (abortMessageGroup) + break; + } + } + } + + async Task ProcessGroupMessagesAsync( + OutboxMessage message, + IOutboxMessageHandler outboxMessageHandler, + CancellationToken cancellationToken = default + ) + { + await outboxMessageHandler.SendMessageAsync(message, cancellationToken); + await _messages.DeleteAsync(message.Id); + await outboxMessageHandler.CleanupMessageAsync(message, cancellationToken); + } + + async Task CheckIfFinalMessageAttempt(OutboxMessage message, Exception e) + { + if (message.Created < DateTimeOffset.UtcNow.Subtract(MessageExpiration)) + { + await PermanentlyFailedMessage(message, e); + return true; + } + else + { + await LogFailedAttempt(message, e); + return false; + } + } + + async Task PermanentlyFailedMessage(OutboxMessage message, Exception e) + { + // log error + _logger.LogError( + e, + "Permanently failed to process message {message.Id}: {message.Method} with content {message.RequestContent} and error message: {e.Message}", + message.Id, + message.Method, + message.RequestContent, + e.Message + ); + await _messages.DeleteAsync(message.Id); + } + + async Task LogFailedAttempt(OutboxMessage message, Exception e) + { + // log error + await _messages.UpdateAsync(m => m.Id == message.Id, b => b.Inc(m => m.Attempts, 1)); + _logger.LogError( + e, + "Attempt {message.Attempts}. Failed to process message {message.Id}: {message.Method} with content {message.RequestContent} and error message: {e.Message}", + message.Attempts + 1, + message.Id, + message.Method, + message.RequestContent, + e.Message + ); + } +} diff --git a/src/SIL.Machine.AspNetCore/Services/MessageOutboxService.cs b/src/SIL.Machine.AspNetCore/Services/MessageOutboxService.cs new file mode 100644 index 000000000..158077af5 --- /dev/null +++ b/src/SIL.Machine.AspNetCore/Services/MessageOutboxService.cs @@ -0,0 +1,52 @@ +using MongoDB.Bson; + +namespace SIL.Machine.AspNetCore.Services; + +public class MessageOutboxService( + IRepository messageIndexes, + IRepository messages, + ISharedFileService sharedFileService +) : IMessageOutboxService +{ + private readonly IRepository _messageIndex = messageIndexes; + private readonly IRepository _messages = messages; + private readonly ISharedFileService _sharedFileService = sharedFileService; + protected int MaxDocumentSize { get; set; } = 1_000_000; + + public async Task EnqueueMessageAsync( + T method, + string groupId, + string? requestContent = null, + string? requestContentPath = null, + CancellationToken cancellationToken = default + ) + { + if (requestContent == null && requestContentPath == null) + { + throw new ArgumentException("Either requestContent or contentPath must be specified."); + } + if (requestContent is not null && requestContent.Length > MaxDocumentSize) + { + throw new ArgumentException( + $"The content is too large for request {method} with group ID {groupId}. " + + $"It is {requestContent.Length} bytes, but the maximum is {MaxDocumentSize} bytes." + ); + } + Outbox outbox = await Outbox.GetOutboxNextIndexAsync(_messageIndex, typeof(T).ToString(), cancellationToken); + OutboxMessage outboxMessage = new OutboxMessage + { + Id = ObjectId.GenerateNewId().ToString(), + Index = outbox.CurrentIndex, + OutboxName = typeof(T).ToString(), + Method = method?.ToString() ?? throw new ArgumentNullException(nameof(method)), + GroupId = groupId, + RequestContent = requestContent + }; + if (requestContentPath != null) + { + await _sharedFileService.MoveAsync(requestContentPath, $"outbox/{outboxMessage.Id}", cancellationToken); + } + await _messages.InsertAsync(outboxMessage, cancellationToken: cancellationToken); + return outboxMessage.Id; + } +} diff --git a/src/SIL.Machine.AspNetCore/Services/NmtEngineService.cs b/src/SIL.Machine.AspNetCore/Services/NmtEngineService.cs index 28af9ee9a..62d32e5d1 100644 --- a/src/SIL.Machine.AspNetCore/Services/NmtEngineService.cs +++ b/src/SIL.Machine.AspNetCore/Services/NmtEngineService.cs @@ -181,12 +181,16 @@ public bool IsLanguageNativeToModel(string language, out string internalCode) private async Task CancelBuildJobAsync(string engineId, CancellationToken cancellationToken) { - (string? buildId, BuildJobState jobState) = await _buildJobService.CancelBuildJobAsync( - engineId, - cancellationToken + string? buildId = null; + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + (buildId, BuildJobState jobState) = await _buildJobService.CancelBuildJobAsync(engineId, ct); + if (buildId is not null && jobState is BuildJobState.None) + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + }, + cancellationToken: cancellationToken ); - if (buildId is not null && jobState is BuildJobState.None) - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); return buildId is not null; } diff --git a/src/SIL.Machine.AspNetCore/Services/NmtPreprocessBuildJob.cs b/src/SIL.Machine.AspNetCore/Services/NmtPreprocessBuildJob.cs index 5ba4d99d1..64c51589a 100644 --- a/src/SIL.Machine.AspNetCore/Services/NmtPreprocessBuildJob.cs +++ b/src/SIL.Machine.AspNetCore/Services/NmtPreprocessBuildJob.cs @@ -4,12 +4,23 @@ public class NmtPreprocessBuildJob( IPlatformService platformService, IRepository engines, IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, ILogger logger, IBuildJobService buildJobService, ISharedFileService sharedFileService, ICorpusService corpusService, ILanguageTagService languageTagService -) : PreprocessBuildJob(platformService, engines, lockFactory, logger, buildJobService, sharedFileService, corpusService) +) + : PreprocessBuildJob( + platformService, + engines, + lockFactory, + dataAccessContext, + logger, + buildJobService, + sharedFileService, + corpusService + ) { private readonly ILanguageTagService _languageTagService = languageTagService; diff --git a/src/SIL.Machine.AspNetCore/Services/PostprocessBuildJob.cs b/src/SIL.Machine.AspNetCore/Services/PostprocessBuildJob.cs index 1bf7a4389..16e35d14b 100644 --- a/src/SIL.Machine.AspNetCore/Services/PostprocessBuildJob.cs +++ b/src/SIL.Machine.AspNetCore/Services/PostprocessBuildJob.cs @@ -4,14 +4,12 @@ public class PostprocessBuildJob( IPlatformService platformService, IRepository engines, IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, IBuildJobService buildJobService, ILogger logger, ISharedFileService sharedFileService -) : HangfireBuildJob<(int, double)>(platformService, engines, lockFactory, buildJobService, logger) +) : HangfireBuildJob<(int, double)>(platformService, engines, lockFactory, dataAccessContext, buildJobService, logger) { - private static readonly JsonSerializerOptions JsonSerializerOptions = - new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; - protected ISharedFileService SharedFileService { get; } = sharedFileService; protected override async Task DoWorkAsync( @@ -25,19 +23,33 @@ CancellationToken cancellationToken { (int corpusSize, double confidence) = data; - // The MT job has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + await PlatformService.InsertPretranslationsAsync( + engineId, + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) { - int additionalCorpusSize = await SaveModelAsync(engineId, buildId); - await PlatformService.BuildCompletedAsync( - buildId, - corpusSize + additionalCorpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None + await DataAccessContext.WithTransactionAsync( + async (ct) => + { + int additionalCorpusSize = await SaveModelAsync(engineId, buildId); + await PlatformService.BuildCompletedAsync( + buildId, + corpusSize + additionalCorpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + await BuildJobService.BuildJobFinishedAsync( + engineId, + buildId, + buildComplete: true, + CancellationToken.None + ); + }, + cancellationToken: CancellationToken.None ); - await BuildJobService.BuildJobFinishedAsync(engineId, buildId, buildComplete: true, CancellationToken.None); } Logger.LogInformation("Build completed ({0}).", buildId); @@ -69,26 +81,4 @@ JobCompletionStatus completionStatus Logger.LogWarning(e, "Unable to to delete job data for build {0}.", buildId); } } - - protected async Task InsertPretranslationsAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - await using Stream targetPretranslateStream = await SharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - JsonSerializerOptions, - cancellationToken - ) - .OfType(); - - await PlatformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } } diff --git a/src/SIL.Machine.AspNetCore/Services/PreprocessBuildJob.cs b/src/SIL.Machine.AspNetCore/Services/PreprocessBuildJob.cs index 37d8535de..3f07ecbe1 100644 --- a/src/SIL.Machine.AspNetCore/Services/PreprocessBuildJob.cs +++ b/src/SIL.Machine.AspNetCore/Services/PreprocessBuildJob.cs @@ -15,12 +15,13 @@ public PreprocessBuildJob( IPlatformService platformService, IRepository engines, IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, ILogger logger, IBuildJobService buildJobService, ISharedFileService sharedFileService, ICorpusService corpusService ) - : base(platformService, engines, lockFactory, buildJobService, logger) + : base(platformService, engines, lockFactory, dataAccessContext, buildJobService, logger) { _sharedFileService = sharedFileService; _corpusService = corpusService; diff --git a/src/SIL.Machine.AspNetCore/Services/S3FileStorage.cs b/src/SIL.Machine.AspNetCore/Services/S3FileStorage.cs index a9d265e9c..7466fe5cc 100644 --- a/src/SIL.Machine.AspNetCore/Services/S3FileStorage.cs +++ b/src/SIL.Machine.AspNetCore/Services/S3FileStorage.cs @@ -111,6 +111,27 @@ public async Task OpenWriteAsync(string path, CancellationToken cancella ); } + public async Task MoveAsync(string sourcePath, string destPath, CancellationToken cancellationToken = default) + { + CopyObjectRequest copyRequest = + new() + { + SourceBucket = _bucketName, + SourceKey = _basePath + Normalize(sourcePath), + DestinationBucket = _bucketName, + DestinationKey = _basePath + Normalize(destPath) + }; + CopyObjectResponse copyResponse = await _client.CopyObjectAsync(copyRequest, cancellationToken); + if (!copyResponse.HttpStatusCode.Equals(HttpStatusCode.OK)) + { + throw new HttpRequestException( + $"Received status code {copyResponse.HttpStatusCode} when attempting to copy {sourcePath} to {destPath}" + ); + } + + await DeleteAsync(sourcePath, cancellationToken: cancellationToken); + } + public async Task DeleteAsync(string path, bool recurse = false, CancellationToken cancellationToken = default) { DeleteObjectRequest request = new() { BucketName = _bucketName, Key = _basePath + Normalize(path) }; diff --git a/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxHandler.cs b/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxHandler.cs new file mode 100644 index 000000000..4d1b2636b --- /dev/null +++ b/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxHandler.cs @@ -0,0 +1,119 @@ +using Serval.Translation.V1; + +namespace SIL.Machine.AspNetCore.Services; + +public enum ServalPlatformMessageMethod +{ + BuildStarted, + BuildCompleted, + BuildCanceled, + BuildFaulted, + BuildRestarting, + InsertPretranslations, + IncrementTranslationEngineCorpusSize +} + +public class ServalPlatformOutboxHandler( + TranslationPlatformApi.TranslationPlatformApiClient client, + ISharedFileService sharedFileService, + ILogger logger +) : IOutboxMessageHandler +{ + private readonly TranslationPlatformApi.TranslationPlatformApiClient _client = client; + private readonly ISharedFileService _sharedFileService = sharedFileService; + private readonly ILogger _logger = logger; + private static readonly JsonSerializerOptions JsonSerializerOptions = + new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; + + private readonly string _name = typeof(ServalPlatformMessageMethod).ToString(); + public string Name => _name; + + public async Task SendMessageAsync(OutboxMessage message, CancellationToken cancellationToken = default) + { + ServalPlatformMessageMethod messageType = Enum.Parse(message.Method); + switch (messageType) + { + case ServalPlatformMessageMethod.BuildStarted: + await _client.BuildStartedAsync( + JsonSerializer.Deserialize(message.RequestContent!), + cancellationToken: cancellationToken + ); + break; + case ServalPlatformMessageMethod.BuildCompleted: + await _client.BuildCompletedAsync( + JsonSerializer.Deserialize(message.RequestContent!), + cancellationToken: cancellationToken + ); + break; + case ServalPlatformMessageMethod.BuildCanceled: + await _client.BuildCanceledAsync( + JsonSerializer.Deserialize(message.RequestContent!), + cancellationToken: cancellationToken + ); + break; + case ServalPlatformMessageMethod.BuildFaulted: + await _client.BuildFaultedAsync( + JsonSerializer.Deserialize(message.RequestContent!), + cancellationToken: cancellationToken + ); + break; + case ServalPlatformMessageMethod.BuildRestarting: + await _client.BuildRestartingAsync( + JsonSerializer.Deserialize(message.RequestContent!), + cancellationToken: cancellationToken + ); + break; + case ServalPlatformMessageMethod.InsertPretranslations: + + { + Stream targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"outbox/{message.Id}", + cancellationToken + ); + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + JsonSerializerOptions, + cancellationToken + ) + .OfType(); + IAsyncEnumerable requests = pretranslations.Select( + p => new InsertPretranslationRequest + { + EngineId = message.RequestContent!, + CorpusId = p.CorpusId, + TextId = p.TextId, + Refs = { p.Refs }, + Translation = p.Translation + } + ); + + using var call = _client.InsertPretranslations(cancellationToken: cancellationToken); + await foreach (var request in requests) + { + await call.RequestStream.WriteAsync(request, cancellationToken: cancellationToken); + } + await call.RequestStream.CompleteAsync(); + } + break; + case ServalPlatformMessageMethod.IncrementTranslationEngineCorpusSize: + await _client.IncrementTranslationEngineCorpusSizeAsync( + JsonSerializer.Deserialize(message.RequestContent!), + cancellationToken: cancellationToken + ); + break; + default: + _logger.LogWarning( + "Unknown method: {message.Method}. Deleting the message from the list.", + message.Method.ToString() + ); + break; + } + } + + public async Task CleanupMessageAsync(OutboxMessage message, CancellationToken cancellationToken = default) + { + if (await _sharedFileService.ExistsAsync($"outbox/{message.Id}", cancellationToken)) + await _sharedFileService.DeleteAsync($"outbox/{message.Id}", cancellationToken); + } +} diff --git a/src/SIL.Machine.AspNetCore/Services/ServalPlatformService.cs b/src/SIL.Machine.AspNetCore/Services/ServalPlatformService.cs index a882a234b..ad89471c6 100644 --- a/src/SIL.Machine.AspNetCore/Services/ServalPlatformService.cs +++ b/src/SIL.Machine.AspNetCore/Services/ServalPlatformService.cs @@ -2,14 +2,20 @@ namespace SIL.Machine.AspNetCore.Services; -public class ServalPlatformService(TranslationPlatformApi.TranslationPlatformApiClient client) : IPlatformService +public class ServalPlatformService( + TranslationPlatformApi.TranslationPlatformApiClient client, + IMessageOutboxService outboxService +) : IPlatformService { private readonly TranslationPlatformApi.TranslationPlatformApiClient _client = client; + private readonly IMessageOutboxService _outboxService = outboxService; public async Task BuildStartedAsync(string buildId, CancellationToken cancellationToken = default) { - await _client.BuildStartedAsync( - new BuildStartedRequest { BuildId = buildId }, + await _outboxService.EnqueueMessageAsync( + ServalPlatformMessageMethod.BuildStarted, + buildId, + JsonSerializer.Serialize(new BuildStartedRequest { BuildId = buildId }), cancellationToken: cancellationToken ); } @@ -21,37 +27,47 @@ public async Task BuildCompletedAsync( CancellationToken cancellationToken = default ) { - await _client.BuildCompletedAsync( - new BuildCompletedRequest - { - BuildId = buildId, - CorpusSize = trainSize, - Confidence = confidence - }, + await _outboxService.EnqueueMessageAsync( + ServalPlatformMessageMethod.BuildCompleted, + buildId, + JsonSerializer.Serialize( + new BuildCompletedRequest + { + BuildId = buildId, + CorpusSize = trainSize, + Confidence = confidence + } + ), cancellationToken: cancellationToken ); } public async Task BuildCanceledAsync(string buildId, CancellationToken cancellationToken = default) { - await _client.BuildCanceledAsync( - new BuildCanceledRequest { BuildId = buildId }, + await _outboxService.EnqueueMessageAsync( + ServalPlatformMessageMethod.BuildCanceled, + buildId, + JsonSerializer.Serialize(new BuildCanceledRequest { BuildId = buildId }), cancellationToken: cancellationToken ); } public async Task BuildFaultedAsync(string buildId, string message, CancellationToken cancellationToken = default) { - await _client.BuildFaultedAsync( - new BuildFaultedRequest { BuildId = buildId, Message = message }, + await _outboxService.EnqueueMessageAsync( + ServalPlatformMessageMethod.BuildFaulted, + buildId, + JsonSerializer.Serialize(new BuildFaultedRequest { BuildId = buildId, Message = message }), cancellationToken: cancellationToken ); } public async Task BuildRestartingAsync(string buildId, CancellationToken cancellationToken = default) { - await _client.BuildRestartingAsync( - new BuildRestartingRequest { BuildId = buildId }, + await _outboxService.EnqueueMessageAsync( + ServalPlatformMessageMethod.BuildRestarting, + buildId, + JsonSerializer.Serialize(new BuildRestartingRequest { BuildId = buildId }), cancellationToken: cancellationToken ); } @@ -71,11 +87,13 @@ public async Task UpdateBuildStatusAsync( if (queueDepth is not null) request.QueueDepth = queueDepth.Value; + // just try to send it - if it fails, it fails. await _client.UpdateBuildStatusAsync(request, cancellationToken: cancellationToken); } public async Task UpdateBuildStatusAsync(string buildId, int step, CancellationToken cancellationToken = default) { + // just try to send it - if it fails, it fails. await _client.UpdateBuildStatusAsync( new UpdateBuildStatusRequest { BuildId = buildId, Step = step }, cancellationToken: cancellationToken @@ -84,27 +102,17 @@ await _client.UpdateBuildStatusAsync( public async Task InsertPretranslationsAsync( string engineId, - IAsyncEnumerable pretranslations, + string path, CancellationToken cancellationToken = default ) { - using var call = _client.InsertPretranslations(cancellationToken: cancellationToken); - await foreach (Pretranslation? pretranslation in pretranslations) - { - await call.RequestStream.WriteAsync( - new InsertPretranslationRequest - { - EngineId = engineId, - CorpusId = pretranslation.CorpusId, - TextId = pretranslation.TextId, - Refs = { pretranslation.Refs }, - Translation = pretranslation.Translation - }, - cancellationToken - ); - } - await call.RequestStream.CompleteAsync(); - await call; + await _outboxService.EnqueueMessageAsync( + ServalPlatformMessageMethod.InsertPretranslations, + engineId, + requestContent: engineId, + requestContentPath: path, + cancellationToken: cancellationToken + ); } public async Task IncrementTrainSizeAsync( @@ -113,8 +121,12 @@ public async Task IncrementTrainSizeAsync( CancellationToken cancellationToken = default ) { - await _client.IncrementTranslationEngineCorpusSizeAsync( - new IncrementTranslationEngineCorpusSizeRequest { EngineId = engineId, Count = count }, + await _outboxService.EnqueueMessageAsync( + ServalPlatformMessageMethod.IncrementTranslationEngineCorpusSize, + engineId, + JsonSerializer.Serialize( + new IncrementTranslationEngineCorpusSizeRequest { EngineId = engineId, Count = count } + ), cancellationToken: cancellationToken ); } diff --git a/src/SIL.Machine.AspNetCore/Services/SharedFileService.cs b/src/SIL.Machine.AspNetCore/Services/SharedFileService.cs index b4244211e..f09b4951c 100644 --- a/src/SIL.Machine.AspNetCore/Services/SharedFileService.cs +++ b/src/SIL.Machine.AspNetCore/Services/SharedFileService.cs @@ -101,4 +101,9 @@ public Task ExistsAsync(string path, CancellationToken cancellationToken = { return _fileStorage.ExistsAsync(path, cancellationToken); } + + public Task MoveAsync(string sourcePath, string destPath, CancellationToken cancellationToken = default) + { + return _fileStorage.MoveAsync(sourcePath, destPath, cancellationToken); + } } diff --git a/src/SIL.Machine.AspNetCore/Services/SmtTransferBuildJob.cs b/src/SIL.Machine.AspNetCore/Services/SmtTransferBuildJob.cs new file mode 100644 index 000000000..21f16b324 --- /dev/null +++ b/src/SIL.Machine.AspNetCore/Services/SmtTransferBuildJob.cs @@ -0,0 +1,157 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferBuildJob( + IPlatformService platformService, + IRepository engines, + IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, + IBuildJobService buildJobService, + ILogger logger, + IRepository trainSegmentPairs, + ITruecaserFactory truecaserFactory, + ISmtModelFactory smtModelFactory, + ICorpusService corpusService +) + : HangfireBuildJob>( + platformService, + engines, + lockFactory, + dataAccessContext, + buildJobService, + logger + ) +{ + private readonly IRepository _trainSegmentPairs = trainSegmentPairs; + private readonly ITruecaserFactory _truecaserFactory = truecaserFactory; + private readonly ISmtModelFactory _smtModelFactory = smtModelFactory; + private readonly ICorpusService _corpusService = corpusService; + + protected override Task InitializeAsync( + string engineId, + string buildId, + IReadOnlyList data, + IDistributedReaderWriterLock @lock, + CancellationToken cancellationToken + ) + { + return _trainSegmentPairs.DeleteAllAsync(p => p.TranslationEngineRef == engineId, cancellationToken); + } + + protected override async Task DoWorkAsync( + string engineId, + string buildId, + IReadOnlyList data, + string? buildOptions, + IDistributedReaderWriterLock @lock, + CancellationToken cancellationToken + ) + { + await PlatformService.BuildStartedAsync(buildId, cancellationToken); + Logger.LogInformation("Build started ({0})", buildId); + var stopwatch = new Stopwatch(); + stopwatch.Start(); + + cancellationToken.ThrowIfCancellationRequested(); + + JsonObject? buildOptionsObject = null; + if (buildOptions is not null) + { + buildOptionsObject = JsonSerializer.Deserialize(buildOptions); + } + + var targetCorpora = new List(); + var parallelCorpora = new List(); + foreach (Corpus corpus in data) + { + ITextCorpus? sourceTextCorpus = _corpusService.CreateTextCorpora(corpus.SourceFiles).FirstOrDefault(); + ITextCorpus? targetTextCorpus = _corpusService.CreateTextCorpora(corpus.TargetFiles).FirstOrDefault(); + if (sourceTextCorpus is null || targetTextCorpus is null) + continue; + + targetCorpora.Add(targetTextCorpus); + parallelCorpora.Add(sourceTextCorpus.AlignRows(targetTextCorpus)); + + if ((bool?)buildOptionsObject?["use_key_terms"] ?? true) + { + ITextCorpus? sourceTermCorpus = _corpusService.CreateTermCorpora(corpus.SourceFiles).FirstOrDefault(); + ITextCorpus? targetTermCorpus = _corpusService.CreateTermCorpora(corpus.TargetFiles).FirstOrDefault(); + if (sourceTermCorpus is not null && targetTermCorpus is not null) + { + IParallelTextCorpus parallelKeyTermsCorpus = sourceTermCorpus.AlignRows(targetTermCorpus); + parallelCorpora.Add(parallelKeyTermsCorpus); + } + } + } + + IParallelTextCorpus parallelCorpus = parallelCorpora.Flatten(); + ITextCorpus targetCorpus = targetCorpora.Flatten(); + + var tokenizer = new LatinWordTokenizer(); + var detokenizer = new LatinWordDetokenizer(); + + using ITrainer smtModelTrainer = await _smtModelFactory.CreateTrainerAsync(engineId, tokenizer, parallelCorpus); + using ITrainer truecaseTrainer = await _truecaserFactory.CreateTrainerAsync(engineId, tokenizer, targetCorpus); + + cancellationToken.ThrowIfCancellationRequested(); + + var progress = new BuildProgress(PlatformService, buildId); + await smtModelTrainer.TrainAsync(progress, cancellationToken); + await truecaseTrainer.TrainAsync(cancellationToken: cancellationToken); + + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new OperationCanceledException(); + + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + cancellationToken.ThrowIfCancellationRequested(); + await smtModelTrainer.SaveAsync(CancellationToken.None); + await truecaseTrainer.SaveAsync(CancellationToken.None); + ITruecaser truecaser = await _truecaserFactory.CreateAsync(engineId); + IReadOnlyList segmentPairs = await _trainSegmentPairs.GetAllAsync( + p => p.TranslationEngineRef == engine.Id, + CancellationToken.None + ); + using ( + IInteractiveTranslationModel smtModel = await _smtModelFactory.CreateAsync( + engineId, + tokenizer, + detokenizer, + truecaser + ) + ) + { + foreach (TrainSegmentPair segmentPair in segmentPairs) + { + await smtModel.TrainSegmentAsync( + segmentPair.Source, + segmentPair.Target, + cancellationToken: CancellationToken.None + ); + } + } + + await DataAccessContext.WithTransactionAsync( + async (ct) => + { + await PlatformService.BuildCompletedAsync( + buildId, + smtModelTrainer.Stats.TrainCorpusSize + segmentPairs.Count, + smtModelTrainer.Stats.Metrics["bleu"] * 100.0, + CancellationToken.None + ); + await BuildJobService.BuildJobFinishedAsync( + engineId, + buildId, + buildComplete: true, + CancellationToken.None + ); + }, + cancellationToken: CancellationToken.None + ); + } + + stopwatch.Stop(); + Logger.LogInformation("Build completed in {0}s ({1})", stopwatch.Elapsed.TotalSeconds, buildId); + } +} diff --git a/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService.cs b/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService.cs index 941731cd8..c00910487 100644 --- a/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService.cs +++ b/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService.cs @@ -150,10 +150,10 @@ async Task TrainSubroutineAsync(SmtTransferEngineState state, CancellationToken } SmtTransferEngineState state = _stateService.Get(engineId); - if (engine.CurrentBuild?.JobState is BuildJobState.Active) - { - await _dataAccessContext.WithTransactionAsync( - async (ct) => + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + if (engine.CurrentBuild?.JobState is BuildJobState.Active) { await _trainSegmentPairs.InsertAsync( new TrainSegmentPair @@ -163,17 +163,17 @@ await _trainSegmentPairs.InsertAsync( Target = targetSegment, SentenceStart = sentenceStart }, - ct + CancellationToken.None ); + await TrainSubroutineAsync(state, CancellationToken.None); + } + else + { await TrainSubroutineAsync(state, ct); - }, - cancellationToken: CancellationToken.None - ); - } - else - { - await TrainSubroutineAsync(state, cancellationToken); - } + } + }, + cancellationToken: cancellationToken + ); state.IsUpdated = true; state.LastUsedTime = DateTime.Now; @@ -233,12 +233,16 @@ public bool IsLanguageNativeToModel(string language, out string internalCode) private async Task CancelBuildJobAsync(string engineId, CancellationToken cancellationToken) { - (string? buildId, BuildJobState jobState) = await _buildJobService.CancelBuildJobAsync( - engineId, - cancellationToken + string? buildId = null; + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + (buildId, BuildJobState jobState) = await _buildJobService.CancelBuildJobAsync(engineId, ct); + if (buildId is not null && jobState is BuildJobState.None) + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + }, + cancellationToken: cancellationToken ); - if (buildId is not null && jobState is BuildJobState.None) - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); return buildId is not null; } diff --git a/src/SIL.Machine.AspNetCore/Services/SmtTransferPostprocessBuildJob.cs b/src/SIL.Machine.AspNetCore/Services/SmtTransferPostprocessBuildJob.cs index 2065fe221..f2b6b438b 100644 --- a/src/SIL.Machine.AspNetCore/Services/SmtTransferPostprocessBuildJob.cs +++ b/src/SIL.Machine.AspNetCore/Services/SmtTransferPostprocessBuildJob.cs @@ -4,6 +4,7 @@ public class SmtTransferPostprocessBuildJob( IPlatformService platformService, IRepository engines, IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, IBuildJobService buildJobService, ILogger logger, ISharedFileService sharedFileService, @@ -11,7 +12,16 @@ public class SmtTransferPostprocessBuildJob( ISmtModelFactory smtModelFactory, ITruecaserFactory truecaserFactory, IOptionsMonitor options -) : PostprocessBuildJob(platformService, engines, lockFactory, buildJobService, logger, sharedFileService) +) + : PostprocessBuildJob( + platformService, + engines, + lockFactory, + dataAccessContext, + buildJobService, + logger, + sharedFileService + ) { private readonly ISmtModelFactory _smtModelFactory = smtModelFactory; private readonly ITruecaserFactory _truecaserFactory = truecaserFactory; diff --git a/src/SIL.Machine.AspNetCore/Services/SmtTransferTrainBuildJob.cs b/src/SIL.Machine.AspNetCore/Services/SmtTransferTrainBuildJob.cs index 5946b50a4..8f2640c3b 100644 --- a/src/SIL.Machine.AspNetCore/Services/SmtTransferTrainBuildJob.cs +++ b/src/SIL.Machine.AspNetCore/Services/SmtTransferTrainBuildJob.cs @@ -4,13 +4,14 @@ public class SmtTransferTrainBuildJob( IPlatformService platformService, IRepository engines, IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, IBuildJobService buildJobService, ILogger logger, ISharedFileService sharedFileService, ITruecaserFactory truecaserFactory, ISmtModelFactory smtModelFactory, ITransferEngineFactory transferEngineFactory -) : HangfireBuildJob(platformService, engines, lockFactory, buildJobService, logger) +) : HangfireBuildJob(platformService, engines, lockFactory, dataAccessContext, buildJobService, logger) { private static readonly JsonWriterOptions PretranslateWriterOptions = new() { Indented = true }; private static readonly JsonSerializerOptions JsonSerializerOptions = diff --git a/src/SIL.Machine.Serval.EngineServer/Program.cs b/src/SIL.Machine.Serval.EngineServer/Program.cs index 19fae81f9..e5f4d46bb 100644 --- a/src/SIL.Machine.Serval.EngineServer/Program.cs +++ b/src/SIL.Machine.Serval.EngineServer/Program.cs @@ -6,10 +6,10 @@ // Add services to the container. builder .Services.AddMachine(builder.Configuration) + .AddBuildJobService() .AddMongoDataAccess() .AddMongoHangfireJobClient() .AddServalTranslationEngineService() - .AddBuildJobService() .AddModelCleanupService() .AddClearMLService(); diff --git a/src/SIL.Machine.Serval.JobServer/Program.cs b/src/SIL.Machine.Serval.JobServer/Program.cs index 6328c45b2..d78bfed80 100644 --- a/src/SIL.Machine.Serval.JobServer/Program.cs +++ b/src/SIL.Machine.Serval.JobServer/Program.cs @@ -4,11 +4,11 @@ builder .Services.AddMachine(builder.Configuration) + .AddBuildJobService() .AddMongoDataAccess() .AddMongoHangfireJobClient() .AddHangfireJobServer() .AddServalPlatformService() - .AddBuildJobService() .AddClearMLService(); if (builder.Environment.IsDevelopment()) { diff --git a/tests/SIL.Machine.AspNetCore.Tests/SIL.Machine.AspNetCore.Tests.csproj b/tests/SIL.Machine.AspNetCore.Tests/SIL.Machine.AspNetCore.Tests.csproj index 5956eaa0d..f9f221847 100644 --- a/tests/SIL.Machine.AspNetCore.Tests/SIL.Machine.AspNetCore.Tests.csproj +++ b/tests/SIL.Machine.AspNetCore.Tests/SIL.Machine.AspNetCore.Tests.csproj @@ -17,6 +17,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/InMemoryStorageTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/InMemoryStorageTests.cs index 3b5052865..61a0cf3e5 100644 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/InMemoryStorageTests.cs +++ b/tests/SIL.Machine.AspNetCore.Tests/Services/InMemoryStorageTests.cs @@ -88,4 +88,20 @@ public async Task DeleteAsync() var files = await fs.ListFilesAsync("test", recurse: true); Assert.That(files, Is.Empty); } + + [Test] + public async Task MoveAsync() + { + using InMemoryStorage fs = new(); + using (StreamWriter sw = new(await fs.OpenWriteAsync("test1/file1"))) + { + string input = "Hello"; + sw.WriteLine(input); + } + await fs.MoveAsync("test1/file1", "test2/file1"); + var files = await fs.ListFilesAsync("test1", recurse: true); + Assert.That(files, Is.Empty); + files = await fs.ListFilesAsync("test2", recurse: true); + Assert.That(files, Is.EquivalentTo(new[] { "test2/file1" })); + } } diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/LocalStorageTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/LocalStorageTests.cs index 280a54bb1..8478c1ab6 100644 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/LocalStorageTests.cs +++ b/tests/SIL.Machine.AspNetCore.Tests/Services/LocalStorageTests.cs @@ -93,4 +93,21 @@ public async Task DeleteFileAsync() IReadOnlyCollection files = await fs.ListFilesAsync("test", recurse: true); Assert.That(files, Is.Empty); } + + [Test] + public async Task MoveAsync() + { + using var tmpDir = new TempDirectory("test"); + using LocalStorage fs = new(tmpDir.Path); + using (StreamWriter sw = new(await fs.OpenWriteAsync("test1/file1"))) + { + string input = "Hello"; + sw.WriteLine(input); + } + await fs.MoveAsync("test1/file1", "test2/file1"); + var files = await fs.ListFilesAsync("test1", recurse: true); + Assert.That(files, Is.Empty); + files = await fs.ListFilesAsync("test2", recurse: true); + Assert.That(files, Is.EquivalentTo(new[] { "test2/file1" })); + } } diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxDeliveryServiceTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxDeliveryServiceTests.cs new file mode 100644 index 000000000..739b65699 --- /dev/null +++ b/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxDeliveryServiceTests.cs @@ -0,0 +1,277 @@ +using Google.Protobuf.WellKnownTypes; +using Grpc.Core; +using NSubstitute.ExceptionExtensions; +using Serval.Translation.V1; + +namespace SIL.Machine.AspNetCore.Services; + +[TestFixture] +public class MessageOutboxDeliveryServiceTests +{ + [Test] + public async Task SendMessages() + { + var env = new TestEnvironment(); + env.AddStandardMessages(); + await env.MessageOutboxDeliveryService.ProcessMessagesOnceAsync(); + Received.InOrder(() => + { + env.Client.BuildStartedAsync(new BuildStartedRequest { BuildId = "A" }); + env.Client.BuildCompletedAsync(Arg.Any()); + env.Client.BuildStartedAsync(new BuildStartedRequest { BuildId = "B" }); + }); + } + + [Test] + public async Task SendMessages_Timeout() + { + var env = new TestEnvironment(); + env.AddStandardMessages(); + + // Timeout is long enough where the message attempt will be incremented, but not deleted. + env.ClientInternalFailure(); + await Task.Delay(100); + await env.MessageOutboxDeliveryService.ProcessMessagesOnceAsync(); + // Each group should try to send one message + Assert.That((await env.Messages.GetAsync(m => m.Id == "B"))!.Attempts, Is.EqualTo(1)); + Assert.That((await env.Messages.GetAsync(m => m.Id == "A"))!.Attempts, Is.EqualTo(0)); + Assert.That((await env.Messages.GetAsync(m => m.Id == "C"))!.Attempts, Is.EqualTo(1)); + + // with now shorter timeout, the messages will be deleted. + // 4 start build attempts, and only one build completed attempt + env.MessageOutboxDeliveryService.SetMessageExpiration(TimeSpan.FromMilliseconds(1)); + await env.MessageOutboxDeliveryService.ProcessMessagesOnceAsync(); + Assert.That(await env.Messages.ExistsAsync(m => true), Is.False); + var startCalls = env + .Client.ReceivedCalls() + .Count(x => x.GetMethodInfo().Name == nameof(env.Client.BuildStartedAsync)); + Assert.That(startCalls, Is.EqualTo(4)); + var completedCalls = env + .Client.ReceivedCalls() + .Count(x => x.GetMethodInfo().Name == nameof(env.Client.BuildCompletedAsync)); + Assert.That(completedCalls, Is.EqualTo(1)); + } + + [Test] + public async Task SendMessagesUnavailable_Failure() + { + var env = new TestEnvironment(); + env.AddStandardMessages(); + env.ClientUnavailableFailure(); + await env.MessageOutboxDeliveryService.ProcessMessagesOnceAsync(); + // Only the first group should be attempted - but not recorded as attempted + Assert.That((await env.Messages.GetAsync(m => m.Id == "B"))!.Attempts, Is.EqualTo(0)); + Assert.That((await env.Messages.GetAsync(m => m.Id == "A"))!.Attempts, Is.EqualTo(0)); + Assert.That((await env.Messages.GetAsync(m => m.Id == "C"))!.Attempts, Is.EqualTo(0)); + env.ClientInternalFailure(); + await env.MessageOutboxDeliveryService.ProcessMessagesOnceAsync(); + Assert.That((await env.Messages.GetAsync(m => m.Id == "B"))!.Attempts, Is.EqualTo(1)); + Assert.That((await env.Messages.GetAsync(m => m.Id == "A"))!.Attempts, Is.EqualTo(0)); + Assert.That((await env.Messages.GetAsync(m => m.Id == "C"))!.Attempts, Is.EqualTo(1)); + env.ClientNoFailure(); + await env.MessageOutboxDeliveryService.ProcessMessagesOnceAsync(); + Assert.That(await env.Messages.ExistsAsync(m => true), Is.False); + // 1 (unavailable) + 2 (internal) + 3 (success) = 6 calls + Assert.That(env.Client.ReceivedCalls().Count(), Is.EqualTo(6)); + } + + [Test] + public async Task LargeMessageContent() + { + var env = new TestEnvironment(); + // large max document size - message not saved to file + var fileIdC = await env.OutboxService.EnqueueMessageAsync( + method: ServalPlatformMessageMethod.BuildStarted, + groupId: "C", + requestContent: JsonSerializer.Serialize(new BuildStartedRequest { BuildId = "C" }), + cancellationToken: CancellationToken.None + ); + Assert.That(await env.SharedFileService.ExistsAsync($"outbox/{fileIdC}"), Is.False); + await env.MessageOutboxDeliveryService.ProcessMessagesOnceAsync(); + // small max document size - throws error + env.OutboxService.SetMaxDocumentSize(1); + Assert.ThrowsAsync( + () => + env.OutboxService.EnqueueMessageAsync( + method: ServalPlatformMessageMethod.BuildStarted, + groupId: "D", + requestContent: JsonSerializer.Serialize(new BuildStartedRequest { BuildId = "D" }), + cancellationToken: CancellationToken.None + ) + ); + } + + [Test] + public async Task PretranslateSaveFile() + { + var env = new TestEnvironment(); + // large max document size - message not saved to file + string pretranslationsPath = "build/C/pretranslations.json"; + using (StreamWriter sw = new(await env.SharedFileService.OpenWriteAsync(pretranslationsPath))) + { + sw.WriteLine("[]"); + } + var fileIdC = await env.OutboxService.EnqueueMessageAsync( + method: ServalPlatformMessageMethod.InsertPretranslations, + groupId: "C", + requestContent: "engineId", + requestContentPath: pretranslationsPath, + cancellationToken: CancellationToken.None + ); + Assert.That(await env.SharedFileService.ExistsAsync(pretranslationsPath), Is.False); + Assert.That(await env.SharedFileService.ExistsAsync($"outbox/{fileIdC}"), Is.True); + await env.MessageOutboxDeliveryService.ProcessMessagesOnceAsync(); + Assert.That(await env.SharedFileService.ExistsAsync($"outbox/{fileIdC}"), Is.False); + } + + public class TestMessageOutboxDeliveryService( + IRepository messages, + IEnumerable outboxMessageHandlers, + ILogger logger + ) : MessageOutboxDeliveryService(messages, outboxMessageHandlers, new MessageOutboxOptions(), logger) + { + public async Task ProcessMessagesOnceAsync() => await ProcessMessagesAsync(); + + public void SetMessageExpiration(TimeSpan messageExpiration) => MessageExpiration = messageExpiration; + } + + public class TestMessageOutboxService( + IRepository messageIndexes, + IRepository messages, + ISharedFileService sharedFileService + ) : MessageOutboxService(messageIndexes, messages, sharedFileService) + { + public void SetMaxDocumentSize(int maxDocumentSize) => MaxDocumentSize = maxDocumentSize; + } + + private class TestEnvironment : ObjectModel.DisposableBase + { + public MemoryRepository MessageIndexes { get; } + public MemoryRepository Messages { get; } + public TestMessageOutboxService OutboxService { get; } + public ISharedFileService SharedFileService { get; } + public TranslationPlatformApi.TranslationPlatformApiClient Client { get; } + public TestMessageOutboxDeliveryService MessageOutboxDeliveryService { get; } + public AsyncClientStreamingCall InsertPretranslationsCall { get; } + + public TestEnvironment() + { + MessageIndexes = new MemoryRepository(); + Messages = new MemoryRepository(); + SharedFileService = new SharedFileService(Substitute.For()); + OutboxService = new TestMessageOutboxService(MessageIndexes, Messages, SharedFileService); + + InsertPretranslationsCall = Grpc.Core.Testing.TestCalls.AsyncClientStreamingCall( + Substitute.For>(), + Task.FromResult(new Empty()), + Task.FromResult(new Metadata()), + () => Status.DefaultSuccess, + () => new Metadata(), + () => { } + ); + + Client = Substitute.For(); + ClientNoFailure(); + + MessageOutboxDeliveryService = new TestMessageOutboxDeliveryService( + Messages, + [ + new ServalPlatformOutboxHandler( + Client, + SharedFileService, + Substitute.For>() + ) + ], + Substitute.For>() + ); + } + + public static AsyncUnaryCall GetEmptyUnaryCall() => + new AsyncUnaryCall( + Task.FromResult(new Empty()), + Task.FromResult(new Metadata()), + () => Status.DefaultSuccess, + () => new Metadata(), + () => { } + ); + + public void ClientNoFailure() + { + Client.BuildStartedAsync(Arg.Any()).Returns(GetEmptyUnaryCall()); + Client.BuildCanceledAsync(Arg.Any()).Returns(GetEmptyUnaryCall()); + Client.BuildFaultedAsync(Arg.Any()).Returns(GetEmptyUnaryCall()); + Client.BuildCompletedAsync(Arg.Any()).Returns(GetEmptyUnaryCall()); + Client + .IncrementTranslationEngineCorpusSizeAsync(Arg.Any()) + .Returns(GetEmptyUnaryCall()); + Client + .InsertPretranslations(cancellationToken: Arg.Any()) + .Returns(InsertPretranslationsCall); + } + + public void ClientInternalFailure() + { + Client + .BuildStartedAsync(Arg.Any()) + .Throws(new RpcException(new Status(StatusCode.Internal, ""))); + Client + .BuildCompletedAsync(Arg.Any()) + .Throws(new RpcException(new Status(StatusCode.Internal, ""))); + } + + public void ClientUnavailableFailure() + { + Client + .BuildStartedAsync(Arg.Any()) + .Throws(new RpcException(new Status(StatusCode.Unavailable, ""))); + Client + .BuildCompletedAsync(Arg.Any()) + .Throws(new RpcException(new Status(StatusCode.Unavailable, ""))); + } + + public void AddStandardMessages() + { + // messages out of order - will be fixed when retrieved + Messages.Add( + new OutboxMessage + { + Id = "A", + Index = 2, + Method = ServalPlatformMessageMethod.BuildCompleted.ToString(), + GroupId = "A", + OutboxName = typeof(ServalPlatformMessageMethod).ToString(), + RequestContent = JsonSerializer.Serialize( + new BuildCompletedRequest + { + BuildId = "A", + CorpusSize = 100, + Confidence = 0.5 + } + ) + } + ); + Messages.Add( + new OutboxMessage + { + Id = "B", + Index = 1, + Method = ServalPlatformMessageMethod.BuildStarted.ToString(), + OutboxName = typeof(ServalPlatformMessageMethod).ToString(), + GroupId = "A", + RequestContent = JsonSerializer.Serialize(new BuildStartedRequest { BuildId = "A" }) + } + ); + Messages.Add( + new OutboxMessage + { + Id = "C", + Index = 3, + Method = ServalPlatformMessageMethod.BuildStarted.ToString(), + OutboxName = typeof(ServalPlatformMessageMethod).ToString(), + GroupId = "B", + RequestContent = JsonSerializer.Serialize(new BuildStartedRequest { BuildId = "B" }) + } + ); + } + } +} diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/NmtEngineServiceTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/NmtEngineServiceTests.cs index 33d25da3b..ed6087634 100644 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/NmtEngineServiceTests.cs +++ b/tests/SIL.Machine.AspNetCore.Tests/Services/NmtEngineServiceTests.cs @@ -169,6 +169,7 @@ public TestEnvironment() Substitute.For(), ClearMLService, SharedFileService, + new MemoryDataAccessContext(), clearMLOptions, buildJobOptions, Substitute.For>() @@ -297,6 +298,7 @@ public override object ActivateJob(Type jobType) _env.PlatformService, _env.Engines, _env._lockFactory, + new MemoryDataAccessContext(), Substitute.For>(), _env.BuildJobService, _env.SharedFileService, @@ -310,6 +312,7 @@ public override object ActivateJob(Type jobType) _env.PlatformService, _env.Engines, _env._lockFactory, + new MemoryDataAccessContext(), _env.BuildJobService, Substitute.For>(), _env.SharedFileService diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/PreprocessBuildJobTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/PreprocessBuildJobTests.cs index d082b8fa2..5150cc759 100644 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/PreprocessBuildJobTests.cs +++ b/tests/SIL.Machine.AspNetCore.Tests/Services/PreprocessBuildJobTests.cs @@ -467,6 +467,7 @@ public PreprocessBuildJob GetBuildJob(TranslationEngineType engineType) PlatformService, Engines, LockFactory, + new MemoryDataAccessContext(), Substitute.For>(), BuildJobService, SharedFileService, @@ -483,6 +484,7 @@ public PreprocessBuildJob GetBuildJob(TranslationEngineType engineType) PlatformService, Engines, LockFactory, + new MemoryDataAccessContext(), Substitute.For>(), BuildJobService, SharedFileService, diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/SmtTransferEngineServiceTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/SmtTransferEngineServiceTests.cs index 40dbed2f5..0d73fc101 100644 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/SmtTransferEngineServiceTests.cs +++ b/tests/SIL.Machine.AspNetCore.Tests/Services/SmtTransferEngineServiceTests.cs @@ -301,6 +301,7 @@ public TestEnvironment(BuildJobRunnerType trainJobRunnerType = BuildJobRunnerTyp Substitute.For(), ClearMLService, SharedFileService, + new MemoryDataAccessContext(), clearMLOptions, buildJobOptions, Substitute.For>() @@ -692,6 +693,7 @@ public override object ActivateJob(Type jobType) _env.PlatformService, _env.Engines, _env._lockFactory, + new MemoryDataAccessContext(), Substitute.For>(), _env.BuildJobService, _env.SharedFileService, @@ -709,6 +711,7 @@ public override object ActivateJob(Type jobType) _env.PlatformService, _env.Engines, _env._lockFactory, + new MemoryDataAccessContext(), _env.BuildJobService, Substitute.For>(), _env.SharedFileService, @@ -724,6 +727,7 @@ public override object ActivateJob(Type jobType) _env.PlatformService, _env.Engines, _env._lockFactory, + new MemoryDataAccessContext(), _env.BuildJobService, Substitute.For>(), _env.SharedFileService, From cd6f3f02b679984a56db020603307d79a58ccac9 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 27 Jun 2024 16:01:24 -0400 Subject: [PATCH 2/9] Minor fixes --- src/SIL.Machine.AspNetCore/Models/Outbox.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/SIL.Machine.AspNetCore/Models/Outbox.cs b/src/SIL.Machine.AspNetCore/Models/Outbox.cs index 8b7acdfb3..3e0397020 100644 --- a/src/SIL.Machine.AspNetCore/Models/Outbox.cs +++ b/src/SIL.Machine.AspNetCore/Models/Outbox.cs @@ -6,8 +6,8 @@ public record Outbox : IEntity public int Revision { get; set; } - public string Name { get; set; } = null!; - public int CurrentIndex { get; set; } + public required string Name { get; init; } = null!; + public required int CurrentIndex { get; set; } public static async Task GetOutboxNextIndexAsync( IRepository indexRepository, From 06cb085f97c14de3d8b45482f893799472b36480 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 27 Jun 2024 16:20:41 -0400 Subject: [PATCH 3/9] Fix bugs --- .../Services/MessageOutboxDeliveryService.cs | 5 +++-- src/SIL.Machine.Serval.EngineServer/appsettings.json | 3 +++ .../Services/MessageOutboxDeliveryServiceTests.cs | 8 +++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/SIL.Machine.AspNetCore/Services/MessageOutboxDeliveryService.cs b/src/SIL.Machine.AspNetCore/Services/MessageOutboxDeliveryService.cs index 08cd891b3..26b55ad05 100644 --- a/src/SIL.Machine.AspNetCore/Services/MessageOutboxDeliveryService.cs +++ b/src/SIL.Machine.AspNetCore/Services/MessageOutboxDeliveryService.cs @@ -3,7 +3,7 @@ public class MessageOutboxDeliveryService( IRepository messages, IEnumerable outboxMessageHandlers, - MessageOutboxOptions options, + IOptionsMonitor options, ILogger logger ) : BackgroundService { @@ -13,7 +13,8 @@ ILogger logger private readonly ILogger _logger = logger; protected TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(10); - protected TimeSpan MessageExpiration { get; set; } = TimeSpan.FromHours(options.MessageExpirationInHours); + protected TimeSpan MessageExpiration { get; set; } = + TimeSpan.FromHours(options.CurrentValue.MessageExpirationInHours); protected override async Task ExecuteAsync(CancellationToken stoppingToken) { diff --git a/src/SIL.Machine.Serval.EngineServer/appsettings.json b/src/SIL.Machine.Serval.EngineServer/appsettings.json index 12f4a051c..7d5b4cdf4 100644 --- a/src/SIL.Machine.Serval.EngineServer/appsettings.json +++ b/src/SIL.Machine.Serval.EngineServer/appsettings.json @@ -32,6 +32,9 @@ "ClearML": { "BuildPollingEnabled": true }, + "MessageOutbox": { + "MessageExpirationInHours": 48 + }, "Logging": { "LogLevel": { "System.Net.Http.HttpClient.Default": "Warning" diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxDeliveryServiceTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxDeliveryServiceTests.cs index 739b65699..b24c2e679 100644 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxDeliveryServiceTests.cs +++ b/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxDeliveryServiceTests.cs @@ -128,7 +128,13 @@ public class TestMessageOutboxDeliveryService( IRepository messages, IEnumerable outboxMessageHandlers, ILogger logger - ) : MessageOutboxDeliveryService(messages, outboxMessageHandlers, new MessageOutboxOptions(), logger) + ) + : MessageOutboxDeliveryService( + messages, + outboxMessageHandlers, + Substitute.For>(), + logger + ) { public async Task ProcessMessagesOnceAsync() => await ProcessMessagesAsync(); From 5c69aca25691ff360cfd6a1262ffa0418daee87f Mon Sep 17 00:00:00 2001 From: John Lambert Date: Fri, 28 Jun 2024 10:08:23 -0400 Subject: [PATCH 4/9] Fix bug --- .../Services/ServalPlatformOutboxHandler.cs | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxHandler.cs b/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxHandler.cs index 4d1b2636b..1f66ef6e9 100644 --- a/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxHandler.cs +++ b/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxHandler.cs @@ -66,7 +66,7 @@ await _client.BuildRestartingAsync( case ServalPlatformMessageMethod.InsertPretranslations: { - Stream targetPretranslateStream = await _sharedFileService.OpenReadAsync( + using Stream targetPretranslateStream = await _sharedFileService.OpenReadAsync( $"outbox/{message.Id}", cancellationToken ); @@ -77,23 +77,24 @@ await _client.BuildRestartingAsync( cancellationToken ) .OfType(); - IAsyncEnumerable requests = pretranslations.Select( - p => new InsertPretranslationRequest - { - EngineId = message.RequestContent!, - CorpusId = p.CorpusId, - TextId = p.TextId, - Refs = { p.Refs }, - Translation = p.Translation - } - ); using var call = _client.InsertPretranslations(cancellationToken: cancellationToken); - await foreach (var request in requests) + await foreach (Pretranslation? pretranslation in pretranslations) { - await call.RequestStream.WriteAsync(request, cancellationToken: cancellationToken); + await call.RequestStream.WriteAsync( + new InsertPretranslationRequest + { + EngineId = message.RequestContent!, + CorpusId = pretranslation.CorpusId, + TextId = pretranslation.TextId, + Refs = { pretranslation.Refs }, + Translation = pretranslation.Translation + }, + cancellationToken + ); } await call.RequestStream.CompleteAsync(); + await call; } break; case ServalPlatformMessageMethod.IncrementTranslationEngineCorpusSize: From 6a8371d4cc01f8d9dfe939154d2969df23ece7c0 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Fri, 28 Jun 2024 14:51:17 -0400 Subject: [PATCH 5/9] Fix tests --- .../MessageOutboxDeliveryServiceTests.cs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxDeliveryServiceTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxDeliveryServiceTests.cs index b24c2e679..ae810457b 100644 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxDeliveryServiceTests.cs +++ b/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxDeliveryServiceTests.cs @@ -124,17 +124,19 @@ public async Task PretranslateSaveFile() Assert.That(await env.SharedFileService.ExistsAsync($"outbox/{fileIdC}"), Is.False); } + private static IOptionsMonitor GetMessageOutboxOptionsMonitor() + { + var options = new MessageOutboxOptions(); + var optionsMonitor = Substitute.For>(); + optionsMonitor.CurrentValue.Returns(options); + return optionsMonitor; + } + public class TestMessageOutboxDeliveryService( IRepository messages, IEnumerable outboxMessageHandlers, ILogger logger - ) - : MessageOutboxDeliveryService( - messages, - outboxMessageHandlers, - Substitute.For>(), - logger - ) + ) : MessageOutboxDeliveryService(messages, outboxMessageHandlers, GetMessageOutboxOptionsMonitor(), logger) { public async Task ProcessMessagesOnceAsync() => await ProcessMessagesAsync(); From 6c9e1ea7890c9b87edffb259412f02181d266a73 Mon Sep 17 00:00:00 2001 From: Damien Daspit Date: Fri, 28 Jun 2024 16:41:02 -0700 Subject: [PATCH 6/9] Fix various issues in transactional outbox - correctly handle scoped services in background services - abstract file handling of content in outbox services - merge Id and Context in Outbox model - Consistently use strings for outbox message method identifiers - split up tests into true unit tests - fix properties in outbox models - fix lifetime of new services --- .../IMachineBuilderExtensions.cs | 19 +- .../IServiceCollectionExtensions.cs | 1 + .../Configuration/MessageOutboxOptions.cs | 3 +- src/SIL.Machine.AspNetCore/Models/Outbox.cs | 20 +- .../Models/OutboxMessage.cs | 15 +- .../Services/ClearMLMonitorService.cs | 16 +- .../Services/FileSystem.cs | 25 ++ .../Services/IFileStorage.cs | 1 - .../Services/IFileSystem.cs | 9 + .../Services/IMessageOutboxService.cs | 9 +- .../Services/IOutboxMessageHandler.cs | 10 +- .../Services/IPlatformService.cs | 6 +- .../Services/ISharedFileService.cs | 1 - .../Services/InMemoryStorage.cs | 9 - .../Services/LocalStorage.cs | 9 - .../Services/MessageOutboxDeliveryService.cs | 103 ++++-- .../Services/MessageOutboxService.cs | 81 ++-- .../Services/ModelCleanupService.cs | 9 +- .../Services/PostprocessBuildJob.cs | 14 +- .../Services/S3FileStorage.cs | 21 -- .../Services/ServalPlatformOutboxConstants.cs | 14 + .../Services/ServalPlatformOutboxHandler.cs | 120 ------ .../ServalPlatformOutboxMessageHandler.cs | 92 +++++ .../Services/ServalPlatformService.cs | 27 +- .../Services/SharedFileService.cs | 5 - src/SIL.Machine.AspNetCore/Usings.cs | 1 + .../Program.cs | 1 + .../Services/InMemoryStorageTests.cs | 16 - .../Services/LocalStorageTests.cs | 17 - .../MessageOutboxDeliveryServiceTests.cs | 346 +++++++----------- .../Services/MessageOutboxServiceTests.cs | 93 +++++ .../Services/ModelCleanupServiceTests.cs | 137 +++---- .../Services/NmtEngineServiceTests.cs | 1 - ...ServalPlatformOutboxMessageHandlerTests.cs | 114 ++++++ .../Services/SmtTransferEngineServiceTests.cs | 1 - tests/SIL.Machine.AspNetCore.Tests/Usings.cs | 5 +- 36 files changed, 765 insertions(+), 606 deletions(-) create mode 100644 src/SIL.Machine.AspNetCore/Services/FileSystem.cs create mode 100644 src/SIL.Machine.AspNetCore/Services/IFileSystem.cs create mode 100644 src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxConstants.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxHandler.cs create mode 100644 src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxMessageHandler.cs create mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxServiceTests.cs create mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/ServalPlatformOutboxMessageHandlerTests.cs diff --git a/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions.cs b/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions.cs index b1f3212bc..66886e2de 100644 --- a/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions.cs +++ b/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions.cs @@ -234,6 +234,8 @@ public static IMachineBuilder AddMemoryDataAccess(this IMachineBuilder builder) o.AddRepository(); o.AddRepository(); o.AddRepository(); + o.AddRepository(); + o.AddRepository(); }); return builder; @@ -279,7 +281,10 @@ await c.Indexes.CreateOrUpdateAsync( ) ); o.AddRepository("outbox_messages"); - o.AddRepository("outboxes"); + o.AddRepository( + "outboxes", + mapSetup: m => m.MapIdProperty(o => o.Id).SetSerializer(new StringSerializer()) + ); } ); builder.Services.AddHealthChecks().AddMongoDb(connectionString!, name: "Mongo"); @@ -298,9 +303,9 @@ public static IMachineBuilder AddServalPlatformService( builder.Services.AddScoped(); - builder.Services.AddSingleton(); + builder.Services.AddSingleton(); - builder.Services.AddSingleton(); + builder.Services.AddScoped(); builder .Services.AddGrpcClient(o => @@ -357,8 +362,6 @@ public static IMachineBuilder AddServalTranslationEngineService( }); builder.AddServalPlatformService(connectionString); - builder.Services.AddHostedService(); - engineTypes ??= builder.Configuration?.GetSection("TranslationEngines").Get() ?? [TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt]; @@ -422,4 +425,10 @@ public static IMachineBuilder AddModelCleanupService(this IMachineBuilder builde builder.Services.AddHostedService(); return builder; } + + public static IMachineBuilder AddMessageOutboxDeliveryService(this IMachineBuilder builder) + { + builder.Services.AddHostedService(); + return builder; + } } diff --git a/src/SIL.Machine.AspNetCore/Configuration/IServiceCollectionExtensions.cs b/src/SIL.Machine.AspNetCore/Configuration/IServiceCollectionExtensions.cs index 0dd26f291..7463e6acd 100644 --- a/src/SIL.Machine.AspNetCore/Configuration/IServiceCollectionExtensions.cs +++ b/src/SIL.Machine.AspNetCore/Configuration/IServiceCollectionExtensions.cs @@ -12,6 +12,7 @@ public static IMachineBuilder AddMachine(this IServiceCollection services, IConf services.AddHealthChecks().AddCheck("S3 Bucket"); services.AddSingleton(); + services.AddTransient(); services.AddScoped(); services.AddSingleton(); diff --git a/src/SIL.Machine.AspNetCore/Configuration/MessageOutboxOptions.cs b/src/SIL.Machine.AspNetCore/Configuration/MessageOutboxOptions.cs index 0b306a900..92bc74eac 100644 --- a/src/SIL.Machine.AspNetCore/Configuration/MessageOutboxOptions.cs +++ b/src/SIL.Machine.AspNetCore/Configuration/MessageOutboxOptions.cs @@ -4,5 +4,6 @@ public class MessageOutboxOptions { public const string Key = "MessageOutbox"; - public int MessageExpirationInHours { get; set; } = 48; + public string DataDir { get; set; } = "outbox"; + public TimeSpan MessageExpirationTimeout { get; set; } = TimeSpan.FromHours(48); } diff --git a/src/SIL.Machine.AspNetCore/Models/Outbox.cs b/src/SIL.Machine.AspNetCore/Models/Outbox.cs index 3e0397020..63e9441e4 100644 --- a/src/SIL.Machine.AspNetCore/Models/Outbox.cs +++ b/src/SIL.Machine.AspNetCore/Models/Outbox.cs @@ -6,23 +6,5 @@ public record Outbox : IEntity public int Revision { get; set; } - public required string Name { get; init; } = null!; - public required int CurrentIndex { get; set; } - - public static async Task GetOutboxNextIndexAsync( - IRepository indexRepository, - string outboxName, - CancellationToken cancellationToken - ) - { - Outbox outbox = ( - await indexRepository.UpdateAsync( - i => i.Name == outboxName, - i => i.Inc(b => b.CurrentIndex, 1), - upsert: true, - cancellationToken: cancellationToken - ) - )!; - return outbox; - } + public int CurrentIndex { get; init; } } diff --git a/src/SIL.Machine.AspNetCore/Models/OutboxMessage.cs b/src/SIL.Machine.AspNetCore/Models/OutboxMessage.cs index f13d3a082..8ec9462cd 100644 --- a/src/SIL.Machine.AspNetCore/Models/OutboxMessage.cs +++ b/src/SIL.Machine.AspNetCore/Models/OutboxMessage.cs @@ -4,11 +4,12 @@ public record OutboxMessage : IEntity { public string Id { get; set; } = ""; public int Revision { get; set; } = 1; - public required int Index { get; set; } - public required string OutboxName { get; set; } - public required string Method { get; set; } - public required string GroupId { get; set; } - public required string? RequestContent { get; set; } - public DateTimeOffset Created { get; set; } = DateTimeOffset.UtcNow; - public int Attempts { get; set; } = 0; + public required int Index { get; init; } + public required string OutboxRef { get; init; } + public required string Method { get; init; } + public required string GroupId { get; init; } + public string? Content { get; init; } + public required bool HasContentStream { get; init; } + public DateTimeOffset Created { get; init; } = DateTimeOffset.UtcNow; + public int Attempts { get; init; } } diff --git a/src/SIL.Machine.AspNetCore/Services/ClearMLMonitorService.cs b/src/SIL.Machine.AspNetCore/Services/ClearMLMonitorService.cs index 952a41254..87e9ed8e9 100644 --- a/src/SIL.Machine.AspNetCore/Services/ClearMLMonitorService.cs +++ b/src/SIL.Machine.AspNetCore/Services/ClearMLMonitorService.cs @@ -4,7 +4,6 @@ public class ClearMLMonitorService( IServiceProvider services, IClearMLService clearMLService, ISharedFileService sharedFileService, - IDataAccessContext dataAccessContext, IOptionsMonitor clearMLOptions, IOptionsMonitor buildJobOptions, ILogger logger @@ -24,7 +23,6 @@ ILogger logger private readonly IClearMLService _clearMLService = clearMLService; private readonly ISharedFileService _sharedFileService = sharedFileService; - private readonly IDataAccessContext _dataAccessContext = dataAccessContext; private readonly ILogger _logger = logger; private readonly Dictionary _curBuildStatus = new(); @@ -85,6 +83,7 @@ await _clearMLService.GetTasksForQueueAsync(_queuePerEngineType[engineType], can _queueSizePerEngineType[engineType] = queuePositionsPerEngineType.Count; } + var dataAccessContext = scope.ServiceProvider.GetRequiredService(); var platformService = scope.ServiceProvider.GetRequiredService(); var lockFactory = scope.ServiceProvider.GetRequiredService(); foreach (TranslationEngine engine in trainingEngines) @@ -119,6 +118,7 @@ or ClearMLTaskStatus.Completed ) { bool canceled = !await TrainJobStartedAsync( + dataAccessContext, lockFactory, buildJobService, platformService, @@ -171,6 +171,7 @@ await UpdateTrainJobStatus( if (canceling) { await TrainJobCanceledAsync( + dataAccessContext, lockFactory, buildJobService, platformService, @@ -185,6 +186,7 @@ await TrainJobCanceledAsync( case ClearMLTaskStatus.Stopped: { await TrainJobCanceledAsync( + dataAccessContext, lockFactory, buildJobService, platformService, @@ -198,6 +200,7 @@ await TrainJobCanceledAsync( case ClearMLTaskStatus.Failed: { await TrainJobFaultedAsync( + dataAccessContext, lockFactory, buildJobService, platformService, @@ -219,6 +222,7 @@ await TrainJobFaultedAsync( } private async Task TrainJobStartedAsync( + IDataAccessContext dataAccessContext, IDistributedReaderWriterLockFactory lockFactory, IBuildJobService buildJobService, IPlatformService platformService, @@ -231,7 +235,7 @@ private async Task TrainJobStartedAsync( IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(engineId, cancellationToken); await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) { - success = await _dataAccessContext.WithTransactionAsync( + success = await dataAccessContext.WithTransactionAsync( async (ct) => { if (!await buildJobService.BuildJobStartedAsync(engineId, buildId, ct)) @@ -281,6 +285,7 @@ CancellationToken cancellationToken } private async Task TrainJobFaultedAsync( + IDataAccessContext dataAccessContext, IDistributedReaderWriterLockFactory lockFactory, IBuildJobService buildJobService, IPlatformService platformService, @@ -295,7 +300,7 @@ CancellationToken cancellationToken IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(engineId, cancellationToken); await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) { - await _dataAccessContext.WithTransactionAsync( + await dataAccessContext.WithTransactionAsync( async (ct) => { await platformService.BuildFaultedAsync(buildId, message, ct); @@ -318,6 +323,7 @@ await buildJobService.BuildJobFinishedAsync( } private async Task TrainJobCanceledAsync( + IDataAccessContext dataAccessContext, IDistributedReaderWriterLockFactory lockFactory, IBuildJobService buildJobService, IPlatformService platformService, @@ -331,7 +337,7 @@ CancellationToken cancellationToken IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(engineId, cancellationToken); await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) { - await _dataAccessContext.WithTransactionAsync( + await dataAccessContext.WithTransactionAsync( async (ct) => { await platformService.BuildCanceledAsync(buildId, ct); diff --git a/src/SIL.Machine.AspNetCore/Services/FileSystem.cs b/src/SIL.Machine.AspNetCore/Services/FileSystem.cs new file mode 100644 index 000000000..a6b5367c1 --- /dev/null +++ b/src/SIL.Machine.AspNetCore/Services/FileSystem.cs @@ -0,0 +1,25 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class FileSystem : IFileSystem +{ + public void CreateDirectory(string path) + { + Directory.CreateDirectory(path); + } + + public void DeleteFile(string path) + { + if (File.Exists(path)) + File.Delete(path); + } + + public Stream OpenWrite(string path) + { + return File.OpenWrite(path); + } + + public Stream OpenRead(string path) + { + return File.OpenRead(path); + } +} diff --git a/src/SIL.Machine.AspNetCore/Services/IFileStorage.cs b/src/SIL.Machine.AspNetCore/Services/IFileStorage.cs index e910a426f..3417cffae 100644 --- a/src/SIL.Machine.AspNetCore/Services/IFileStorage.cs +++ b/src/SIL.Machine.AspNetCore/Services/IFileStorage.cs @@ -16,6 +16,5 @@ Task> ListFilesAsync( Task GetDownloadUrlAsync(string path, DateTime expiresAt, CancellationToken cancellationToken = default); - Task MoveAsync(string sourcePath, string destPath, CancellationToken cancellationToken = default); Task DeleteAsync(string path, bool recurse = false, CancellationToken cancellationToken = default); } diff --git a/src/SIL.Machine.AspNetCore/Services/IFileSystem.cs b/src/SIL.Machine.AspNetCore/Services/IFileSystem.cs new file mode 100644 index 000000000..17f066028 --- /dev/null +++ b/src/SIL.Machine.AspNetCore/Services/IFileSystem.cs @@ -0,0 +1,9 @@ +namespace SIL.Machine.AspNetCore.Services; + +public interface IFileSystem +{ + void DeleteFile(string path); + void CreateDirectory(string path); + Stream OpenWrite(string path); + Stream OpenRead(string path); +} diff --git a/src/SIL.Machine.AspNetCore/Services/IMessageOutboxService.cs b/src/SIL.Machine.AspNetCore/Services/IMessageOutboxService.cs index 997f1efd5..d2b0a975e 100644 --- a/src/SIL.Machine.AspNetCore/Services/IMessageOutboxService.cs +++ b/src/SIL.Machine.AspNetCore/Services/IMessageOutboxService.cs @@ -2,11 +2,12 @@ public interface IMessageOutboxService { - public Task EnqueueMessageAsync( - T method, + public Task EnqueueMessageAsync( + string outboxId, + string method, string groupId, - string? requestContent = null, - string? requestContentPath = null, + string? content = null, + Stream? contentStream = null, CancellationToken cancellationToken = default ); } diff --git a/src/SIL.Machine.AspNetCore/Services/IOutboxMessageHandler.cs b/src/SIL.Machine.AspNetCore/Services/IOutboxMessageHandler.cs index a27ade201..1d39a94d3 100644 --- a/src/SIL.Machine.AspNetCore/Services/IOutboxMessageHandler.cs +++ b/src/SIL.Machine.AspNetCore/Services/IOutboxMessageHandler.cs @@ -2,8 +2,12 @@ public interface IOutboxMessageHandler { - public string Name { get; } + public string OutboxId { get; } - public Task SendMessageAsync(OutboxMessage message, CancellationToken cancellationToken = default); - public Task CleanupMessageAsync(OutboxMessage message, CancellationToken cancellationToken = default); + public Task HandleMessageAsync( + string method, + string? content, + Stream? contentStream, + CancellationToken cancellationToken = default + ); } diff --git a/src/SIL.Machine.AspNetCore/Services/IPlatformService.cs b/src/SIL.Machine.AspNetCore/Services/IPlatformService.cs index c27188597..3d8f35c57 100644 --- a/src/SIL.Machine.AspNetCore/Services/IPlatformService.cs +++ b/src/SIL.Machine.AspNetCore/Services/IPlatformService.cs @@ -22,5 +22,9 @@ Task BuildCompletedAsync( Task BuildFaultedAsync(string buildId, string message, CancellationToken cancellationToken = default); Task BuildRestartingAsync(string buildId, CancellationToken cancellationToken = default); - Task InsertPretranslationsAsync(string engineId, string path, CancellationToken cancellationToken = default); + Task InsertPretranslationsAsync( + string engineId, + Stream pretranslationsStream, + CancellationToken cancellationToken = default + ); } diff --git a/src/SIL.Machine.AspNetCore/Services/ISharedFileService.cs b/src/SIL.Machine.AspNetCore/Services/ISharedFileService.cs index ea17ad113..f082a79c2 100644 --- a/src/SIL.Machine.AspNetCore/Services/ISharedFileService.cs +++ b/src/SIL.Machine.AspNetCore/Services/ISharedFileService.cs @@ -19,7 +19,6 @@ Task> ListFilesAsync( Task OpenWriteAsync(string path, CancellationToken cancellationToken = default); Task ExistsAsync(string path, CancellationToken cancellationToken = default); - Task MoveAsync(string sourcePath, string destPath, CancellationToken cancellationToken = default); Task DeleteAsync(string path, CancellationToken cancellationToken = default); } diff --git a/src/SIL.Machine.AspNetCore/Services/InMemoryStorage.cs b/src/SIL.Machine.AspNetCore/Services/InMemoryStorage.cs index 3824bfe0e..7deccb6e9 100644 --- a/src/SIL.Machine.AspNetCore/Services/InMemoryStorage.cs +++ b/src/SIL.Machine.AspNetCore/Services/InMemoryStorage.cs @@ -133,15 +133,6 @@ public async Task DeleteAsync(string path, bool recurse, CancellationToken cance } } - public Task MoveAsync(string sourcePath, string destPath, CancellationToken cancellationToken = default) - { - if (!_memoryStreams.TryGetValue(Normalize(sourcePath), out Entry? entry)) - throw new FileNotFoundException($"Unable to find file {sourcePath}"); - _memoryStreams[Normalize(destPath)] = entry; - _memoryStreams.Remove(Normalize(sourcePath), out _); - return Task.CompletedTask; - } - protected override void DisposeManagedResources() { foreach (Entry stream in _memoryStreams.Values) diff --git a/src/SIL.Machine.AspNetCore/Services/LocalStorage.cs b/src/SIL.Machine.AspNetCore/Services/LocalStorage.cs index 42c573c38..38e9049bd 100644 --- a/src/SIL.Machine.AspNetCore/Services/LocalStorage.cs +++ b/src/SIL.Machine.AspNetCore/Services/LocalStorage.cs @@ -59,15 +59,6 @@ public Task OpenWriteAsync(string path, CancellationToken cancellationTo return Task.FromResult(File.OpenWrite(pathUri.LocalPath)); } - public Task MoveAsync(string sourcePath, string destPath, CancellationToken cancellationToken = default) - { - Uri sourcePathUri = new(_basePath, Normalize(sourcePath)); - Uri destPathUri = new(_basePath, Normalize(destPath)); - Directory.CreateDirectory(Path.GetDirectoryName(destPathUri.LocalPath)!); - File.Move(sourcePathUri.LocalPath, destPathUri.LocalPath); - return Task.CompletedTask; - } - public async Task DeleteAsync(string path, bool recurse, CancellationToken cancellationToken = default) { Uri pathUri = new(_basePath, Normalize(path)); diff --git a/src/SIL.Machine.AspNetCore/Services/MessageOutboxDeliveryService.cs b/src/SIL.Machine.AspNetCore/Services/MessageOutboxDeliveryService.cs index 26b55ad05..7e4050f22 100644 --- a/src/SIL.Machine.AspNetCore/Services/MessageOutboxDeliveryService.cs +++ b/src/SIL.Machine.AspNetCore/Services/MessageOutboxDeliveryService.cs @@ -1,56 +1,60 @@ namespace SIL.Machine.AspNetCore.Services; public class MessageOutboxDeliveryService( - IRepository messages, + IServiceProvider services, IEnumerable outboxMessageHandlers, + IFileSystem fileSystem, IOptionsMonitor options, ILogger logger ) : BackgroundService { - private readonly IRepository _messages = messages; - private readonly Dictionary _outboxMessageHandlers = - outboxMessageHandlers.ToDictionary(o => o.Name); + private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(10); + private readonly IServiceProvider _services = services; + private readonly Dictionary _outboxMessageHandlers = + outboxMessageHandlers.ToDictionary(o => o.OutboxId); + private readonly IFileSystem _fileSystem = fileSystem; + private readonly IOptionsMonitor _options = options; private readonly ILogger _logger = logger; - protected TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(10); - protected TimeSpan MessageExpiration { get; set; } = - TimeSpan.FromHours(options.CurrentValue.MessageExpirationInHours); protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - using ISubscription subscription = await _messages.SubscribeAsync(e => true); + using IServiceScope scope = _services.CreateScope(); + var messages = scope.ServiceProvider.GetRequiredService>(); + using ISubscription subscription = await messages.SubscribeAsync(e => true, stoppingToken); while (true) { await subscription.WaitForChangeAsync(timeout: Timeout, cancellationToken: stoppingToken); if (stoppingToken.IsCancellationRequested) break; - await ProcessMessagesAsync(); + await ProcessMessagesAsync(messages, stoppingToken); } } - protected async Task ProcessMessagesAsync(CancellationToken cancellationToken = default) + internal async Task ProcessMessagesAsync( + IRepository messages, + CancellationToken cancellationToken = default + ) { - bool anyMessages = await _messages.ExistsAsync(m => true); + bool anyMessages = await messages.ExistsAsync(m => true, cancellationToken); if (!anyMessages) return; - IReadOnlyList messages = await _messages.GetAllAsync(); + IReadOnlyList curMessages = await messages.GetAllAsync(cancellationToken); - IEnumerable> messageGroups = messages.GroupBy( - m => new { m.GroupId, m.OutboxName }, - m => m, - (key, element) => element.OrderBy(m => m.Index).ToList() - ); + IEnumerable> messageGroups = curMessages + .OrderBy(m => m.Index) + .GroupBy(m => (m.OutboxRef, m.GroupId)); - foreach (List messageGroup in messageGroups) + foreach (IGrouping<(string OutboxId, string GroupId), OutboxMessage> messageGroup in messageGroups) { bool abortMessageGroup = false; - var outboxMessageHandler = _outboxMessageHandlers[messageGroup.First().OutboxName]; + IOutboxMessageHandler outboxMessageHandler = _outboxMessageHandlers[messageGroup.Key.OutboxId]; foreach (OutboxMessage message in messageGroup) { try { - await ProcessGroupMessagesAsync(message, outboxMessageHandler, cancellationToken); + await ProcessGroupMessagesAsync(messages, message, outboxMessageHandler, cancellationToken); } catch (RpcException e) { @@ -67,18 +71,18 @@ protected async Task ProcessMessagesAsync(CancellationToken cancellationToken = case StatusCode.Internal: case StatusCode.ResourceExhausted: case StatusCode.Unknown: - abortMessageGroup = !await CheckIfFinalMessageAttempt(message, e); + abortMessageGroup = !await CheckIfFinalMessageAttempt(messages, message, e); break; case StatusCode.InvalidArgument: default: // log error - await PermanentlyFailedMessage(message, e); + await PermanentlyFailedMessage(messages, message, e); break; } } catch (Exception e) { - await PermanentlyFailedMessage(message, e); + await PermanentlyFailedMessage(messages, message, e); break; } if (abortMessageGroup) @@ -87,56 +91,77 @@ protected async Task ProcessMessagesAsync(CancellationToken cancellationToken = } } - async Task ProcessGroupMessagesAsync( + private async Task ProcessGroupMessagesAsync( + IRepository messages, OutboxMessage message, IOutboxMessageHandler outboxMessageHandler, CancellationToken cancellationToken = default ) { - await outboxMessageHandler.SendMessageAsync(message, cancellationToken); - await _messages.DeleteAsync(message.Id); - await outboxMessageHandler.CleanupMessageAsync(message, cancellationToken); + Stream? contentStream = null; + string filePath = Path.Combine(_options.CurrentValue.DataDir, message.Id); + if (message.HasContentStream) + contentStream = _fileSystem.OpenRead(filePath); + try + { + await outboxMessageHandler.HandleMessageAsync( + message.Method, + message.Content, + contentStream, + cancellationToken + ); + await messages.DeleteAsync(message.Id); + } + finally + { + contentStream?.Dispose(); + } + _fileSystem.DeleteFile(filePath); } - async Task CheckIfFinalMessageAttempt(OutboxMessage message, Exception e) + private async Task CheckIfFinalMessageAttempt( + IRepository messages, + OutboxMessage message, + Exception e + ) { - if (message.Created < DateTimeOffset.UtcNow.Subtract(MessageExpiration)) + if (message.Created < DateTimeOffset.UtcNow.Subtract(_options.CurrentValue.MessageExpirationTimeout)) { - await PermanentlyFailedMessage(message, e); + await PermanentlyFailedMessage(messages, message, e); return true; } else { - await LogFailedAttempt(message, e); + await LogFailedAttempt(messages, message, e); return false; } } - async Task PermanentlyFailedMessage(OutboxMessage message, Exception e) + private async Task PermanentlyFailedMessage(IRepository messages, OutboxMessage message, Exception e) { // log error _logger.LogError( e, - "Permanently failed to process message {message.Id}: {message.Method} with content {message.RequestContent} and error message: {e.Message}", + "Permanently failed to process message {Id}: {Method} with content {Content} and error message: {ErrorMessage}", message.Id, message.Method, - message.RequestContent, + message.Content, e.Message ); - await _messages.DeleteAsync(message.Id); + await messages.DeleteAsync(message.Id); } - async Task LogFailedAttempt(OutboxMessage message, Exception e) + private async Task LogFailedAttempt(IRepository messages, OutboxMessage message, Exception e) { // log error - await _messages.UpdateAsync(m => m.Id == message.Id, b => b.Inc(m => m.Attempts, 1)); + await messages.UpdateAsync(m => m.Id == message.Id, b => b.Inc(m => m.Attempts, 1)); _logger.LogError( e, - "Attempt {message.Attempts}. Failed to process message {message.Id}: {message.Method} with content {message.RequestContent} and error message: {e.Message}", + "Attempt {Attempts}. Failed to process message {Id}: {Method} with content {Content} and error message: {ErrorMessage}", message.Attempts + 1, message.Id, message.Method, - message.RequestContent, + message.Content, e.Message ); } diff --git a/src/SIL.Machine.AspNetCore/Services/MessageOutboxService.cs b/src/SIL.Machine.AspNetCore/Services/MessageOutboxService.cs index 158077af5..2383fcfbd 100644 --- a/src/SIL.Machine.AspNetCore/Services/MessageOutboxService.cs +++ b/src/SIL.Machine.AspNetCore/Services/MessageOutboxService.cs @@ -1,52 +1,75 @@ -using MongoDB.Bson; - -namespace SIL.Machine.AspNetCore.Services; +namespace SIL.Machine.AspNetCore.Services; public class MessageOutboxService( - IRepository messageIndexes, + IRepository outboxes, IRepository messages, - ISharedFileService sharedFileService + IIdGenerator idGenerator, + IFileSystem fileSystem, + IOptionsMonitor options ) : IMessageOutboxService { - private readonly IRepository _messageIndex = messageIndexes; + private readonly IRepository _outboxes = outboxes; private readonly IRepository _messages = messages; - private readonly ISharedFileService _sharedFileService = sharedFileService; - protected int MaxDocumentSize { get; set; } = 1_000_000; + private readonly IIdGenerator _idGenerator = idGenerator; + private readonly IFileSystem _fileSystem = fileSystem; + private readonly IOptionsMonitor _options = options; + internal int MaxDocumentSize { get; set; } = 1_000_000; - public async Task EnqueueMessageAsync( - T method, + public async Task EnqueueMessageAsync( + string outboxId, + string method, string groupId, - string? requestContent = null, - string? requestContentPath = null, + string? content = null, + Stream? contentStream = null, CancellationToken cancellationToken = default ) { - if (requestContent == null && requestContentPath == null) + if (content == null && contentStream == null) { - throw new ArgumentException("Either requestContent or contentPath must be specified."); + throw new ArgumentException("Either content or contentStream must be specified."); } - if (requestContent is not null && requestContent.Length > MaxDocumentSize) + if (content is not null && content.Length > MaxDocumentSize) { throw new ArgumentException( $"The content is too large for request {method} with group ID {groupId}. " - + $"It is {requestContent.Length} bytes, but the maximum is {MaxDocumentSize} bytes." + + $"It is {content.Length} bytes, but the maximum is {MaxDocumentSize} bytes." ); } - Outbox outbox = await Outbox.GetOutboxNextIndexAsync(_messageIndex, typeof(T).ToString(), cancellationToken); - OutboxMessage outboxMessage = new OutboxMessage + Outbox outbox = ( + await _outboxes.UpdateAsync( + outboxId, + u => u.Inc(o => o.CurrentIndex, 1), + upsert: true, + cancellationToken: cancellationToken + ) + )!; + OutboxMessage outboxMessage = + new() + { + Id = _idGenerator.GenerateId(), + Index = outbox.CurrentIndex, + OutboxRef = outboxId, + Method = method, + GroupId = groupId, + Content = content, + HasContentStream = contentStream is not null + }; + string filePath = Path.Combine(_options.CurrentValue.DataDir, outboxMessage.Id); + try { - Id = ObjectId.GenerateNewId().ToString(), - Index = outbox.CurrentIndex, - OutboxName = typeof(T).ToString(), - Method = method?.ToString() ?? throw new ArgumentNullException(nameof(method)), - GroupId = groupId, - RequestContent = requestContent - }; - if (requestContentPath != null) + if (contentStream is not null) + { + await using Stream fileStream = _fileSystem.OpenWrite(filePath); + await contentStream.CopyToAsync(fileStream, cancellationToken); + } + await _messages.InsertAsync(outboxMessage, cancellationToken: cancellationToken); + return outboxMessage.Id; + } + catch { - await _sharedFileService.MoveAsync(requestContentPath, $"outbox/{outboxMessage.Id}", cancellationToken); + if (contentStream is not null) + _fileSystem.DeleteFile(filePath); + throw; } - await _messages.InsertAsync(outboxMessage, cancellationToken: cancellationToken); - return outboxMessage.Id; } } diff --git a/src/SIL.Machine.AspNetCore/Services/ModelCleanupService.cs b/src/SIL.Machine.AspNetCore/Services/ModelCleanupService.cs index 2d132b27d..f081a0f7a 100644 --- a/src/SIL.Machine.AspNetCore/Services/ModelCleanupService.cs +++ b/src/SIL.Machine.AspNetCore/Services/ModelCleanupService.cs @@ -3,21 +3,20 @@ namespace SIL.Machine.AspNetCore.Services; public class ModelCleanupService( IServiceProvider services, ISharedFileService sharedFileService, - IRepository engines, ILogger logger ) : RecurrentTask("Model Cleanup Service", services, RefreshPeriod, logger) { private readonly ISharedFileService _sharedFileService = sharedFileService; private readonly ILogger _logger = logger; - private readonly IRepository _engines = engines; private static readonly TimeSpan RefreshPeriod = TimeSpan.FromDays(1); protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken) { - await CheckModelsAsync(cancellationToken); + var engines = scope.ServiceProvider.GetRequiredService>(); + await CheckModelsAsync(engines, cancellationToken); } - private async Task CheckModelsAsync(CancellationToken cancellationToken) + internal async Task CheckModelsAsync(IRepository engines, CancellationToken cancellationToken) { _logger.LogInformation("Running model cleanup job"); IReadOnlyCollection paths = await _sharedFileService.ListFilesAsync( @@ -25,7 +24,7 @@ private async Task CheckModelsAsync(CancellationToken cancellationToken) cancellationToken: cancellationToken ); // Get all NMT engine ids from the database - IReadOnlyList? allEngines = await _engines.GetAllAsync(cancellationToken: cancellationToken); + IReadOnlyList? allEngines = await engines.GetAllAsync(cancellationToken: cancellationToken); IEnumerable validNmtFilenames = allEngines .Where(e => e.Type == TranslationEngineType.Nmt) .Select(e => NmtEngineService.GetModelPath(e.EngineId, e.BuildRevision)); diff --git a/src/SIL.Machine.AspNetCore/Services/PostprocessBuildJob.cs b/src/SIL.Machine.AspNetCore/Services/PostprocessBuildJob.cs index 16e35d14b..dd1d7b387 100644 --- a/src/SIL.Machine.AspNetCore/Services/PostprocessBuildJob.cs +++ b/src/SIL.Machine.AspNetCore/Services/PostprocessBuildJob.cs @@ -23,11 +23,15 @@ CancellationToken cancellationToken { (int corpusSize, double confidence) = data; - await PlatformService.InsertPretranslationsAsync( - engineId, - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); + await using ( + Stream pretranslationsStream = await SharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ) + ) + { + await PlatformService.InsertPretranslationsAsync(engineId, pretranslationsStream, cancellationToken); + } await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) { diff --git a/src/SIL.Machine.AspNetCore/Services/S3FileStorage.cs b/src/SIL.Machine.AspNetCore/Services/S3FileStorage.cs index 7466fe5cc..a9d265e9c 100644 --- a/src/SIL.Machine.AspNetCore/Services/S3FileStorage.cs +++ b/src/SIL.Machine.AspNetCore/Services/S3FileStorage.cs @@ -111,27 +111,6 @@ public async Task OpenWriteAsync(string path, CancellationToken cancella ); } - public async Task MoveAsync(string sourcePath, string destPath, CancellationToken cancellationToken = default) - { - CopyObjectRequest copyRequest = - new() - { - SourceBucket = _bucketName, - SourceKey = _basePath + Normalize(sourcePath), - DestinationBucket = _bucketName, - DestinationKey = _basePath + Normalize(destPath) - }; - CopyObjectResponse copyResponse = await _client.CopyObjectAsync(copyRequest, cancellationToken); - if (!copyResponse.HttpStatusCode.Equals(HttpStatusCode.OK)) - { - throw new HttpRequestException( - $"Received status code {copyResponse.HttpStatusCode} when attempting to copy {sourcePath} to {destPath}" - ); - } - - await DeleteAsync(sourcePath, cancellationToken: cancellationToken); - } - public async Task DeleteAsync(string path, bool recurse = false, CancellationToken cancellationToken = default) { DeleteObjectRequest request = new() { BucketName = _bucketName, Key = _basePath + Normalize(path) }; diff --git a/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxConstants.cs b/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxConstants.cs new file mode 100644 index 000000000..39e6656ad --- /dev/null +++ b/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxConstants.cs @@ -0,0 +1,14 @@ +namespace SIL.Machine.AspNetCore.Services; + +public static class ServalPlatformOutboxConstants +{ + public const string OutboxId = "ServalPlatform"; + + public const string BuildStarted = "BuildStarted"; + public const string BuildCompleted = "BuildCompleted"; + public const string BuildCanceled = "BuildCanceled"; + public const string BuildFaulted = "BuildFaulted"; + public const string BuildRestarting = "BuildRestarting"; + public const string InsertPretranslations = "InsertPretranslations"; + public const string IncrementTranslationEngineCorpusSize = "IncrementTranslationEngineCorpusSize"; +} diff --git a/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxHandler.cs b/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxHandler.cs deleted file mode 100644 index 1f66ef6e9..000000000 --- a/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxHandler.cs +++ /dev/null @@ -1,120 +0,0 @@ -using Serval.Translation.V1; - -namespace SIL.Machine.AspNetCore.Services; - -public enum ServalPlatformMessageMethod -{ - BuildStarted, - BuildCompleted, - BuildCanceled, - BuildFaulted, - BuildRestarting, - InsertPretranslations, - IncrementTranslationEngineCorpusSize -} - -public class ServalPlatformOutboxHandler( - TranslationPlatformApi.TranslationPlatformApiClient client, - ISharedFileService sharedFileService, - ILogger logger -) : IOutboxMessageHandler -{ - private readonly TranslationPlatformApi.TranslationPlatformApiClient _client = client; - private readonly ISharedFileService _sharedFileService = sharedFileService; - private readonly ILogger _logger = logger; - private static readonly JsonSerializerOptions JsonSerializerOptions = - new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; - - private readonly string _name = typeof(ServalPlatformMessageMethod).ToString(); - public string Name => _name; - - public async Task SendMessageAsync(OutboxMessage message, CancellationToken cancellationToken = default) - { - ServalPlatformMessageMethod messageType = Enum.Parse(message.Method); - switch (messageType) - { - case ServalPlatformMessageMethod.BuildStarted: - await _client.BuildStartedAsync( - JsonSerializer.Deserialize(message.RequestContent!), - cancellationToken: cancellationToken - ); - break; - case ServalPlatformMessageMethod.BuildCompleted: - await _client.BuildCompletedAsync( - JsonSerializer.Deserialize(message.RequestContent!), - cancellationToken: cancellationToken - ); - break; - case ServalPlatformMessageMethod.BuildCanceled: - await _client.BuildCanceledAsync( - JsonSerializer.Deserialize(message.RequestContent!), - cancellationToken: cancellationToken - ); - break; - case ServalPlatformMessageMethod.BuildFaulted: - await _client.BuildFaultedAsync( - JsonSerializer.Deserialize(message.RequestContent!), - cancellationToken: cancellationToken - ); - break; - case ServalPlatformMessageMethod.BuildRestarting: - await _client.BuildRestartingAsync( - JsonSerializer.Deserialize(message.RequestContent!), - cancellationToken: cancellationToken - ); - break; - case ServalPlatformMessageMethod.InsertPretranslations: - - { - using Stream targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"outbox/{message.Id}", - cancellationToken - ); - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - JsonSerializerOptions, - cancellationToken - ) - .OfType(); - - using var call = _client.InsertPretranslations(cancellationToken: cancellationToken); - await foreach (Pretranslation? pretranslation in pretranslations) - { - await call.RequestStream.WriteAsync( - new InsertPretranslationRequest - { - EngineId = message.RequestContent!, - CorpusId = pretranslation.CorpusId, - TextId = pretranslation.TextId, - Refs = { pretranslation.Refs }, - Translation = pretranslation.Translation - }, - cancellationToken - ); - } - await call.RequestStream.CompleteAsync(); - await call; - } - break; - case ServalPlatformMessageMethod.IncrementTranslationEngineCorpusSize: - await _client.IncrementTranslationEngineCorpusSizeAsync( - JsonSerializer.Deserialize(message.RequestContent!), - cancellationToken: cancellationToken - ); - break; - default: - _logger.LogWarning( - "Unknown method: {message.Method}. Deleting the message from the list.", - message.Method.ToString() - ); - break; - } - } - - public async Task CleanupMessageAsync(OutboxMessage message, CancellationToken cancellationToken = default) - { - if (await _sharedFileService.ExistsAsync($"outbox/{message.Id}", cancellationToken)) - await _sharedFileService.DeleteAsync($"outbox/{message.Id}", cancellationToken); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxMessageHandler.cs b/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxMessageHandler.cs new file mode 100644 index 000000000..9c32ff0e0 --- /dev/null +++ b/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxMessageHandler.cs @@ -0,0 +1,92 @@ +using Serval.Translation.V1; + +namespace SIL.Machine.AspNetCore.Services; + +public class ServalPlatformOutboxMessageHandler(TranslationPlatformApi.TranslationPlatformApiClient client) + : IOutboxMessageHandler +{ + private readonly TranslationPlatformApi.TranslationPlatformApiClient _client = client; + private static readonly JsonSerializerOptions JsonSerializerOptions = + new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; + + public string OutboxId => ServalPlatformOutboxConstants.OutboxId; + + public async Task HandleMessageAsync( + string method, + string? content, + Stream? contentStream, + CancellationToken cancellationToken = default + ) + { + switch (method) + { + case ServalPlatformOutboxConstants.BuildStarted: + await _client.BuildStartedAsync( + JsonSerializer.Deserialize(content!), + cancellationToken: cancellationToken + ); + break; + case ServalPlatformOutboxConstants.BuildCompleted: + await _client.BuildCompletedAsync( + JsonSerializer.Deserialize(content!), + cancellationToken: cancellationToken + ); + break; + case ServalPlatformOutboxConstants.BuildCanceled: + await _client.BuildCanceledAsync( + JsonSerializer.Deserialize(content!), + cancellationToken: cancellationToken + ); + break; + case ServalPlatformOutboxConstants.BuildFaulted: + await _client.BuildFaultedAsync( + JsonSerializer.Deserialize(content!), + cancellationToken: cancellationToken + ); + break; + case ServalPlatformOutboxConstants.BuildRestarting: + await _client.BuildRestartingAsync( + JsonSerializer.Deserialize(content!), + cancellationToken: cancellationToken + ); + break; + case ServalPlatformOutboxConstants.InsertPretranslations: + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + contentStream!, + JsonSerializerOptions, + cancellationToken + ) + .OfType(); + + using (var call = _client.InsertPretranslations(cancellationToken: cancellationToken)) + { + await foreach (Pretranslation pretranslation in pretranslations) + { + await call.RequestStream.WriteAsync( + new InsertPretranslationRequest + { + EngineId = content!, + CorpusId = pretranslation.CorpusId, + TextId = pretranslation.TextId, + Refs = { pretranslation.Refs }, + Translation = pretranslation.Translation + }, + cancellationToken + ); + } + await call.RequestStream.CompleteAsync(); + await call; + } + break; + case ServalPlatformOutboxConstants.IncrementTranslationEngineCorpusSize: + await _client.IncrementTranslationEngineCorpusSizeAsync( + JsonSerializer.Deserialize(content!), + cancellationToken: cancellationToken + ); + break; + default: + throw new InvalidOperationException($"Encountered a message with the unrecognized method '{method}'."); + } + } +} diff --git a/src/SIL.Machine.AspNetCore/Services/ServalPlatformService.cs b/src/SIL.Machine.AspNetCore/Services/ServalPlatformService.cs index ad89471c6..cdd2e0d05 100644 --- a/src/SIL.Machine.AspNetCore/Services/ServalPlatformService.cs +++ b/src/SIL.Machine.AspNetCore/Services/ServalPlatformService.cs @@ -13,7 +13,8 @@ IMessageOutboxService outboxService public async Task BuildStartedAsync(string buildId, CancellationToken cancellationToken = default) { await _outboxService.EnqueueMessageAsync( - ServalPlatformMessageMethod.BuildStarted, + ServalPlatformOutboxConstants.OutboxId, + ServalPlatformOutboxConstants.BuildStarted, buildId, JsonSerializer.Serialize(new BuildStartedRequest { BuildId = buildId }), cancellationToken: cancellationToken @@ -28,7 +29,8 @@ public async Task BuildCompletedAsync( ) { await _outboxService.EnqueueMessageAsync( - ServalPlatformMessageMethod.BuildCompleted, + ServalPlatformOutboxConstants.OutboxId, + ServalPlatformOutboxConstants.BuildCompleted, buildId, JsonSerializer.Serialize( new BuildCompletedRequest @@ -45,7 +47,8 @@ await _outboxService.EnqueueMessageAsync( public async Task BuildCanceledAsync(string buildId, CancellationToken cancellationToken = default) { await _outboxService.EnqueueMessageAsync( - ServalPlatformMessageMethod.BuildCanceled, + ServalPlatformOutboxConstants.OutboxId, + ServalPlatformOutboxConstants.BuildCanceled, buildId, JsonSerializer.Serialize(new BuildCanceledRequest { BuildId = buildId }), cancellationToken: cancellationToken @@ -55,7 +58,8 @@ await _outboxService.EnqueueMessageAsync( public async Task BuildFaultedAsync(string buildId, string message, CancellationToken cancellationToken = default) { await _outboxService.EnqueueMessageAsync( - ServalPlatformMessageMethod.BuildFaulted, + ServalPlatformOutboxConstants.OutboxId, + ServalPlatformOutboxConstants.BuildFaulted, buildId, JsonSerializer.Serialize(new BuildFaultedRequest { BuildId = buildId, Message = message }), cancellationToken: cancellationToken @@ -65,7 +69,8 @@ await _outboxService.EnqueueMessageAsync( public async Task BuildRestartingAsync(string buildId, CancellationToken cancellationToken = default) { await _outboxService.EnqueueMessageAsync( - ServalPlatformMessageMethod.BuildRestarting, + ServalPlatformOutboxConstants.OutboxId, + ServalPlatformOutboxConstants.BuildRestarting, buildId, JsonSerializer.Serialize(new BuildRestartingRequest { BuildId = buildId }), cancellationToken: cancellationToken @@ -102,15 +107,16 @@ await _client.UpdateBuildStatusAsync( public async Task InsertPretranslationsAsync( string engineId, - string path, + Stream pretranslationsStream, CancellationToken cancellationToken = default ) { await _outboxService.EnqueueMessageAsync( - ServalPlatformMessageMethod.InsertPretranslations, + ServalPlatformOutboxConstants.OutboxId, + ServalPlatformOutboxConstants.InsertPretranslations, engineId, - requestContent: engineId, - requestContentPath: path, + engineId, + pretranslationsStream, cancellationToken: cancellationToken ); } @@ -122,7 +128,8 @@ public async Task IncrementTrainSizeAsync( ) { await _outboxService.EnqueueMessageAsync( - ServalPlatformMessageMethod.IncrementTranslationEngineCorpusSize, + ServalPlatformOutboxConstants.OutboxId, + ServalPlatformOutboxConstants.IncrementTranslationEngineCorpusSize, engineId, JsonSerializer.Serialize( new IncrementTranslationEngineCorpusSizeRequest { EngineId = engineId, Count = count } diff --git a/src/SIL.Machine.AspNetCore/Services/SharedFileService.cs b/src/SIL.Machine.AspNetCore/Services/SharedFileService.cs index f09b4951c..b4244211e 100644 --- a/src/SIL.Machine.AspNetCore/Services/SharedFileService.cs +++ b/src/SIL.Machine.AspNetCore/Services/SharedFileService.cs @@ -101,9 +101,4 @@ public Task ExistsAsync(string path, CancellationToken cancellationToken = { return _fileStorage.ExistsAsync(path, cancellationToken); } - - public Task MoveAsync(string sourcePath, string destPath, CancellationToken cancellationToken = default) - { - return _fileStorage.MoveAsync(sourcePath, destPath, cancellationToken); - } } diff --git a/src/SIL.Machine.AspNetCore/Usings.cs b/src/SIL.Machine.AspNetCore/Usings.cs index 7b5434f72..c1b0f091e 100644 --- a/src/SIL.Machine.AspNetCore/Usings.cs +++ b/src/SIL.Machine.AspNetCore/Usings.cs @@ -38,6 +38,7 @@ global using Microsoft.Extensions.Hosting; global using Microsoft.Extensions.Logging; global using Microsoft.Extensions.Options; +global using MongoDB.Bson.Serialization.Serializers; global using MongoDB.Driver; global using MongoDB.Driver.Linq; global using Nito.AsyncEx; diff --git a/src/SIL.Machine.Serval.EngineServer/Program.cs b/src/SIL.Machine.Serval.EngineServer/Program.cs index e5f4d46bb..029e03df9 100644 --- a/src/SIL.Machine.Serval.EngineServer/Program.cs +++ b/src/SIL.Machine.Serval.EngineServer/Program.cs @@ -11,6 +11,7 @@ .AddMongoHangfireJobClient() .AddServalTranslationEngineService() .AddModelCleanupService() + .AddMessageOutboxDeliveryService() .AddClearMLService(); if (builder.Environment.IsDevelopment()) diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/InMemoryStorageTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/InMemoryStorageTests.cs index 61a0cf3e5..3b5052865 100644 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/InMemoryStorageTests.cs +++ b/tests/SIL.Machine.AspNetCore.Tests/Services/InMemoryStorageTests.cs @@ -88,20 +88,4 @@ public async Task DeleteAsync() var files = await fs.ListFilesAsync("test", recurse: true); Assert.That(files, Is.Empty); } - - [Test] - public async Task MoveAsync() - { - using InMemoryStorage fs = new(); - using (StreamWriter sw = new(await fs.OpenWriteAsync("test1/file1"))) - { - string input = "Hello"; - sw.WriteLine(input); - } - await fs.MoveAsync("test1/file1", "test2/file1"); - var files = await fs.ListFilesAsync("test1", recurse: true); - Assert.That(files, Is.Empty); - files = await fs.ListFilesAsync("test2", recurse: true); - Assert.That(files, Is.EquivalentTo(new[] { "test2/file1" })); - } } diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/LocalStorageTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/LocalStorageTests.cs index 8478c1ab6..280a54bb1 100644 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/LocalStorageTests.cs +++ b/tests/SIL.Machine.AspNetCore.Tests/Services/LocalStorageTests.cs @@ -93,21 +93,4 @@ public async Task DeleteFileAsync() IReadOnlyCollection files = await fs.ListFilesAsync("test", recurse: true); Assert.That(files, Is.Empty); } - - [Test] - public async Task MoveAsync() - { - using var tmpDir = new TempDirectory("test"); - using LocalStorage fs = new(tmpDir.Path); - using (StreamWriter sw = new(await fs.OpenWriteAsync("test1/file1"))) - { - string input = "Hello"; - sw.WriteLine(input); - } - await fs.MoveAsync("test1/file1", "test2/file1"); - var files = await fs.ListFilesAsync("test1", recurse: true); - Assert.That(files, Is.Empty); - files = await fs.ListFilesAsync("test2", recurse: true); - Assert.That(files, Is.EquivalentTo(new[] { "test2/file1" })); - } } diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxDeliveryServiceTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxDeliveryServiceTests.cs index ae810457b..db38ecdb3 100644 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxDeliveryServiceTests.cs +++ b/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxDeliveryServiceTests.cs @@ -1,240 +1,140 @@ -using Google.Protobuf.WellKnownTypes; -using Grpc.Core; -using NSubstitute.ExceptionExtensions; -using Serval.Translation.V1; - -namespace SIL.Machine.AspNetCore.Services; +namespace SIL.Machine.AspNetCore.Services; [TestFixture] public class MessageOutboxDeliveryServiceTests { + private const string OutboxId = "TestOutbox"; + private const string Method1 = "Method1"; + private const string Method2 = "Method2"; + [Test] - public async Task SendMessages() + public async Task ProcessMessagesAsync() { var env = new TestEnvironment(); env.AddStandardMessages(); - await env.MessageOutboxDeliveryService.ProcessMessagesOnceAsync(); + await env.ProcessMessagesAsync(); Received.InOrder(() => { - env.Client.BuildStartedAsync(new BuildStartedRequest { BuildId = "A" }); - env.Client.BuildCompletedAsync(Arg.Any()); - env.Client.BuildStartedAsync(new BuildStartedRequest { BuildId = "B" }); + env.Handler.HandleMessageAsync(Method2, "B", null, Arg.Any()); + env.Handler.HandleMessageAsync(Method1, "A", null, Arg.Any()); + env.Handler.HandleMessageAsync(Method2, "C", null, Arg.Any()); }); + Assert.That(env.Messages.Count, Is.EqualTo(0)); } [Test] - public async Task SendMessages_Timeout() + public async Task ProcessMessagesAsync_Timeout() { var env = new TestEnvironment(); env.AddStandardMessages(); // Timeout is long enough where the message attempt will be incremented, but not deleted. - env.ClientInternalFailure(); - await Task.Delay(100); - await env.MessageOutboxDeliveryService.ProcessMessagesOnceAsync(); + env.EnableHandlerFailure(StatusCode.Internal); + await env.ProcessMessagesAsync(); // Each group should try to send one message - Assert.That((await env.Messages.GetAsync(m => m.Id == "B"))!.Attempts, Is.EqualTo(1)); - Assert.That((await env.Messages.GetAsync(m => m.Id == "A"))!.Attempts, Is.EqualTo(0)); - Assert.That((await env.Messages.GetAsync(m => m.Id == "C"))!.Attempts, Is.EqualTo(1)); + Assert.That(env.Messages.Get("B").Attempts, Is.EqualTo(1)); + Assert.That(env.Messages.Get("A").Attempts, Is.EqualTo(0)); + Assert.That(env.Messages.Get("C").Attempts, Is.EqualTo(1)); // with now shorter timeout, the messages will be deleted. // 4 start build attempts, and only one build completed attempt - env.MessageOutboxDeliveryService.SetMessageExpiration(TimeSpan.FromMilliseconds(1)); - await env.MessageOutboxDeliveryService.ProcessMessagesOnceAsync(); - Assert.That(await env.Messages.ExistsAsync(m => true), Is.False); - var startCalls = env - .Client.ReceivedCalls() - .Count(x => x.GetMethodInfo().Name == nameof(env.Client.BuildStartedAsync)); - Assert.That(startCalls, Is.EqualTo(4)); - var completedCalls = env - .Client.ReceivedCalls() - .Count(x => x.GetMethodInfo().Name == nameof(env.Client.BuildCompletedAsync)); - Assert.That(completedCalls, Is.EqualTo(1)); + env.Options.CurrentValue.Returns( + new MessageOutboxOptions { MessageExpirationTimeout = TimeSpan.FromMilliseconds(1) } + ); + await env.ProcessMessagesAsync(); + Assert.That(env.Messages.Count, Is.EqualTo(0)); + _ = env + .Handler.Received(1) + .HandleMessageAsync(Method1, Arg.Any(), Arg.Any(), Arg.Any()); + _ = env + .Handler.Received(4) + .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()); } [Test] - public async Task SendMessagesUnavailable_Failure() + public async Task ProcessMessagesAsync_UnavailableFailure() { var env = new TestEnvironment(); env.AddStandardMessages(); - env.ClientUnavailableFailure(); - await env.MessageOutboxDeliveryService.ProcessMessagesOnceAsync(); - // Only the first group should be attempted - but not recorded as attempted - Assert.That((await env.Messages.GetAsync(m => m.Id == "B"))!.Attempts, Is.EqualTo(0)); - Assert.That((await env.Messages.GetAsync(m => m.Id == "A"))!.Attempts, Is.EqualTo(0)); - Assert.That((await env.Messages.GetAsync(m => m.Id == "C"))!.Attempts, Is.EqualTo(0)); - env.ClientInternalFailure(); - await env.MessageOutboxDeliveryService.ProcessMessagesOnceAsync(); - Assert.That((await env.Messages.GetAsync(m => m.Id == "B"))!.Attempts, Is.EqualTo(1)); - Assert.That((await env.Messages.GetAsync(m => m.Id == "A"))!.Attempts, Is.EqualTo(0)); - Assert.That((await env.Messages.GetAsync(m => m.Id == "C"))!.Attempts, Is.EqualTo(1)); - env.ClientNoFailure(); - await env.MessageOutboxDeliveryService.ProcessMessagesOnceAsync(); - Assert.That(await env.Messages.ExistsAsync(m => true), Is.False); - // 1 (unavailable) + 2 (internal) + 3 (success) = 6 calls - Assert.That(env.Client.ReceivedCalls().Count(), Is.EqualTo(6)); - } - [Test] - public async Task LargeMessageContent() - { - var env = new TestEnvironment(); - // large max document size - message not saved to file - var fileIdC = await env.OutboxService.EnqueueMessageAsync( - method: ServalPlatformMessageMethod.BuildStarted, - groupId: "C", - requestContent: JsonSerializer.Serialize(new BuildStartedRequest { BuildId = "C" }), - cancellationToken: CancellationToken.None - ); - Assert.That(await env.SharedFileService.ExistsAsync($"outbox/{fileIdC}"), Is.False); - await env.MessageOutboxDeliveryService.ProcessMessagesOnceAsync(); - // small max document size - throws error - env.OutboxService.SetMaxDocumentSize(1); - Assert.ThrowsAsync( - () => - env.OutboxService.EnqueueMessageAsync( - method: ServalPlatformMessageMethod.BuildStarted, - groupId: "D", - requestContent: JsonSerializer.Serialize(new BuildStartedRequest { BuildId = "D" }), - cancellationToken: CancellationToken.None - ) - ); + env.EnableHandlerFailure(StatusCode.Unavailable); + await env.ProcessMessagesAsync(); + // Only the first group should be attempted - but not recorded as attempted + Assert.That(env.Messages.Get("B").Attempts, Is.EqualTo(0)); + Assert.That(env.Messages.Get("A").Attempts, Is.EqualTo(0)); + Assert.That(env.Messages.Get("C").Attempts, Is.EqualTo(0)); + _ = env + .Handler.Received(1) + .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()); + + env.Handler.ClearReceivedCalls(); + env.EnableHandlerFailure(StatusCode.Internal); + await env.ProcessMessagesAsync(); + Assert.That(env.Messages.Get("B").Attempts, Is.EqualTo(1)); + Assert.That(env.Messages.Get("A").Attempts, Is.EqualTo(0)); + Assert.That(env.Messages.Get("C").Attempts, Is.EqualTo(1)); + _ = env + .Handler.Received(2) + .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()); + + env.Handler.ClearReceivedCalls(); + env.DisableHandlerFailure(); + await env.ProcessMessagesAsync(); + Assert.That(env.Messages.Count, Is.EqualTo(0)); + _ = env + .Handler.Received(1) + .HandleMessageAsync(Method1, Arg.Any(), Arg.Any(), Arg.Any()); + _ = env + .Handler.Received(2) + .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()); } [Test] - public async Task PretranslateSaveFile() + public async Task ProcessMessagesAsync_File() { var env = new TestEnvironment(); - // large max document size - message not saved to file - string pretranslationsPath = "build/C/pretranslations.json"; - using (StreamWriter sw = new(await env.SharedFileService.OpenWriteAsync(pretranslationsPath))) - { - sw.WriteLine("[]"); - } - var fileIdC = await env.OutboxService.EnqueueMessageAsync( - method: ServalPlatformMessageMethod.InsertPretranslations, - groupId: "C", - requestContent: "engineId", - requestContentPath: pretranslationsPath, - cancellationToken: CancellationToken.None - ); - Assert.That(await env.SharedFileService.ExistsAsync(pretranslationsPath), Is.False); - Assert.That(await env.SharedFileService.ExistsAsync($"outbox/{fileIdC}"), Is.True); - await env.MessageOutboxDeliveryService.ProcessMessagesOnceAsync(); - Assert.That(await env.SharedFileService.ExistsAsync($"outbox/{fileIdC}"), Is.False); + env.AddContentStreamMessages(); + + await env.ProcessMessagesAsync(); + Assert.That(env.Messages.Count, Is.EqualTo(0)); + _ = env + .Handler.Received(1) + .HandleMessageAsync(Method1, "A", Arg.Is(s => s != null), Arg.Any()); + env.FileSystem.Received().DeleteFile(Path.Combine("outbox", "A")); } - private static IOptionsMonitor GetMessageOutboxOptionsMonitor() + private class TestEnvironment { - var options = new MessageOutboxOptions(); - var optionsMonitor = Substitute.For>(); - optionsMonitor.CurrentValue.Returns(options); - return optionsMonitor; - } - - public class TestMessageOutboxDeliveryService( - IRepository messages, - IEnumerable outboxMessageHandlers, - ILogger logger - ) : MessageOutboxDeliveryService(messages, outboxMessageHandlers, GetMessageOutboxOptionsMonitor(), logger) - { - public async Task ProcessMessagesOnceAsync() => await ProcessMessagesAsync(); - - public void SetMessageExpiration(TimeSpan messageExpiration) => MessageExpiration = messageExpiration; - } - - public class TestMessageOutboxService( - IRepository messageIndexes, - IRepository messages, - ISharedFileService sharedFileService - ) : MessageOutboxService(messageIndexes, messages, sharedFileService) - { - public void SetMaxDocumentSize(int maxDocumentSize) => MaxDocumentSize = maxDocumentSize; - } - - private class TestEnvironment : ObjectModel.DisposableBase - { - public MemoryRepository MessageIndexes { get; } - public MemoryRepository Messages { get; } - public TestMessageOutboxService OutboxService { get; } - public ISharedFileService SharedFileService { get; } - public TranslationPlatformApi.TranslationPlatformApiClient Client { get; } - public TestMessageOutboxDeliveryService MessageOutboxDeliveryService { get; } - public AsyncClientStreamingCall InsertPretranslationsCall { get; } - public TestEnvironment() { - MessageIndexes = new MemoryRepository(); + Outboxes = new MemoryRepository(); Messages = new MemoryRepository(); - SharedFileService = new SharedFileService(Substitute.For()); - OutboxService = new TestMessageOutboxService(MessageIndexes, Messages, SharedFileService); - - InsertPretranslationsCall = Grpc.Core.Testing.TestCalls.AsyncClientStreamingCall( - Substitute.For>(), - Task.FromResult(new Empty()), - Task.FromResult(new Metadata()), - () => Status.DefaultSuccess, - () => new Metadata(), - () => { } - ); - Client = Substitute.For(); - ClientNoFailure(); - - MessageOutboxDeliveryService = new TestMessageOutboxDeliveryService( - Messages, - [ - new ServalPlatformOutboxHandler( - Client, - SharedFileService, - Substitute.For>() - ) - ], + Handler = Substitute.For(); + Handler.OutboxId.Returns(OutboxId); + FileSystem = Substitute.For(); + Options = Substitute.For>(); + Options.CurrentValue.Returns(new MessageOutboxOptions()); + + Service = new MessageOutboxDeliveryService( + Substitute.For(), + [Handler], + FileSystem, + Options, Substitute.For>() ); } - public static AsyncUnaryCall GetEmptyUnaryCall() => - new AsyncUnaryCall( - Task.FromResult(new Empty()), - Task.FromResult(new Metadata()), - () => Status.DefaultSuccess, - () => new Metadata(), - () => { } - ); - - public void ClientNoFailure() - { - Client.BuildStartedAsync(Arg.Any()).Returns(GetEmptyUnaryCall()); - Client.BuildCanceledAsync(Arg.Any()).Returns(GetEmptyUnaryCall()); - Client.BuildFaultedAsync(Arg.Any()).Returns(GetEmptyUnaryCall()); - Client.BuildCompletedAsync(Arg.Any()).Returns(GetEmptyUnaryCall()); - Client - .IncrementTranslationEngineCorpusSizeAsync(Arg.Any()) - .Returns(GetEmptyUnaryCall()); - Client - .InsertPretranslations(cancellationToken: Arg.Any()) - .Returns(InsertPretranslationsCall); - } - - public void ClientInternalFailure() - { - Client - .BuildStartedAsync(Arg.Any()) - .Throws(new RpcException(new Status(StatusCode.Internal, ""))); - Client - .BuildCompletedAsync(Arg.Any()) - .Throws(new RpcException(new Status(StatusCode.Internal, ""))); - } + public MemoryRepository Outboxes { get; } + public MemoryRepository Messages { get; } + public MessageOutboxDeliveryService Service { get; } + public IOutboxMessageHandler Handler { get; } + public IOptionsMonitor Options { get; } + public IFileSystem FileSystem { get; } - public void ClientUnavailableFailure() + public Task ProcessMessagesAsync() { - Client - .BuildStartedAsync(Arg.Any()) - .Throws(new RpcException(new Status(StatusCode.Unavailable, ""))); - Client - .BuildCompletedAsync(Arg.Any()) - .Throws(new RpcException(new Status(StatusCode.Unavailable, ""))); + return Service.ProcessMessagesAsync(Messages); } public void AddStandardMessages() @@ -245,17 +145,11 @@ public void AddStandardMessages() { Id = "A", Index = 2, - Method = ServalPlatformMessageMethod.BuildCompleted.ToString(), + Method = Method1, GroupId = "A", - OutboxName = typeof(ServalPlatformMessageMethod).ToString(), - RequestContent = JsonSerializer.Serialize( - new BuildCompletedRequest - { - BuildId = "A", - CorpusSize = 100, - Confidence = 0.5 - } - ) + OutboxRef = OutboxId, + Content = "A", + HasContentStream = false } ); Messages.Add( @@ -263,10 +157,11 @@ public void AddStandardMessages() { Id = "B", Index = 1, - Method = ServalPlatformMessageMethod.BuildStarted.ToString(), - OutboxName = typeof(ServalPlatformMessageMethod).ToString(), + Method = Method2, + OutboxRef = OutboxId, GroupId = "A", - RequestContent = JsonSerializer.Serialize(new BuildStartedRequest { BuildId = "A" }) + Content = "B", + HasContentStream = false } ); Messages.Add( @@ -274,12 +169,53 @@ public void AddStandardMessages() { Id = "C", Index = 3, - Method = ServalPlatformMessageMethod.BuildStarted.ToString(), - OutboxName = typeof(ServalPlatformMessageMethod).ToString(), + Method = Method2, + OutboxRef = OutboxId, GroupId = "B", - RequestContent = JsonSerializer.Serialize(new BuildStartedRequest { BuildId = "B" }) + Content = "C", + HasContentStream = false } ); } + + public void AddContentStreamMessages() + { + // messages out of order - will be fixed when retrieved + Messages.Add( + new OutboxMessage + { + Id = "A", + Index = 2, + Method = Method1, + GroupId = "A", + OutboxRef = OutboxId, + Content = "A", + HasContentStream = true + } + ); + FileSystem + .OpenRead(Path.Combine("outbox", "A")) + .Returns(ci => new MemoryStream(Encoding.UTF8.GetBytes("Content"))); + } + + public void EnableHandlerFailure(StatusCode code) + { + Handler + .HandleMessageAsync(Method1, Arg.Any(), Arg.Any(), Arg.Any()) + .ThrowsAsync(new RpcException(new Status(code, ""))); + Handler + .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()) + .ThrowsAsync(new RpcException(new Status(code, ""))); + } + + public void DisableHandlerFailure() + { + Handler + .HandleMessageAsync(Method1, Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.CompletedTask); + Handler + .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.CompletedTask); + } } } diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxServiceTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxServiceTests.cs new file mode 100644 index 000000000..7b5d750a1 --- /dev/null +++ b/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxServiceTests.cs @@ -0,0 +1,93 @@ +namespace SIL.Machine.AspNetCore.Services; + +[TestFixture] +public class MessageOutboxServiceTests +{ + private const string OutboxId = "TestOutbox"; + private const string Method = "TestMethod"; + + [Test] + public async Task EnqueueMessageAsync_NoContentStream() + { + TestEnvironment env = new(); + + await env.Service.EnqueueMessageAsync(OutboxId, Method, "A", "content"); + + Outbox outbox = env.Outboxes.Get(OutboxId); + Assert.That(outbox.CurrentIndex, Is.EqualTo(1)); + + OutboxMessage message = env.Messages.Get("1"); + Assert.That(message.OutboxRef, Is.EqualTo(OutboxId)); + Assert.That(message.Method, Is.EqualTo(Method)); + Assert.That(message.Index, Is.EqualTo(1)); + Assert.That(message.Content, Is.EqualTo("content")); + Assert.That(message.HasContentStream, Is.False); + } + + [Test] + public async Task EnqueueMessageAsync_ExistingOutbox() + { + TestEnvironment env = new(); + env.Outboxes.Add(new Outbox { Id = OutboxId, CurrentIndex = 1 }); + + await env.Service.EnqueueMessageAsync(OutboxId, Method, "A", "content"); + + Outbox outbox = env.Outboxes.Get(OutboxId); + Assert.That(outbox.CurrentIndex, Is.EqualTo(2)); + + OutboxMessage message = env.Messages.Get("1"); + Assert.That(message.OutboxRef, Is.EqualTo(OutboxId)); + Assert.That(message.Method, Is.EqualTo(Method)); + Assert.That(message.Index, Is.EqualTo(2)); + Assert.That(message.Content, Is.EqualTo("content")); + Assert.That(message.HasContentStream, Is.False); + } + + [Test] + public async Task EnqueueMessageAsync_HasContentStream() + { + TestEnvironment env = new(); + await using MemoryStream fileStream = new(); + env.FileSystem.OpenWrite(Path.Combine("outbox", "1")).Returns(fileStream); + + await using MemoryStream stream = new(Encoding.UTF8.GetBytes("content")); + await env.Service.EnqueueMessageAsync(OutboxId, Method, "A", "content", stream); + + OutboxMessage message = env.Messages.Get("1"); + Assert.That(message.OutboxRef, Is.EqualTo(OutboxId)); + Assert.That(message.Method, Is.EqualTo(Method)); + Assert.That(message.Index, Is.EqualTo(1)); + Assert.That(message.Content, Is.EqualTo("content")); + Assert.That(message.HasContentStream, Is.True); + Assert.That(fileStream.ToArray(), Is.EqualTo(stream.ToArray())); + } + + [Test] + public void EnqueueMessageAsync_ContentTooLarge() + { + TestEnvironment env = new(); + env.Service.MaxDocumentSize = 5; + + Assert.ThrowsAsync(() => env.Service.EnqueueMessageAsync(OutboxId, Method, "A", "content")); + } + + private class TestEnvironment + { + public TestEnvironment() + { + Outboxes = new MemoryRepository(); + Messages = new MemoryRepository(); + var idGenerator = Substitute.For(); + idGenerator.GenerateId().Returns("1"); + FileSystem = Substitute.For(); + var options = Substitute.For>(); + options.CurrentValue.Returns(new MessageOutboxOptions()); + Service = new MessageOutboxService(Outboxes, Messages, idGenerator, FileSystem, options); + } + + public MemoryRepository Outboxes { get; } + public MemoryRepository Messages { get; } + public IFileSystem FileSystem { get; } + public MessageOutboxService Service { get; } + } +} diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/ModelCleanupServiceTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/ModelCleanupServiceTests.cs index 34a3e7cbd..49d50480a 100644 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/ModelCleanupServiceTests.cs +++ b/tests/SIL.Machine.AspNetCore.Tests/Services/ModelCleanupServiceTests.cs @@ -3,8 +3,6 @@ [TestFixture] public class ModelCleanupServiceTests { - private readonly ISharedFileService _sharedFileService = new SharedFileService(Substitute.For()); - private readonly MemoryRepository _engines = new MemoryRepository(); private static readonly List ValidFiles = [ "models/engineId1_1.tar.gz", @@ -21,79 +19,86 @@ public class ModelCleanupServiceTests "models/engineId1_1.differentExtension" ]; - private async Task SetUpAsync() + [Test] + public async Task CheckModelsAsync_ValidFiles() + { + TestEnvironment env = new(); + await env.CreateFilesAsync(); + + Assert.That( + await env.SharedFileService.ListFilesAsync("models"), + Is.EquivalentTo(ValidFiles.Concat(InvalidFiles)) + ); + await env.CheckModelsAsync(); + // only valid files exist after running service + Assert.That(await env.SharedFileService.ListFilesAsync("models"), Is.EquivalentTo(ValidFiles)); + } + + private class TestEnvironment { - _engines.Add( - new TranslationEngine + private readonly MemoryRepository _engines; + + public TestEnvironment() + { + _engines = new MemoryRepository(); + _engines.Add( + new TranslationEngine + { + Id = "engine1", + EngineId = "engineId1", + Type = TranslationEngineType.Nmt, + SourceLanguage = "es", + TargetLanguage = "en", + BuildRevision = 1, + IsModelPersisted = true + } + ); + _engines.Add( + new TranslationEngine + { + Id = "engine2", + EngineId = "engineId2", + Type = TranslationEngineType.Nmt, + SourceLanguage = "es", + TargetLanguage = "en", + BuildRevision = 2, + IsModelPersisted = true + } + ); + + SharedFileService = new SharedFileService(Substitute.For()); + + Service = new ModelCleanupService( + Substitute.For(), + SharedFileService, + Substitute.For>() + ); + } + + public ModelCleanupService Service { get; } + public ISharedFileService SharedFileService { get; } + + public async Task CreateFilesAsync() + { + foreach (string path in ValidFiles) { - Id = "engine1", - EngineId = "engineId1", - Type = TranslationEngineType.Nmt, - SourceLanguage = "es", - TargetLanguage = "en", - BuildRevision = 1, - IsModelPersisted = true + await WriteFileStubAsync(path, "content"); } - ); - _engines.Add( - new TranslationEngine + foreach (string path in InvalidFiles) { - Id = "engine2", - EngineId = "engineId2", - Type = TranslationEngineType.Nmt, - SourceLanguage = "es", - TargetLanguage = "en", - BuildRevision = 2, - IsModelPersisted = true + await WriteFileStubAsync(path, "content"); } - ); - async Task WriteFileStub(string path, string content) - { - using StreamWriter streamWriter = - new(await _sharedFileService.OpenWriteAsync(path, CancellationToken.None)); - await streamWriter.WriteAsync(content); } - foreach (string path in ValidFiles) + + public Task CheckModelsAsync() { - await WriteFileStub(path, "content"); + return Service.CheckModelsAsync(_engines, CancellationToken.None); } - foreach (string path in InvalidFiles) + + private async Task WriteFileStubAsync(string path, string content) { - await WriteFileStub(path, "content"); + using StreamWriter streamWriter = new(await SharedFileService.OpenWriteAsync(path, CancellationToken.None)); + await streamWriter.WriteAsync(content); } } - - public class TestModelCleanupService( - IServiceProvider serviceProvider, - ISharedFileService sharedFileService, - IRepository engines, - ILogger logger - ) : ModelCleanupService(serviceProvider, sharedFileService, engines, logger) - { - public async Task DoWorkAsync() => - await base.DoWorkAsync(Substitute.For(), CancellationToken.None); - } - - [Test] - public async Task DoWorkAsync_ValidFiles() - { - await SetUpAsync(); - - var cleanupJob = new TestModelCleanupService( - Substitute.For(), - _sharedFileService, - _engines, - Substitute.For>() - ); - Assert.That( - _sharedFileService.ListFilesAsync("models").Result.ToHashSet(), - Is.EquivalentTo(ValidFiles.Concat(InvalidFiles).ToHashSet()) - ); - await cleanupJob.DoWorkAsync(); - // only valid files exist after running service - Assert.That( - _sharedFileService.ListFilesAsync("models").Result.ToHashSet(), - Is.EquivalentTo(ValidFiles.ToHashSet()) - ); - } } diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/NmtEngineServiceTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/NmtEngineServiceTests.cs index ed6087634..1249ad8e9 100644 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/NmtEngineServiceTests.cs +++ b/tests/SIL.Machine.AspNetCore.Tests/Services/NmtEngineServiceTests.cs @@ -169,7 +169,6 @@ public TestEnvironment() Substitute.For(), ClearMLService, SharedFileService, - new MemoryDataAccessContext(), clearMLOptions, buildJobOptions, Substitute.For>() diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/ServalPlatformOutboxMessageHandlerTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/ServalPlatformOutboxMessageHandlerTests.cs new file mode 100644 index 000000000..a49a504ae --- /dev/null +++ b/tests/SIL.Machine.AspNetCore.Tests/Services/ServalPlatformOutboxMessageHandlerTests.cs @@ -0,0 +1,114 @@ +using Google.Protobuf.WellKnownTypes; +using Serval.Translation.V1; + +namespace SIL.Machine.AspNetCore.Services; + +[TestFixture] +public class ServalPlatformOutboxMessageHandlerTests +{ + private static readonly JsonSerializerOptions JsonSerializerOptions = + new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; + + [Test] + public async Task HandleMessageAsync_BuildStarted() + { + TestEnvironment env = new(); + + await env.Handler.HandleMessageAsync( + ServalPlatformOutboxConstants.BuildStarted, + JsonSerializer.Serialize(new BuildStartedRequest { BuildId = "C" }), + null + ); + + _ = env.Client.Received(1).BuildStartedAsync(Arg.Is(x => x.BuildId == "C")); + } + + [Test] + public async Task HandleMessageAsync_InsertPretranslations() + { + TestEnvironment env = new(); + await using (MemoryStream stream = new()) + { + await JsonSerializer.SerializeAsync( + stream, + new[] + { + new Pretranslation + { + CorpusId = "corpus1", + TextId = "MAT", + Refs = ["MAT 1:1"], + Translation = "translation" + } + }, + JsonSerializerOptions + ); + stream.Seek(0, SeekOrigin.Begin); + await env.Handler.HandleMessageAsync( + ServalPlatformOutboxConstants.InsertPretranslations, + "engine1", + stream + ); + } + + _ = env.Client.Received(1).InsertPretranslations(); + _ = env + .PretranslationWriter.Received(1) + .WriteAsync( + new InsertPretranslationRequest + { + EngineId = "engine1", + CorpusId = "corpus1", + TextId = "MAT", + Refs = { "MAT 1:1" }, + Translation = "translation" + }, + Arg.Any() + ); + } + + private class TestEnvironment + { + public TestEnvironment() + { + Client = Substitute.For(); + Client.BuildStartedAsync(Arg.Any()).Returns(CreateEmptyUnaryCall()); + Client.BuildCanceledAsync(Arg.Any()).Returns(CreateEmptyUnaryCall()); + Client.BuildFaultedAsync(Arg.Any()).Returns(CreateEmptyUnaryCall()); + Client.BuildCompletedAsync(Arg.Any()).Returns(CreateEmptyUnaryCall()); + Client + .IncrementTranslationEngineCorpusSizeAsync(Arg.Any()) + .Returns(CreateEmptyUnaryCall()); + PretranslationWriter = Substitute.For>(); + Client + .InsertPretranslations(cancellationToken: Arg.Any()) + .Returns( + TestCalls.AsyncClientStreamingCall( + PretranslationWriter, + Task.FromResult(new Empty()), + Task.FromResult(new Metadata()), + () => Status.DefaultSuccess, + () => new Metadata(), + () => { } + ) + ); + + Handler = new ServalPlatformOutboxMessageHandler(Client); + } + + public TranslationPlatformApi.TranslationPlatformApiClient Client { get; } + public ServalPlatformOutboxMessageHandler Handler { get; } + public IClientStreamWriter PretranslationWriter { get; } + + private static AsyncUnaryCall CreateEmptyUnaryCall() + { + return new AsyncUnaryCall( + Task.FromResult(new Empty()), + Task.FromResult(new Metadata()), + () => Status.DefaultSuccess, + () => new Metadata(), + () => { } + ); + } + } +} diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/SmtTransferEngineServiceTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/SmtTransferEngineServiceTests.cs index 0d73fc101..c88a1351e 100644 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/SmtTransferEngineServiceTests.cs +++ b/tests/SIL.Machine.AspNetCore.Tests/Services/SmtTransferEngineServiceTests.cs @@ -301,7 +301,6 @@ public TestEnvironment(BuildJobRunnerType trainJobRunnerType = BuildJobRunnerTyp Substitute.For(), ClearMLService, SharedFileService, - new MemoryDataAccessContext(), clearMLOptions, buildJobOptions, Substitute.For>() diff --git a/tests/SIL.Machine.AspNetCore.Tests/Usings.cs b/tests/SIL.Machine.AspNetCore.Tests/Usings.cs index c4806736a..94f9c3a80 100644 --- a/tests/SIL.Machine.AspNetCore.Tests/Usings.cs +++ b/tests/SIL.Machine.AspNetCore.Tests/Usings.cs @@ -1,15 +1,18 @@ global using System.IO.Compression; +global using System.Text; global using System.Text.Json; global using System.Text.Json.Nodes; +global using Grpc.Core; +global using Grpc.Core.Testing; global using Hangfire; global using Hangfire.Storage; -global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Hosting; global using Microsoft.Extensions.Hosting.Internal; global using Microsoft.Extensions.Logging; global using Microsoft.Extensions.Options; global using NSubstitute; global using NSubstitute.ClearExtensions; +global using NSubstitute.ExceptionExtensions; global using NSubstitute.ReceivedExtensions; global using NUnit.Framework; global using RichardSzalay.MockHttp; From 2207ae3371b8e6dc09ad3d9a73e206c4ba56a2ff Mon Sep 17 00:00:00 2001 From: Damien Daspit Date: Mon, 1 Jul 2024 11:14:27 -0700 Subject: [PATCH 7/9] Fix serialization of OutboxMessage.OutboxRef --- .../Configuration/IMachineBuilderExtensions.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions.cs b/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions.cs index 66886e2de..3d10565f6 100644 --- a/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions.cs +++ b/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions.cs @@ -280,7 +280,10 @@ await c.Indexes.CreateOrUpdateAsync( ) ) ); - o.AddRepository("outbox_messages"); + o.AddRepository( + "outbox_messages", + mapSetup: m => m.MapProperty(m => m.OutboxRef).SetSerializer(new StringSerializer()) + ); o.AddRepository( "outboxes", mapSetup: m => m.MapIdProperty(o => o.Id).SetSerializer(new StringSerializer()) From 908db33433302fa4413902bf493e9f43cc2e3047 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Mon, 1 Jul 2024 17:39:49 -0400 Subject: [PATCH 8/9] outbox Dir --- .../Configuration/MessageOutboxOptions.cs | 2 +- .../Services/MessageOutboxDeliveryService.cs | 8 +++++++- .../Services/MessageOutboxService.cs | 2 +- src/SIL.Machine.Serval.EngineServer/appsettings.json | 3 ++- src/SIL.Machine.Serval.JobServer/appsettings.json | 4 ++++ 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/SIL.Machine.AspNetCore/Configuration/MessageOutboxOptions.cs b/src/SIL.Machine.AspNetCore/Configuration/MessageOutboxOptions.cs index 92bc74eac..e4b9946bf 100644 --- a/src/SIL.Machine.AspNetCore/Configuration/MessageOutboxOptions.cs +++ b/src/SIL.Machine.AspNetCore/Configuration/MessageOutboxOptions.cs @@ -4,6 +4,6 @@ public class MessageOutboxOptions { public const string Key = "MessageOutbox"; - public string DataDir { get; set; } = "outbox"; + public string OutboxDir { get; set; } = "outbox"; public TimeSpan MessageExpirationTimeout { get; set; } = TimeSpan.FromHours(48); } diff --git a/src/SIL.Machine.AspNetCore/Services/MessageOutboxDeliveryService.cs b/src/SIL.Machine.AspNetCore/Services/MessageOutboxDeliveryService.cs index 7e4050f22..092f66913 100644 --- a/src/SIL.Machine.AspNetCore/Services/MessageOutboxDeliveryService.cs +++ b/src/SIL.Machine.AspNetCore/Services/MessageOutboxDeliveryService.cs @@ -19,6 +19,7 @@ ILogger logger protected override async Task ExecuteAsync(CancellationToken stoppingToken) { + Initialize(); using IServiceScope scope = _services.CreateScope(); var messages = scope.ServiceProvider.GetRequiredService>(); using ISubscription subscription = await messages.SubscribeAsync(e => true, stoppingToken); @@ -31,6 +32,11 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } } + private void Initialize() + { + _fileSystem.CreateDirectory(_options.CurrentValue.OutboxDir); + } + internal async Task ProcessMessagesAsync( IRepository messages, CancellationToken cancellationToken = default @@ -99,7 +105,7 @@ private async Task ProcessGroupMessagesAsync( ) { Stream? contentStream = null; - string filePath = Path.Combine(_options.CurrentValue.DataDir, message.Id); + string filePath = Path.Combine(_options.CurrentValue.OutboxDir, message.Id); if (message.HasContentStream) contentStream = _fileSystem.OpenRead(filePath); try diff --git a/src/SIL.Machine.AspNetCore/Services/MessageOutboxService.cs b/src/SIL.Machine.AspNetCore/Services/MessageOutboxService.cs index 2383fcfbd..c69767d80 100644 --- a/src/SIL.Machine.AspNetCore/Services/MessageOutboxService.cs +++ b/src/SIL.Machine.AspNetCore/Services/MessageOutboxService.cs @@ -54,7 +54,7 @@ await _outboxes.UpdateAsync( Content = content, HasContentStream = contentStream is not null }; - string filePath = Path.Combine(_options.CurrentValue.DataDir, outboxMessage.Id); + string filePath = Path.Combine(_options.CurrentValue.OutboxDir, outboxMessage.Id); try { if (contentStream is not null) diff --git a/src/SIL.Machine.Serval.EngineServer/appsettings.json b/src/SIL.Machine.Serval.EngineServer/appsettings.json index 7d5b4cdf4..e5267100a 100644 --- a/src/SIL.Machine.Serval.EngineServer/appsettings.json +++ b/src/SIL.Machine.Serval.EngineServer/appsettings.json @@ -33,7 +33,8 @@ "BuildPollingEnabled": true }, "MessageOutbox": { - "MessageExpirationInHours": 48 + "MessageExpirationInHours": 48, + "OutboxDir": "/var/lib/machine/outbox" }, "Logging": { "LogLevel": { diff --git a/src/SIL.Machine.Serval.JobServer/appsettings.json b/src/SIL.Machine.Serval.JobServer/appsettings.json index 4ff49d691..ff0b0f54e 100644 --- a/src/SIL.Machine.Serval.JobServer/appsettings.json +++ b/src/SIL.Machine.Serval.JobServer/appsettings.json @@ -32,6 +32,10 @@ "ClearML": { "BuildPollingEnabled": false }, + "MessageOutbox": { + "MessageExpirationInHours": 48, + "OutboxDir": "/var/lib/machine/outbox" + }, "Logging": { "LogLevel": { "System.Net.Http.HttpClient.Default": "Warning" From e52be9dee04b96595bbb8a49b2b967ff44a0a66c Mon Sep 17 00:00:00 2001 From: Damien Daspit Date: Mon, 1 Jul 2024 14:46:09 -0700 Subject: [PATCH 9/9] Remove obsolete outbox setting --- src/SIL.Machine.Serval.EngineServer/appsettings.json | 1 - src/SIL.Machine.Serval.JobServer/appsettings.json | 1 - 2 files changed, 2 deletions(-) diff --git a/src/SIL.Machine.Serval.EngineServer/appsettings.json b/src/SIL.Machine.Serval.EngineServer/appsettings.json index e5267100a..271163fff 100644 --- a/src/SIL.Machine.Serval.EngineServer/appsettings.json +++ b/src/SIL.Machine.Serval.EngineServer/appsettings.json @@ -33,7 +33,6 @@ "BuildPollingEnabled": true }, "MessageOutbox": { - "MessageExpirationInHours": 48, "OutboxDir": "/var/lib/machine/outbox" }, "Logging": { diff --git a/src/SIL.Machine.Serval.JobServer/appsettings.json b/src/SIL.Machine.Serval.JobServer/appsettings.json index ff0b0f54e..738a4f288 100644 --- a/src/SIL.Machine.Serval.JobServer/appsettings.json +++ b/src/SIL.Machine.Serval.JobServer/appsettings.json @@ -33,7 +33,6 @@ "BuildPollingEnabled": false }, "MessageOutbox": { - "MessageExpirationInHours": 48, "OutboxDir": "/var/lib/machine/outbox" }, "Logging": {