diff --git a/samples/ApiExample/ApiExample.csproj b/samples/ApiExample/ApiExample.csproj index 5e541c5a..7d6b519e 100644 --- a/samples/ApiExample/ApiExample.csproj +++ b/samples/ApiExample/ApiExample.csproj @@ -1,4 +1,4 @@ - + Exe diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs index 67b8ef3d..03945db0 100644 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs @@ -1,4 +1,4 @@ -using Polly.Extensions.Http; +using Polly.Extensions.Http; using Serval.Translation.V1; namespace Microsoft.Extensions.DependencyInjection; diff --git a/src/Machine/src/Serval.Machine.Shared/Models/Build.cs b/src/Machine/src/Serval.Machine.Shared/Models/Build.cs index aca20540..a1758463 100644 --- a/src/Machine/src/Serval.Machine.Shared/Models/Build.cs +++ b/src/Machine/src/Serval.Machine.Shared/Models/Build.cs @@ -1,4 +1,4 @@ -namespace Serval.Machine.Shared.Models; +namespace Serval.Machine.Shared.Models; public enum BuildJobState { @@ -29,4 +29,5 @@ public record Build public required BuildJobRunnerType BuildJobRunner { get; init; } public required BuildStage Stage { get; init; } public string? Options { get; set; } + public IReadOnlyDictionary ExecutionData { get; init; } = new Dictionary(); } diff --git a/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj b/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj index f9756293..41a037a0 100644 --- a/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj +++ b/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj @@ -1,4 +1,4 @@ - + net8.0 diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs index 79b30f6b..69c4c7d2 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs @@ -27,4 +27,11 @@ Task InsertPretranslationsAsync( Stream pretranslationsStream, CancellationToken cancellationToken = default ); + + Task UpdateBuildExecutionDataAsync( + string engineId, + string buildId, + IReadOnlyDictionary executionData, + CancellationToken cancellationToken = default + ); } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs index 2d2e06f7..2d67c90b 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs @@ -68,6 +68,13 @@ CancellationToken cancellationToken ); } + var executionData = new Dictionary() + { + { "trainCount", trainCount.ToString(CultureInfo.InvariantCulture) }, + { "pretranslateCount", pretranslateCount.ToString(CultureInfo.InvariantCulture) } + }; + await PlatformService.UpdateBuildExecutionDataAsync(engineId, buildId, executionData, cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); bool canceling = !await BuildJobService.StartBuildJobAsync( diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxConstants.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxConstants.cs index 493cb9ed..1ad8d425 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxConstants.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxConstants.cs @@ -11,4 +11,5 @@ public static class ServalPlatformOutboxConstants public const string BuildRestarting = "BuildRestarting"; public const string InsertPretranslations = "InsertPretranslations"; public const string IncrementTranslationEngineCorpusSize = "IncrementTranslationEngineCorpusSize"; + public const string UpdateBuildExecutionData = "UpdateBuildExecutionData"; } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxMessageHandler.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxMessageHandler.cs index 41132504..490ed650 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxMessageHandler.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxMessageHandler.cs @@ -85,6 +85,12 @@ await _client.IncrementTranslationEngineCorpusSizeAsync( cancellationToken: cancellationToken ); break; + case ServalPlatformOutboxConstants.UpdateBuildExecutionData: + await _client.UpdateBuildExecutionDataAsync( + JsonSerializer.Deserialize(content!), + cancellationToken: cancellationToken + ); + break; default: throw new InvalidOperationException($"Encountered a message with the unrecognized method '{method}'."); } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformService.cs index 429fbb72..fece316c 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformService.cs @@ -137,4 +137,22 @@ await _outboxService.EnqueueMessageAsync( cancellationToken: cancellationToken ); } + + public async Task UpdateBuildExecutionDataAsync( + string engineId, + string buildId, + IReadOnlyDictionary executionData, + CancellationToken cancellationToken = default + ) + { + var request = new UpdateBuildExecutionDataRequest { EngineId = engineId, BuildId = buildId }; + request.ExecutionData.Add((IDictionary)executionData); + await _outboxService.EnqueueMessageAsync( + ServalPlatformOutboxConstants.OutboxId, + ServalPlatformOutboxConstants.UpdateBuildExecutionData, + engineId, + JsonSerializer.Serialize(request), + cancellationToken: cancellationToken + ); + } } diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index 91631ede..d6a37eaa 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -7644,6 +7644,9 @@ public partial class TranslationBuild [Newtonsoft.Json.JsonProperty("deploymentVersion", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? DeploymentVersion { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("executionData", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IDictionary? ExecutionData { get; set; } = default!; + } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] diff --git a/src/Serval/src/Serval.Client/Serval.Client.csproj b/src/Serval/src/Serval.Client/Serval.Client.csproj index 68c6aa05..d3d3c53c 100644 --- a/src/Serval/src/Serval.Client/Serval.Client.csproj +++ b/src/Serval/src/Serval.Client/Serval.Client.csproj @@ -1,4 +1,4 @@ - + netstandard2.1 diff --git a/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto b/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto index 84b24ab1..235e884d 100644 --- a/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto +++ b/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto @@ -14,6 +14,7 @@ service TranslationPlatformApi { rpc IncrementTranslationEngineCorpusSize(IncrementTranslationEngineCorpusSizeRequest) returns (google.protobuf.Empty); rpc InsertPretranslations(stream InsertPretranslationsRequest) returns (google.protobuf.Empty); + rpc UpdateBuildExecutionData(UpdateBuildExecutionDataRequest) returns (google.protobuf.Empty); } message UpdateBuildStatusRequest { @@ -59,3 +60,9 @@ message InsertPretranslationsRequest { repeated string refs = 4; string translation = 5; } + +message UpdateBuildExecutionDataRequest { + string engine_id = 1; + string build_id = 2; + map execution_data = 3; +} diff --git a/src/Serval/src/Serval.Shared/Serval.Shared.csproj b/src/Serval/src/Serval.Shared/Serval.Shared.csproj index f2607b7b..4b163b13 100644 --- a/src/Serval/src/Serval.Shared/Serval.Shared.csproj +++ b/src/Serval/src/Serval.Shared/Serval.Shared.csproj @@ -1,4 +1,4 @@ - + net8.0 diff --git a/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs index d0092c0b..cdc08935 100644 --- a/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs +++ b/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs @@ -30,6 +30,11 @@ await c.Indexes.CreateOrUpdateAsync( await c.Indexes.CreateOrUpdateAsync( new CreateIndexModel(Builders.IndexKeys.Ascending(b => b.DateCreated)) ); + // migrate by adding ExecutionData field + await c.UpdateManyAsync( + Builders.Filter.Exists(b => b.ExecutionData, false), + Builders.Update.Set(b => b.ExecutionData, new Dictionary()) + ); } ); configurator.AddRepository( diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs b/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs index eb009161..0d21ab1a 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs +++ b/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Contracts; public record TranslationBuildDto { @@ -28,4 +28,5 @@ public record TranslationBuildDto /// public object? Options { get; init; } public string? DeploymentVersion { get; init; } + public IReadOnlyDictionary? ExecutionData { get; init; } } diff --git a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs b/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs index 47358472..4f4224d4 100644 --- a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs +++ b/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Controllers; +namespace Serval.Translation.Controllers; #pragma warning disable CS0612 // Type or member is obsolete @@ -1549,7 +1549,8 @@ private TranslationBuildDto Map(Build source) State = source.State, DateFinished = source.DateFinished, Options = source.Options, - DeploymentVersion = source.DeploymentVersion + DeploymentVersion = source.DeploymentVersion, + ExecutionData = source.ExecutionData }; } diff --git a/src/Serval/src/Serval.Translation/Models/Build.cs b/src/Serval/src/Serval.Translation/Models/Build.cs index 49168679..bc6e4a62 100644 --- a/src/Serval/src/Serval.Translation/Models/Build.cs +++ b/src/Serval/src/Serval.Translation/Models/Build.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Models; +namespace Serval.Translation.Models; public record Build : IInitializableEntity { @@ -13,9 +13,10 @@ public record Build : IInitializableEntity public string? Message { get; init; } public int? QueueDepth { get; init; } public JobState State { get; init; } = JobState.Pending; + public DateTime DateCreated { get; init; } = DateTime.UtcNow; public DateTime? DateFinished { get; init; } public IReadOnlyDictionary? Options { get; init; } public string? DeploymentVersion { get; init; } + public IReadOnlyDictionary ExecutionData { get; init; } = new Dictionary(); public bool? IsInitialized { get; set; } - public DateTime? DateCreated { get; set; } } diff --git a/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs b/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs index 615e8c89..7082c015 100644 --- a/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs +++ b/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs @@ -98,7 +98,7 @@ await _publishEndpoint.Publish( Owner = engine.Owner, BuildState = build.State, Message = build.Message!, - DateFinished = build.DateFinished!.Value + DateFinished = build.DateFinished!.Value, }, ct ); @@ -265,6 +265,37 @@ await _builds.UpdateAsync( return Empty; } + public override async Task UpdateBuildExecutionData( + UpdateBuildExecutionDataRequest request, + ServerCallContext context + ) + { + var build = await _builds.GetAsync(request.BuildId, cancellationToken: context.CancellationToken); + if (build == null) + { + throw new RpcException(new Status(StatusCode.NotFound, "Build not found.")); + } + + var updatedExecutionData = new Dictionary(build.ExecutionData); + + foreach (var entry in request.ExecutionData) + { + updatedExecutionData[entry.Key] = entry.Value; + } + + await _builds.UpdateAsync( + b => b.Id == request.BuildId, + u => + { + foreach (KeyValuePair entry in request.ExecutionData) + u.Set(b => b.ExecutionData[entry.Key], entry.Value); + }, + cancellationToken: context.CancellationToken + ); + + return new Empty(); + } + public override async Task IncrementTranslationEngineCorpusSize( IncrementTranslationEngineCorpusSizeRequest request, ServerCallContext context diff --git a/src/Serval/src/Serval.Webhooks/Serval.Webhooks.csproj b/src/Serval/src/Serval.Webhooks/Serval.Webhooks.csproj index bb1af79a..b36d305a 100644 --- a/src/Serval/src/Serval.Webhooks/Serval.Webhooks.csproj +++ b/src/Serval/src/Serval.Webhooks/Serval.Webhooks.csproj @@ -1,4 +1,4 @@ - + net8.0 diff --git a/src/Serval/test/Serval.E2ETests/ServalApiTests.cs b/src/Serval/test/Serval.E2ETests/ServalApiTests.cs index 2fb9f86a..26106235 100644 --- a/src/Serval/test/Serval.E2ETests/ServalApiTests.cs +++ b/src/Serval/test/Serval.E2ETests/ServalApiTests.cs @@ -136,6 +136,21 @@ public async Task NmtBatch() engineId, cId2 ); + + TranslationBuild build = await _helperClient.TranslationEnginesClient.GetCurrentBuildAsync(engineId); + Assert.That(build.ExecutionData, Is.Not.Null); + + var executionData = build.ExecutionData!; + + Assert.That(executionData, Contains.Key("trainCount")); + Assert.That(executionData, Contains.Key("pretranslateCount")); + + int trainCount = Convert.ToInt32(executionData["trainCount"], CultureInfo.InvariantCulture); + int pretranslateCount = Convert.ToInt32(executionData["pretranslateCount"], CultureInfo.InvariantCulture); + + Assert.That(trainCount, Is.GreaterThan(0)); + Assert.That(pretranslateCount, Is.GreaterThan(0)); + Assert.That(lTrans2, Has.Count.EqualTo(13)); // just 2 John } diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs index 10f3ca14..4ab47e10 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs @@ -1,3 +1,4 @@ +using System.Globalization; using Serval.Translation.V1; namespace Serval.Translation.Services; @@ -90,6 +91,70 @@ await env.PlatformService.UpdateBuildStatus( Assert.That(env.Builds.Get("b0").PercentCompleted, Is.EqualTo(0.5)); } + [Test] + public async Task UpdateBuildExecutionData() + { + var env = new TestEnvironment(); + + var engine = new Engine() + { + Id = "e0", + Owner = "owner1", + Type = "nmt", + SourceLanguage = "en", + TargetLanguage = "es", + Corpora = [] + }; + await env.Engines.InsertAsync(engine); + + var build = new Build() + { + Id = "123", + EngineRef = "e0", + ExecutionData = new Dictionary + { + { "trainCount", "0" }, + { "pretranslateCount", "0" }, + { "staticCount", "0" } + } + }; + await env.Builds.InsertAsync(build); + + Assert.That(build.ExecutionData, Is.Not.Null); + + var executionData = build.ExecutionData; + + Assert.That(executionData, Contains.Key("trainCount")); + Assert.That(executionData, Contains.Key("pretranslateCount")); + + int trainCount = Convert.ToInt32(executionData["trainCount"], CultureInfo.InvariantCulture); + int pretranslateCount = Convert.ToInt32(executionData["pretranslateCount"], CultureInfo.InvariantCulture); + int staticCount = Convert.ToInt32(executionData["staticCount"], CultureInfo.InvariantCulture); + + Assert.That(trainCount, Is.EqualTo(0)); + Assert.That(pretranslateCount, Is.EqualTo(0)); + Assert.That(staticCount, Is.EqualTo(0)); + + var updateRequest = new UpdateBuildExecutionDataRequest() { BuildId = "123", EngineId = engine.Id }; + updateRequest.ExecutionData.Add( + new Dictionary { { "trainCount", "4" }, { "pretranslateCount", "5" } } + ); + + await env.PlatformService.UpdateBuildExecutionData(updateRequest, env.ServerCallContext); + + build = await env.Builds.GetAsync(c => c.Id == build.Id); + + executionData = build!.ExecutionData; + + trainCount = Convert.ToInt32(executionData["trainCount"], CultureInfo.InvariantCulture); + pretranslateCount = Convert.ToInt32(executionData["pretranslateCount"], CultureInfo.InvariantCulture); + staticCount = Convert.ToInt32(executionData["staticCount"], CultureInfo.InvariantCulture); + + Assert.That(trainCount, Is.GreaterThan(0)); + Assert.That(pretranslateCount, Is.GreaterThan(0)); + Assert.That(staticCount, Is.EqualTo(0)); + } + [Test] public async Task IncrementCorpusSizeAsync() {