This commit adds deployment version to the build

It also adds statistics about the build represented as execution data

samples/ApiExample/ApiExample.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>

src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs

@@ -1,4 +1,4 @@
-using Polly.Extensions.Http;
+using Polly.Extensions.Http;
 using Serval.Translation.V1;
 
 namespace Microsoft.Extensions.DependencyInjection;

src/Machine/src/Serval.Machine.Shared/Models/Build.cs

@@ -1,4 +1,4 @@
-namespace Serval.Machine.Shared.Models;
+namespace Serval.Machine.Shared.Models;
 
 public enum BuildJobState
 {
@@ -29,4 +29,5 @@ public record Build
     public required BuildJobRunnerType BuildJobRunner { get; init; }
     public required BuildStage Stage { get; init; }
     public string? Options { get; set; }
+    public IReadOnlyDictionary<string, string> ExecutionData { get; init; } = new Dictionary<string, string>();
 }

src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
 	<PropertyGroup>
 		<TargetFramework>net8.0</TargetFramework>

src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs

@@ -27,4 +27,11 @@ Task InsertPretranslationsAsync(
         Stream pretranslationsStream,
         CancellationToken cancellationToken = default
     );
+
+    Task UpdateBuildExecutionDataAsync(
+        string engineId,
+        string buildId,
+        IReadOnlyDictionary<string, string> executionData,
+        CancellationToken cancellationToken = default
+    );
 }

src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs

@@ -68,6 +68,13 @@ CancellationToken cancellationToken
             );
         }
 
+        var executionData = new Dictionary<string, string>()
+        {
+            { "trainCount", trainCount.ToString(CultureInfo.InvariantCulture) },
+            { "pretranslateCount", pretranslateCount.ToString(CultureInfo.InvariantCulture) }
+        };
+        await PlatformService.UpdateBuildExecutionDataAsync(engineId, buildId, executionData, cancellationToken);
+
         cancellationToken.ThrowIfCancellationRequested();
 
         bool canceling = !await BuildJobService.StartBuildJobAsync(

src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxConstants.cs

@@ -11,4 +11,5 @@ public static class ServalPlatformOutboxConstants
     public const string BuildRestarting = "BuildRestarting";
     public const string InsertPretranslations = "InsertPretranslations";
     public const string IncrementTranslationEngineCorpusSize = "IncrementTranslationEngineCorpusSize";
+    public const string UpdateBuildExecutionData = "UpdateBuildExecutionData";
 }

src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxMessageHandler.cs

@@ -85,6 +85,12 @@ await _client.IncrementTranslationEngineCorpusSizeAsync(
                     cancellationToken: cancellationToken
                 );
                 break;
+            case ServalPlatformOutboxConstants.UpdateBuildExecutionData:
+                await _client.UpdateBuildExecutionDataAsync(
+                    JsonSerializer.Deserialize<UpdateBuildExecutionDataRequest>(content!),
+                    cancellationToken: cancellationToken
+                );
+                break;
             default:
                 throw new InvalidOperationException($"Encountered a message with the unrecognized method '{method}'.");
         }

src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformService.cs

@@ -137,4 +137,22 @@ await _outboxService.EnqueueMessageAsync(
             cancellationToken: cancellationToken
         );
     }
+
+    public async Task UpdateBuildExecutionDataAsync(
+        string engineId,
+        string buildId,
+        IReadOnlyDictionary<string, string> executionData,
+        CancellationToken cancellationToken = default
+    )
+    {
+        var request = new UpdateBuildExecutionDataRequest { EngineId = engineId, BuildId = buildId };
+        request.ExecutionData.Add((IDictionary<string, string>)executionData);
+        await _outboxService.EnqueueMessageAsync(
+            ServalPlatformOutboxConstants.OutboxId,
+            ServalPlatformOutboxConstants.UpdateBuildExecutionData,
+            engineId,
+            JsonSerializer.Serialize(request),
+            cancellationToken: cancellationToken
+        );
+    }
 }

src/Serval/src/Serval.Client/Client.g.cs

@@ -7644,6 +7644,9 @@ public partial class TranslationBuild
         [Newtonsoft.Json.JsonProperty("deploymentVersion", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
         public string? DeploymentVersion { get; set; } = default!;
 
+        [Newtonsoft.Json.JsonProperty("executionData", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public System.Collections.Generic.IDictionary<string, string>? ExecutionData { get; set; } = default!;
+
     }
 
     [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]

src/Serval/src/Serval.Client/Serval.Client.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
 	<PropertyGroup>
 		<TargetFramework>netstandard2.1</TargetFramework>

src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto

@@ -14,6 +14,7 @@ service TranslationPlatformApi {
 
     rpc IncrementTranslationEngineCorpusSize(IncrementTranslationEngineCorpusSizeRequest) returns (google.protobuf.Empty);
     rpc InsertPretranslations(stream InsertPretranslationsRequest) returns (google.protobuf.Empty);
+    rpc UpdateBuildExecutionData(UpdateBuildExecutionDataRequest) returns (google.protobuf.Empty);
 }
 
 message UpdateBuildStatusRequest {
@@ -59,3 +60,9 @@ message InsertPretranslationsRequest {
     repeated string refs = 4;
     string translation = 5;
 }
+
+message UpdateBuildExecutionDataRequest {
+    string engine_id = 1;
+    string build_id = 2;
+    map<string, string> execution_data = 3;
+}

src/Serval/src/Serval.Shared/Serval.Shared.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
 	<PropertyGroup>
 		<TargetFramework>net8.0</TargetFramework>

src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs

@@ -30,6 +30,11 @@ await c.Indexes.CreateOrUpdateAsync(
                 await c.Indexes.CreateOrUpdateAsync(
                     new CreateIndexModel<Build>(Builders<Build>.IndexKeys.Ascending(b => b.DateCreated))
                 );
+                // migrate by adding ExecutionData field
+                await c.UpdateManyAsync(
+                    Builders<Build>.Filter.Exists(b => b.ExecutionData, false),
+                    Builders<Build>.Update.Set(b => b.ExecutionData, new Dictionary<string, string>())
+                );
             }
         );
         configurator.AddRepository<Pretranslation>(

src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs

@@ -1,4 +1,4 @@
-namespace Serval.Translation.Contracts;
+namespace Serval.Translation.Contracts;
 
 public record TranslationBuildDto
 {
@@ -28,4 +28,5 @@ public record TranslationBuildDto
     /// </example>
     public object? Options { get; init; }
     public string? DeploymentVersion { get; init; }
+    public IReadOnlyDictionary<string, string>? ExecutionData { get; init; }
 }

src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs

@@ -1,4 +1,4 @@
-namespace Serval.Translation.Controllers;
+namespace Serval.Translation.Controllers;
 
 #pragma warning disable CS0612 // Type or member is obsolete
 
@@ -1549,7 +1549,8 @@ private TranslationBuildDto Map(Build source)
             State = source.State,
             DateFinished = source.DateFinished,
             Options = source.Options,
-            DeploymentVersion = source.DeploymentVersion
+            DeploymentVersion = source.DeploymentVersion,
+            ExecutionData = source.ExecutionData
         };
     }
 

src/Serval/src/Serval.Translation/Models/Build.cs

@@ -1,4 +1,4 @@
-namespace Serval.Translation.Models;
+namespace Serval.Translation.Models;
 
 public record Build : IInitializableEntity
 {
@@ -16,6 +16,7 @@ public record Build : IInitializableEntity
     public DateTime? DateFinished { get; init; }
     public IReadOnlyDictionary<string, object>? Options { get; init; }
     public string? DeploymentVersion { get; init; }
+    public IReadOnlyDictionary<string, string> ExecutionData { get; init; } = new Dictionary<string, string>();
     public bool? IsInitialized { get; set; }
     public DateTime? DateCreated { get; set; }
 }

src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs

@@ -98,7 +98,7 @@ await _publishEndpoint.Publish(
                         Owner = engine.Owner,
                         BuildState = build.State,
                         Message = build.Message!,
-                        DateFinished = build.DateFinished!.Value
+                        DateFinished = build.DateFinished!.Value,
                     },
                     ct
                 );
@@ -265,6 +265,37 @@ await _builds.UpdateAsync(
         return Empty;
     }
 
+    public override async Task<Empty> UpdateBuildExecutionData(
+        UpdateBuildExecutionDataRequest request,
+        ServerCallContext context
+    )
+    {
+        var build = await _builds.GetAsync(request.BuildId, cancellationToken: context.CancellationToken);
+        if (build == null)
+        {
+            throw new RpcException(new Status(StatusCode.NotFound, "Build not found."));
+        }
+
+        var updatedExecutionData = new Dictionary<string, string>(build.ExecutionData);
+
+        foreach (var entry in request.ExecutionData)
+        {
+            updatedExecutionData[entry.Key] = entry.Value;
+        }
+
+        await _builds.UpdateAsync(
+            b => b.Id == request.BuildId,
+            u =>
+            {
+                foreach (KeyValuePair<string, string> entry in request.ExecutionData)
+                    u.Set(b => b.ExecutionData[entry.Key], entry.Value);
+            },
+            cancellationToken: context.CancellationToken
+        );
+
+        return new Empty();
+    }
+
     public override async Task<Empty> IncrementTranslationEngineCorpusSize(
         IncrementTranslationEngineCorpusSizeRequest request,
         ServerCallContext context

src/Serval/src/Serval.Webhooks/Serval.Webhooks.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
 	<PropertyGroup>
 		<TargetFramework>net8.0</TargetFramework>

src/Serval/test/Serval.E2ETests/ServalApiTests.cs

@@ -136,6 +136,21 @@ public async Task NmtBatch()
             engineId,
             cId2
         );
+
+        TranslationBuild build = await _helperClient.TranslationEnginesClient.GetCurrentBuildAsync(engineId);
+        Assert.That(build.ExecutionData, Is.Not.Null);
+
+        var executionData = build.ExecutionData!;
+
+        Assert.That(executionData, Contains.Key("trainCount"));
+        Assert.That(executionData, Contains.Key("pretranslateCount"));
+
+        int trainCount = Convert.ToInt32(executionData["trainCount"], CultureInfo.InvariantCulture);
+        int pretranslateCount = Convert.ToInt32(executionData["pretranslateCount"], CultureInfo.InvariantCulture);
+
+        Assert.That(trainCount, Is.GreaterThan(0));
+        Assert.That(pretranslateCount, Is.GreaterThan(0));
+
         Assert.That(lTrans2, Has.Count.EqualTo(13)); // just 2 John
     }
 

src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs

@@ -1,3 +1,4 @@
+using System.Globalization;
 using Serval.Translation.V1;
 
 namespace Serval.Translation.Services;
@@ -90,6 +91,70 @@ await env.PlatformService.UpdateBuildStatus(
         Assert.That(env.Builds.Get("b0").PercentCompleted, Is.EqualTo(0.5));
     }
 
+    [Test]
+    public async Task UpdateBuildExecutionData()
+    {
+        var env = new TestEnvironment();
+
+        var engine = new Engine()
+        {
+            Id = "e0",
+            Owner = "owner1",
+            Type = "nmt",
+            SourceLanguage = "en",
+            TargetLanguage = "es",
+            Corpora = []
+        };
+        await env.Engines.InsertAsync(engine);
+
+        var build = new Build()
+        {
+            Id = "123",
+            EngineRef = "e0",
+            ExecutionData = new Dictionary<string, string>
+            {
+                { "trainCount", "0" },
+                { "pretranslateCount", "0" },
+                { "staticCount", "0" }
+            }
+        };
+        await env.Builds.InsertAsync(build);
+
+        Assert.That(build.ExecutionData, Is.Not.Null);
+
+        var executionData = build.ExecutionData;
+
+        Assert.That(executionData, Contains.Key("trainCount"));
+        Assert.That(executionData, Contains.Key("pretranslateCount"));
+
+        int trainCount = Convert.ToInt32(executionData["trainCount"], CultureInfo.InvariantCulture);
+        int pretranslateCount = Convert.ToInt32(executionData["pretranslateCount"], CultureInfo.InvariantCulture);
+        int staticCount = Convert.ToInt32(executionData["staticCount"], CultureInfo.InvariantCulture);
+
+        Assert.That(trainCount, Is.EqualTo(0));
+        Assert.That(pretranslateCount, Is.EqualTo(0));
+        Assert.That(staticCount, Is.EqualTo(0));
+
+        var updateRequest = new UpdateBuildExecutionDataRequest() { BuildId = "123", EngineId = engine.Id };
+        updateRequest.ExecutionData.Add(
+            new Dictionary<string, string> { { "trainCount", "4" }, { "pretranslateCount", "5" } }
+        );
+
+        await env.PlatformService.UpdateBuildExecutionData(updateRequest, env.ServerCallContext);
+
+        build = await env.Builds.GetAsync(c => c.Id == build.Id);
+
+        executionData = build!.ExecutionData;
+
+        trainCount = Convert.ToInt32(executionData["trainCount"], CultureInfo.InvariantCulture);
+        pretranslateCount = Convert.ToInt32(executionData["pretranslateCount"], CultureInfo.InvariantCulture);
+        staticCount = Convert.ToInt32(executionData["staticCount"], CultureInfo.InvariantCulture);
+
+        Assert.That(trainCount, Is.GreaterThan(0));
+        Assert.That(pretranslateCount, Is.GreaterThan(0));
+        Assert.That(staticCount, Is.EqualTo(0));
+    }
+
     [Test]
     public async Task IncrementCorpusSizeAsync()
     {