Broken, that is, less broken

sillsdev · Nov 12, 2024 · c760b43 · c760b43
1 parent 0dc5784
commit c760b43
Show file tree

Hide file tree

Showing 21 changed files with 362 additions and 109 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -100,6 +100,8 @@ services:
       - BuildJob__ClearML__0__DockerImage=${MACHINE_PY_IMAGE:-ghcr.io/sillsdev/machine.py:latest}
       - BuildJob__ClearML__1__Queue=lambert_24gb.cpu_only
       - BuildJob__ClearML__1__DockerImage=${MACHINE_PY_CPU_IMAGE:-ghcr.io/sillsdev/machine.py:latest.cpu_only}
+      - BuildJob__ClearML__2__Queue=lambert_24gb.cpu_only
+      - BuildJob__ClearML__2__DockerImage=${MACHINE_PY_CPU_IMAGE:-ghcr.io/sillsdev/machine.py:latest.cpu_only}
       - SharedFile__Uri=s3://silnlp/docker-compose/
       - "SharedFile__S3AccessKeyId=${AWS_ACCESS_KEY_ID:?access key needed}"
       - "SharedFile__S3SecretAccessKey=${AWS_SECRET_ACCESS_KEY:?secret key needed}"
@@ -146,6 +148,8 @@ services:
       - BuildJob__ClearML__0__DockerImage=${MACHINE_PY_IMAGE:-ghcr.io/sillsdev/machine.py:latest}
       - BuildJob__ClearML__1__Queue=lambert_24gb.cpu_only
       - BuildJob__ClearML__1__DockerImage=${MACHINE_PY_CPU_IMAGE:-ghcr.io/sillsdev/machine.py:latest.cpu_only}
+      - BuildJob__ClearML__2__Queue=lambert_24gb.cpu_only
+      - BuildJob__ClearML__2__DockerImage=${MACHINE_PY_CPU_IMAGE:-ghcr.io/sillsdev/machine.py:latest.cpu_only}
       - SharedFile__Uri=s3://silnlp/docker-compose/
       - "SharedFile__S3AccessKeyId=${AWS_ACCESS_KEY_ID:?access key needed}"
       - "SharedFile__S3SecretAccessKey=${AWS_SECRET_ACCESS_KEY:?secret key needed}"

diff --git a/src/Machine/src/Serval.Machine.EngineServer/Program.cs b/src/Machine/src/Serval.Machine.EngineServer/Program.cs
@@ -11,6 +11,8 @@
     .AddMongoDataAccess()
     .AddMongoHangfireJobClient()
     .AddServalTranslationEngineService()
+    .AddServalWordAlignmentEngineService()
+    .AddServalPlatformService()
     .AddModelCleanupService()
     .AddMessageOutboxDeliveryService()
     .AddClearMLService();
@@ -36,6 +38,7 @@
 var app = builder.Build();
 
 app.MapServalTranslationEngineService();
+app.MapServalWordAlignmentEngineService();
 app.MapHangfireDashboard();
 
 app.Run();
diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IEndpointRouteBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IEndpointRouteBuilderExtensions.cs
@@ -9,4 +9,11 @@ public static IEndpointRouteBuilder MapServalTranslationEngineService(this IEndp
 
         return builder;
     }
+
+    public static IEndpointRouteBuilder MapServalWordAlignmentEngineService(this IEndpointRouteBuilder builder)
+    {
+        builder.MapGrpcService<ServalWordAlignmentEngineServiceV1>();
+
+        return builder;
+    }
 }
diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs
@@ -165,9 +165,12 @@ public static IMachineBuilder AddHangfireJobServer(
         IEnumerable<EngineType>? engineTypes = null
     )
     {
-        engineTypes ??=
+        engineTypes ??= (
             builder.Configuration.GetSection("TranslationEngines").Get<EngineType[]?>()
-            ?? [EngineType.SmtTransfer, EngineType.Nmt];
+            ?? [EngineType.SmtTransfer, EngineType.Nmt]
+        ).Concat(
+            builder.Configuration.GetSection("WordAlignmentEngines").Get<EngineType[]?>() ?? [EngineType.Statistical]
+        );
         var queues = new List<string>();
         foreach (EngineType engineType in engineTypes.Distinct())
         {
@@ -181,6 +184,11 @@ public static IMachineBuilder AddHangfireJobServer(
                 case EngineType.Nmt:
                     queues.Add("nmt");
                     break;
+                case EngineType.Statistical:
+                    queues.Add("statistical");
+                    break;
+                default:
+                    throw new ArgumentOutOfRangeException(engineType.ToString());
             }
         }
 
@@ -321,10 +329,21 @@ public static IMachineBuilder AddServalPlatformService(
                                 new MethodName
                                 {
                                     Service = "serval.translation.v1.TranslationPlatformApi",
-                                    Method = "UpdateBuildStatus"
+                                    Method = "UpdateTranslationBuildStatus"
                                 }
                             }
                         },
+                        new MethodConfig
+                        {
+                            Names =
+                            {
+                                new MethodName
+                                {
+                                    Service = "serval.word_alignment.v1.WordAlignmentPlatformApi",
+                                    Method = "UpdateWordAlignmentBuildStatus"
+                                }
+                            }
+                        }
                     }
                 };
             });
@@ -344,7 +363,6 @@ public static IMachineBuilder AddServalTranslationEngineService(
             options.Interceptors.Add<UnimplementedInterceptor>();
             options.Interceptors.Add<TimeoutInterceptor>();
         });
-        builder.AddServalPlatformService(connectionString);
 
         engineTypes ??=
             builder.Configuration.GetSection("TranslationEngines").Get<EngineType[]?>()
@@ -354,20 +372,67 @@ public static IMachineBuilder AddServalTranslationEngineService(
             switch (engineType)
             {
                 case EngineType.SmtTransfer:
-                    builder.Services.AddSingleton<SmtTransferEngineStateService>();
-                    builder.Services.AddHostedService<SmtTransferEngineCommitService>();
-                    builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser();
+                    builder.AddThot();
                     builder.Services.AddScoped<ITranslationEngineService, SmtTransferEngineService>();
                     break;
                 case EngineType.Nmt:
                     builder.Services.AddScoped<ITranslationEngineService, NmtEngineService>();
                     break;
+                default:
+                    throw new ArgumentOutOfRangeException(engineType.ToString());
+            }
+        }
+
+        return builder;
+    }
+
+    public static IMachineBuilder AddServalWordAlignmentEngineService(
+        this IMachineBuilder builder,
+        string? connectionString = null,
+        IEnumerable<EngineType>? engineTypes = null
+    )
+    {
+        builder.Services.AddGrpc(options =>
+        {
+            options.Interceptors.Add<CancellationInterceptor>();
+            options.Interceptors.Add<UnimplementedInterceptor>();
+            options.Interceptors.Add<TimeoutInterceptor>();
+        });
+
+        engineTypes ??=
+            builder.Configuration.GetSection("WordAlignmentEngines").Get<EngineType[]?>() ?? [EngineType.Statistical];
+
+        foreach (EngineType engineType in engineTypes.Distinct())
+        {
+            switch (engineType)
+            {
+                case EngineType.Statistical:
+                    builder.AddThot();
+                    builder.Services.AddScoped<IWordAlignmentEngineService, StatisticalEngineService>();
+                    break;
+                default:
+                    throw new ArgumentOutOfRangeException(engineType.ToString());
             }
         }
 
         return builder;
     }
 
+    public static IMachineBuilder AddThot(this IMachineBuilder builder)
+    {
+        try
+        {
+            builder.Services.AddSingleton<SmtTransferEngineStateService>();
+            builder.Services.AddHostedService<SmtTransferEngineCommitService>();
+            builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser();
+        }
+        catch (ArgumentException)
+        {
+            // if this has already been run, don't run it again
+        }
+        return builder;
+    }
+
     public static IMachineBuilder AddBuildJobService(this IMachineBuilder builder, string? smtTransferEngineDir = null)
     {
         builder.Services.AddScoped<IBuildJobService<TranslationEngine>, TranslationBuildJobService>();
@@ -376,13 +441,16 @@ public static IMachineBuilder AddBuildJobService(this IMachineBuilder builder, s
         builder.Services.AddScoped<IBuildJobRunner, ClearMLBuildJobRunner>();
         builder.Services.AddScoped<IClearMLBuildJobFactory, NmtClearMLBuildJobFactory>();
         builder.Services.AddScoped<IClearMLBuildJobFactory, SmtTransferClearMLBuildJobFactory>();
+        builder.Services.AddScoped<IClearMLBuildJobFactory, StatisticalClearMLBuildJobFactory>();
+
         builder.Services.AddSingleton<ClearMLMonitorService>();
         builder.Services.AddSingleton<IClearMLQueueService>(x => x.GetRequiredService<ClearMLMonitorService>());
         builder.Services.AddHostedService(p => p.GetRequiredService<ClearMLMonitorService>());
 
         builder.Services.AddScoped<IBuildJobRunner, HangfireBuildJobRunner>();
         builder.Services.AddScoped<IHangfireBuildJobFactory, NmtHangfireBuildJobFactory>();
         builder.Services.AddScoped<IHangfireBuildJobFactory, SmtTransferHangfireBuildJobFactory>();
+        builder.Services.AddScoped<IHangfireBuildJobFactory, StatisticalHangfireBuildJobFactory>();
 
         if (smtTransferEngineDir is null)
         {

diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/StatisticalEngineOptions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/StatisticalEngineOptions.cs
@@ -0,0 +1,12 @@
+namespace Serval.Machine.Shared.Configuration;
+
+public class StatisticalEngineOptions
+{
+    public const string Key = "StatisticalEngine";
+
+    public string EnginesDir { get; set; } = "word_alignment_engines";
+    public TimeSpan EngineCommitFrequency { get; set; } = TimeSpan.FromMinutes(5);
+    public TimeSpan InactiveEngineTimeout { get; set; } = TimeSpan.FromMinutes(10);
+    public TimeSpan SaveModelTimeout { get; set; } = TimeSpan.FromMinutes(5);
+    public TimeSpan EngineCommitTimeout { get; set; } = TimeSpan.FromMinutes(2);
+}
diff --git a/src/Machine/src/Serval.Machine.Shared/Models/MonolingualCorpus.cs b/src/Machine/src/Serval.Machine.Shared/Models/MonolingualCorpus.cs
@@ -7,6 +7,6 @@ public record MonolingualCorpus
     public required IReadOnlyList<CorpusFile> Files { get; set; }
     public HashSet<string>? TrainOnTextIds { get; set; }
     public Dictionary<string, HashSet<int>>? TrainOnChapters { get; set; }
-    public HashSet<string>? PretranslateTextIds { get; set; }
-    public Dictionary<string, HashSet<int>>? PretranslateChapters { get; set; }
+    public HashSet<string>? InferenceTextIds { get; set; }
+    public Dictionary<string, HashSet<int>>? InferenceChapters { get; set; }
 }
diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs
@@ -19,7 +19,7 @@ public Job CreateJob(string engineId, string buildId, BuildStage stage, object?
                     buildOptions
                 ),
             BuildStage.Postprocess
-                => CreateJob<TranslationEngine, PostprocessBuildJob, (int, double)>(
+                => CreateJob<TranslationEngine, PostprocessBuildJob<TranslationEngine>, (int, double)>(
                     engineId,
                     buildId,
                     "nmt",

diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs
@@ -1,21 +1,15 @@
 namespace Serval.Machine.Shared.Services;
 
-public class PostprocessBuildJob(
+public class PostprocessBuildJob<TEngine>(
     IPlatformService platformService,
-    IRepository<TranslationEngine> engines,
+    IRepository<TEngine> engines,
     IDataAccessContext dataAccessContext,
-    IBuildJobService<TranslationEngine> buildJobService,
-    ILogger<PostprocessBuildJob> logger,
+    IBuildJobService<TEngine> buildJobService,
+    ILogger<PostprocessBuildJob<TEngine>> logger,
     ISharedFileService sharedFileService,
     IOptionsMonitor<BuildJobOptions> options
-)
-    : HangfireBuildJob<TranslationEngine, (int, double)>(
-        platformService,
-        engines,
-        dataAccessContext,
-        buildJobService,
-        logger
-    )
+) : HangfireBuildJob<TEngine, (int, double)>(platformService, engines, dataAccessContext, buildJobService, logger)
+    where TEngine : ITrainingEngine
 {
     protected ISharedFileService SharedFileService { get; } = sharedFileService;
     private readonly BuildJobOptions _buildJobOptions = options.CurrentValue;