more broken

sillsdev · Nov 5, 2024 · ea5aa40 · ea5aa40
1 parent d9f6118
commit ea5aa40
Show file tree

Hide file tree

Showing 14 changed files with 60 additions and 48 deletions.
diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLBuildJobRunner.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLBuildJobRunner.cs
@@ -11,7 +11,7 @@ IOptionsMonitor<BuildJobOptions> options
         buildJobFactories.ToDictionary(f => f.EngineType);
 
     private readonly Dictionary<EngineType, ClearMLBuildQueue> _options = options.CurrentValue.ClearML.ToDictionary(o =>
-        o.EngineType
+        Enum.Parse<EngineType>(o.EngineType)
     );
 
     public BuildJobRunnerType Type => BuildJobRunnerType.ClearML;

diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs
@@ -1,6 +1,4 @@
-using System.Linq;
-
-namespace Serval.Machine.Shared.Services;
+namespace Serval.Machine.Shared.Services;
 
 public class ClearMLMonitorService(
     IServiceProvider services,
@@ -51,7 +49,7 @@ private async Task MonitorClearMLTasksPerDomain<TEngine>(IServiceScope scope, Ca
         try
         {
             var translationBuildJobService = scope.ServiceProvider.GetRequiredService<
-                IBuildJobService<TranslationEngine>
+                IBuildJobService<ITrainingEngine>
             >();
             var wordAlignmentBuildJobService = scope.ServiceProvider.GetRequiredService<
                 IBuildJobService<WordAlignmentEngine>
@@ -100,7 +98,7 @@ await _clearMLService.GetTasksByIdAsync(
 
             var dataAccessContext = scope.ServiceProvider.GetRequiredService<IDataAccessContext>();
             var platformService = scope.ServiceProvider.GetRequiredService<IPlatformService>();
-            foreach (TranslationEngine engine in trainingEngines)
+            foreach (ITrainingEngine engine in trainingEngines)
             {
                 if (engine.CurrentBuild is null || !tasks.TryGetValue(engine.CurrentBuild.JobId, out ClearMLTask? task))
                     continue;

diff --git a/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJob.cs
@@ -1,12 +1,13 @@
 namespace Serval.Machine.Shared.Services;
 
-public abstract class HangfireBuildJob(
+public abstract class HangfireBuildJob<TEngine>(
     IPlatformService platformService,
-    IRepository<TranslationEngine> engines,
+    IRepository<TEngine> engines,
     IDataAccessContext dataAccessContext,
-    IBuildJobService buildJobService,
-    ILogger<HangfireBuildJob> logger
-) : HangfireBuildJob<object?>(platformService, engines, dataAccessContext, buildJobService, logger)
+    IBuildJobService<TEngine> buildJobService,
+    ILogger<HangfireBuildJob<TEngine>> logger
+) : HangfireBuildJob<TEngine, object?>(platformService, engines, dataAccessContext, buildJobService, logger)
+    where TEngine : ITrainingEngine
 {
     public virtual Task RunAsync(
         string engineId,
@@ -19,24 +20,25 @@ CancellationToken cancellationToken
     }
 }
 
-public abstract class HangfireBuildJob<T>(
+public abstract class HangfireBuildJob<TEngine, TData>(
     IPlatformService platformService,
-    IRepository<TranslationEngine> engines,
+    IRepository<TEngine> engines,
     IDataAccessContext dataAccessContext,
-    IBuildJobService buildJobService,
-    ILogger<HangfireBuildJob<T>> logger
+    IBuildJobService<TEngine> buildJobService,
+    ILogger<HangfireBuildJob<TEngine, TData>> logger
 )
+    where TEngine : ITrainingEngine
 {
     protected IPlatformService PlatformService { get; } = platformService;
-    protected IRepository<TranslationEngine> Engines { get; } = engines;
+    protected IRepository<TEngine> Engines { get; } = engines;
     protected IDataAccessContext DataAccessContext { get; } = dataAccessContext;
-    protected IBuildJobService BuildJobService { get; } = buildJobService;
-    protected ILogger<HangfireBuildJob<T>> Logger { get; } = logger;
+    protected IBuildJobService<TEngine> BuildJobService { get; } = buildJobService;
+    protected ILogger<HangfireBuildJob<TEngine, TData>> Logger { get; } = logger;
 
     public virtual async Task RunAsync(
         string engineId,
         string buildId,
-        T data,
+        TData data,
         string? buildOptions,
         CancellationToken cancellationToken
     )
@@ -56,7 +58,7 @@ CancellationToken cancellationToken
         catch (OperationCanceledException)
         {
             // Check if the cancellation was initiated by an API call or a shutdown.
-            TranslationEngine? engine = await Engines.GetAsync(
+            TEngine? engine = await Engines.GetAsync(
                 e => e.EngineId == engineId && e.CurrentBuild != null && e.CurrentBuild.BuildId == buildId,
                 CancellationToken.None
             );
@@ -123,20 +125,30 @@ await BuildJobService.BuildJobFinishedAsync(
         }
     }
 
-    protected virtual Task InitializeAsync(string engineId, string buildId, T data, CancellationToken cancellationToken)
+    protected virtual Task InitializeAsync(
+        string engineId,
+        string buildId,
+        TData data,
+        CancellationToken cancellationToken
+    )
     {
         return Task.CompletedTask;
     }
 
     protected abstract Task DoWorkAsync(
         string engineId,
         string buildId,
-        T data,
+        TData data,
         string? buildOptions,
         CancellationToken cancellationToken
     );
 
-    protected virtual Task CleanupAsync(string engineId, string buildId, T data, JobCompletionStatus completionStatus)
+    protected virtual Task CleanupAsync(
+        string engineId,
+        string buildId,
+        TData data,
+        JobCompletionStatus completionStatus
+    )
     {
         return Task.CompletedTask;
     }

diff --git a/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJobRunner.cs b/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJobRunner.cs
@@ -1,9 +1,10 @@
 namespace Serval.Machine.Shared.Services;
 
-public class HangfireBuildJobRunner(
+public class HangfireBuildJobRunner<TEngine>(
     IBackgroundJobClient jobClient,
     IEnumerable<IHangfireBuildJobFactory> buildJobFactories
 ) : IBuildJobRunner
+    where TEngine : ITrainingEngine
 {
     public static Job CreateJob<TJob, TData>(
         string engineId,
@@ -12,7 +13,7 @@ public static Job CreateJob<TJob, TData>(
         object? data,
         string? buildOptions
     )
-        where TJob : HangfireBuildJob<TData>
+        where TJob : HangfireBuildJob<TEngine, TData>
     {
         ArgumentNullException.ThrowIfNull(data);
         // Token "None" is used here because hangfire injects the proper cancellation token
@@ -23,7 +24,7 @@ public static Job CreateJob<TJob, TData>(
     }
 
     public static Job CreateJob<TJob>(string engineId, string buildId, string queue, string? buildOptions)
-        where TJob : HangfireBuildJob
+        where TJob : HangfireBuildJob<TEngine>
     {
         // Token "None" is used here because hangfire injects the proper cancellation token
         return Job.FromExpression<TJob>(

diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobService.cs b/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobService.cs
@@ -3,7 +3,7 @@
 public interface IBuildJobService<TEngine>
     where TEngine : ITrainingEngine
 {
-    Task<IReadOnlyList<ITrainingEngine>> GetBuildingEnginesAsync(
+    Task<IReadOnlyList<TEngine>> GetBuildingEnginesAsync(
         BuildJobRunnerType runner,
         CancellationToken cancellationToken = default
     );

diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IClearMLQueueService.cs b/src/Machine/src/Serval.Machine.Shared/Services/IClearMLQueueService.cs
@@ -2,6 +2,5 @@
 
 public interface IClearMLQueueService
 {
-    public int GetQueueSize<TEnum>(TEnum engineType)
-        where TEnum : Enum;
+    public int GetQueueSize(EngineType engineType);
 }
diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs
@@ -1,6 +1,4 @@
-using static Serval.Machine.Shared.Services.HangfireBuildJobRunner;
-
-namespace Serval.Machine.Shared.Services;
+namespace Serval.Machine.Shared.Services;
 
 public class NmtHangfireBuildJobFactory : IHangfireBuildJobFactory
 {

diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs
@@ -5,12 +5,12 @@ public class NmtPreprocessBuildJob(
     IRepository<TranslationEngine> engines,
     IDataAccessContext dataAccessContext,
     ILogger<NmtPreprocessBuildJob> logger,
-    IBuildJobService buildJobService,
+    IBuildJobService<TranslationEngine> buildJobService,
     ISharedFileService sharedFileService,
     ICorpusService corpusService,
     ILanguageTagService languageTagService
 )
-    : PreprocessBuildJob(
+    : PreprocessBuildJob<TranslationEngine>(
         platformService,
         engines,
         dataAccessContext,

diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs
@@ -1,6 +1,7 @@
 namespace Serval.Machine.Shared.Services;
 
-public class PreprocessBuildJob : HangfireBuildJob<IReadOnlyList<ParallelCorpus>>
+public class PreprocessBuildJob<TEngine> : HangfireBuildJob<TEngine, IReadOnlyList<ParallelCorpus>>
+    where TEngine : ITrainingEngine
 {
     private static readonly JsonWriterOptions PretranslateWriterOptions = new() { Indented = true };
 
@@ -13,10 +14,10 @@ public class PreprocessBuildJob : HangfireBuildJob<IReadOnlyList<ParallelCorpus>
 
     public PreprocessBuildJob(
         IPlatformService platformService,
-        IRepository<TranslationEngine> engines,
+        IRepository<TEngine> engines,
         IDataAccessContext dataAccessContext,
-        ILogger<PreprocessBuildJob> logger,
-        IBuildJobService buildJobService,
+        ILogger<PreprocessBuildJob<TEngine>> logger,
+        IBuildJobService<TEngine> buildJobService,
         ISharedFileService sharedFileService,
         ICorpusService corpusService
     )

diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPreprocessBuildJob.cs
@@ -4,14 +4,14 @@ public class SmtTransferPreprocessBuildJob(
     IPlatformService platformService,
     IRepository<TranslationEngine> engines,
     IDataAccessContext dataAccessContext,
-    ILogger<PreprocessBuildJob> logger,
-    IBuildJobService buildJobService,
+    ILogger<SmtTransferPreprocessBuildJob> logger,
+    IBuildJobService<TranslationEngine> buildJobService,
     ISharedFileService sharedFileService,
     ICorpusService corpusService,
     IDistributedReaderWriterLockFactory lockFactory,
     IRepository<TrainSegmentPair> trainSegmentPairs
 )
-    : PreprocessBuildJob(
+    : PreprocessBuildJob<TranslationEngine>(
         platformService,
         engines,
         dataAccessContext,

diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferTrainBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferTrainBuildJob.cs
@@ -10,7 +10,7 @@ public class SmtTransferTrainBuildJob(
     ITruecaserFactory truecaserFactory,
     ISmtModelFactory smtModelFactory,
     ITransferEngineFactory transferEngineFactory
-) : HangfireBuildJob(platformService, engines, dataAccessContext, buildJobService, logger)
+) : HangfireBuildJob<TranslationEngine>(platformService, engines, dataAccessContext, buildJobService, logger)
 {
     private static readonly JsonWriterOptions PretranslateWriterOptions = new() { Indented = true };
     private static readonly JsonSerializerOptions JsonSerializerOptions =

diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs
@@ -147,9 +147,9 @@ public TestEnvironment()
                     ]
                 }
             );
-            BuildJobService = new BuildJobService(
+            BuildJobService = new BuildJobService<TranslationEngine>(
                 [
-                    new HangfireBuildJobRunner(_jobClient, [new NmtHangfireBuildJobFactory()]),
+                    new HangfireBuildJobRunner<TranslationEngine>(_jobClient, [new NmtHangfireBuildJobFactory()]),
                     new ClearMLBuildJobRunner(
                         ClearMLService,
                         [

diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs
@@ -754,9 +754,9 @@ public TestEnvironment()
                 )
                 .Returns(Task.FromResult("job1"));
             SharedFileService = new SharedFileService(Substitute.For<ILoggerFactory>());
-            BuildJobService = new BuildJobService(
+            BuildJobService = new BuildJobService<TranslationEngine>(
                 [
-                    new HangfireBuildJobRunner(
+                    new HangfireBuildJobRunner<TranslationEngine>(
                         Substitute.For<IBackgroundJobClient>(),
                         [new NmtHangfireBuildJobFactory()]
                     ),

diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs
@@ -319,9 +319,12 @@ public TestEnvironment(BuildJobRunnerType trainJobRunnerType = BuildJobRunnerTyp
                 buildJobOptions,
                 Substitute.For<ILogger<ClearMLMonitorService>>()
             );
-            BuildJobService = new BuildJobService(
+            BuildJobService = new BuildJobService<TranslationEngine>(
                 [
-                    new HangfireBuildJobRunner(_jobClient, [new SmtTransferHangfireBuildJobFactory()]),
+                    new HangfireBuildJobRunner<TranslationEngine>(
+                        _jobClient,
+                        [new SmtTransferHangfireBuildJobFactory()]
+                    ),
                     new ClearMLBuildJobRunner(
                         ClearMLService,
                         [new SmtTransferClearMLBuildJobFactory(SharedFileService, Engines)],