From c1dae96e01fb1df32dc1e67fa8df653b9ff65337 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Mon, 4 Nov 2024 12:40:26 -0500 Subject: [PATCH] a machine start --- .../appsettings.json | 15 +- .../Serval.Machine.JobServer/appsettings.json | 13 +- .../Configuration/ClearMLBuildQueue.cs | 2 +- .../Models/WordAlignmentEngine.cs | 13 ++ .../Services/ClearMLBuildJobRunner.cs | 2 +- .../Services/ClearMLMonitorService.cs | 31 ++- .../Services/IWordAlignmentEngineService.cs | 34 +++ .../ServalWordAlignmentEngineServiceV1.cs | 205 ++++++++++++++++++ .../Services/SmtTransferEngineService.cs | 3 +- .../Services/StatisticalEngineService.cs | 183 ++++++++++++++++ .../Services/WordAlignmentEngineType.cs | 6 + .../Services/NmtEngineServiceTests.cs | 4 +- .../Services/PreprocessBuildJobTests.cs | 4 +- .../Services/SmtTransferEngineServiceTests.cs | 4 +- 14 files changed, 493 insertions(+), 26 deletions(-) create mode 100644 src/Machine/src/Serval.Machine.Shared/Models/WordAlignmentEngine.cs create mode 100644 src/Machine/src/Serval.Machine.Shared/Services/IWordAlignmentEngineService.cs create mode 100644 src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentEngineServiceV1.cs create mode 100644 src/Machine/src/Serval.Machine.Shared/Services/StatisticalEngineService.cs create mode 100644 src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentEngineType.cs diff --git a/src/Machine/src/Serval.Machine.EngineServer/appsettings.json b/src/Machine/src/Serval.Machine.EngineServer/appsettings.json index f17d77f3..f6dc8eb0 100644 --- a/src/Machine/src/Serval.Machine.EngineServer/appsettings.json +++ b/src/Machine/src/Serval.Machine.EngineServer/appsettings.json @@ -10,18 +10,27 @@ "SmtTransfer", "Nmt" ], + "WordAlignmentEngines": [ + "Statistical" + ], "BuildJob": { "ClearML": [ { - "TranslationEngineType": "Nmt", + "EngineType": "Nmt", "ModelType": "huggingface", "Queue": "jobs_backlog", "DockerImage": "ghcr.io/sillsdev/machine.py:latest" }, { - "TranslationEngineType": "SmtTransfer", + "EngineType": "SmtTransfer", + "ModelType": "thot", + "Queue": "jobs_backlog.cpu_only", + "DockerImage": "ghcr.io/sillsdev/machine.py:latest" + }, + { + "EngineType": "Statistical", "ModelType": "thot", - "Queue": "cpu_only", + "Queue": "jobs_backlog.cpu_only", "DockerImage": "ghcr.io/sillsdev/machine.py:latest" } ] diff --git a/src/Machine/src/Serval.Machine.JobServer/appsettings.json b/src/Machine/src/Serval.Machine.JobServer/appsettings.json index d5aada0d..e6c1dda9 100644 --- a/src/Machine/src/Serval.Machine.JobServer/appsettings.json +++ b/src/Machine/src/Serval.Machine.JobServer/appsettings.json @@ -10,16 +10,25 @@ "SmtTransfer", "Nmt" ], + "WordAlignmentEngines": [ + "Statistical" + ], "BuildJob": { "ClearML": [ { - "TranslationEngineType": "Nmt", + "EngineType": "Nmt", "ModelType": "huggingface", "Queue": "jobs_backlog", "DockerImage": "ghcr.io/sillsdev/machine.py:latest" }, { - "TranslationEngineType": "SmtTransfer", + "EngineType": "SmtTransfer", + "ModelType": "thot", + "Queue": "jobs_backlog.cpu_only", + "DockerImage": "ghcr.io/sillsdev/machine.py:latest" + }, + { + "EngineType": "Statistical", "ModelType": "thot", "Queue": "jobs_backlog.cpu_only", "DockerImage": "ghcr.io/sillsdev/machine.py:latest" diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/ClearMLBuildQueue.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/ClearMLBuildQueue.cs index 53e25245..593424de 100644 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/ClearMLBuildQueue.cs +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/ClearMLBuildQueue.cs @@ -2,7 +2,7 @@ public class ClearMLBuildQueue { - public TranslationEngineType TranslationEngineType { get; set; } + public string EngineType { get; set; } = ""; public string ModelType { get; set; } = ""; public string Queue { get; set; } = "default"; public string DockerImage { get; set; } = ""; diff --git a/src/Machine/src/Serval.Machine.Shared/Models/WordAlignmentEngine.cs b/src/Machine/src/Serval.Machine.Shared/Models/WordAlignmentEngine.cs new file mode 100644 index 00000000..776f305c --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Models/WordAlignmentEngine.cs @@ -0,0 +1,13 @@ +namespace Serval.Machine.Shared.Models; + +public record WordAlignmentEngine : IEntity +{ + public string Id { get; set; } = ""; + public int Revision { get; set; } = 1; + public required string EngineId { get; init; } + public required WordAlignmentEngineType Type { get; init; } + public required string SourceLanguage { get; init; } + public required string TargetLanguage { get; init; } + public int BuildRevision { get; init; } + public Build? CurrentBuild { get; init; } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLBuildJobRunner.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLBuildJobRunner.cs index 910dd957..794f1b8b 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLBuildJobRunner.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLBuildJobRunner.cs @@ -11,7 +11,7 @@ IOptionsMonitor options buildJobFactories.ToDictionary(f => f.EngineType); private readonly Dictionary _options = - options.CurrentValue.ClearML.ToDictionary(o => o.TranslationEngineType); + options.CurrentValue.ClearML.ToDictionary(o => o.EngineType); public BuildJobRunnerType Type => BuildJobRunnerType.ClearML; diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs index c14be661..b256c7c7 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs @@ -26,20 +26,27 @@ ILogger logger private readonly ILogger _logger = logger; private readonly Dictionary _curBuildStatus = new(); - private readonly IReadOnlyDictionary _queuePerEngineType = - buildJobOptions.CurrentValue.ClearML.ToDictionary(x => x.TranslationEngineType, x => x.Queue); + private readonly IReadOnlyDictionary _queuePerEngineType = + buildJobOptions.CurrentValue.ClearML.ToDictionary(x => x.EngineType, x => x.Queue); - private readonly IDictionary _queueSizePerEngineType = new ConcurrentDictionary< - TranslationEngineType, - int - >(buildJobOptions.CurrentValue.ClearML.ToDictionary(x => x.TranslationEngineType, x => 0)); + private readonly IDictionary _queueSizePerEngineType = new ConcurrentDictionary( + buildJobOptions.CurrentValue.ClearML.ToDictionary(x => x.EngineType, x => 0) + ); - public int GetQueueSize(TranslationEngineType engineType) + public int GetQueueSize(TEnum engineType) + where TEnum : Enum { - return _queueSizePerEngineType[engineType]; + return _queueSizePerEngineType[engineType.ToString()]; } protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken) + { + await MonitorClearMLTasksPerDomain(scope, cancellationToken); + await MonitorClearMLTasksPerDomain(scope, cancellationToken); + } + + private async Task MonitorClearMLTasksPerDomain(IServiceScope scope, CancellationToken cancellationToken) + where TEngine : Enum { try { @@ -57,13 +64,13 @@ await _clearMLService.GetTasksByIdAsync( cancellationToken ) ).ToDictionary(t => t.Id); - Dictionary> queuePositionsPerEngineType = new(); + Dictionary> queuePositionsPerEngineType = new(); - foreach ((TranslationEngineType engineType, string queueName) in _queuePerEngineType) + foreach ((string engineType, string queueName) in _queuePerEngineType) { var tasksPerEngineType = tasks .Where(kvp => - trainingEngines.Where(te => te.CurrentBuild?.JobId == kvp.Key).FirstOrDefault()?.Type + trainingEngines.Where(te => te.CurrentBuild?.JobId == kvp.Key).FirstOrDefault()?.Type.ToString() == engineType ) .Select(kvp => kvp.Value) @@ -96,7 +103,7 @@ await UpdateTrainJobStatus( engine.CurrentBuild.BuildId, new ProgressStatus(step: 0, percentCompleted: 0.0), //CurrentBuild.BuildId should always equal the corresponding task.Name - queuePositionsPerEngineType[engine.Type][engine.CurrentBuild.BuildId] + 1, + queuePositionsPerEngineType[engine.Type.ToString()][engine.CurrentBuild.BuildId] + 1, cancellationToken ); } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IWordAlignmentEngineService.cs b/src/Machine/src/Serval.Machine.Shared/Services/IWordAlignmentEngineService.cs new file mode 100644 index 00000000..8495efce --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/IWordAlignmentEngineService.cs @@ -0,0 +1,34 @@ +namespace Serval.Machine.Shared.Services; + +public interface IWordAlignmentEngineService +{ + WordAlignmentEngineType WordAlignmentEngine { get; } + + Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + bool? isModelPersisted = null, + CancellationToken cancellationToken = default + ); + Task DeleteAsync(string engineId, CancellationToken cancellationToken = default); + + Task GetBestPhraseAlignmentAsync( + string sourceSegment, + string targetSegment, + CancellationToken cancellationToken = default + ); + + Task StartBuildAsync( + string engineId, + string buildId, + string? buildOptions, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ); + + Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default); + + int GetQueueSize(); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentEngineServiceV1.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentEngineServiceV1.cs new file mode 100644 index 00000000..2010c237 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentEngineServiceV1.cs @@ -0,0 +1,205 @@ +using Google.Protobuf.WellKnownTypes; +using Serval.WordAlignment.V1; + +namespace Serval.Machine.Shared.Services; + +public class ServalWordAlignmentEngineServiceV1(IEnumerable engineServices) + : WordAlignmentEngineApi.WordAlignmentEngineApiBase +{ + private static readonly Empty Empty = new(); + + private readonly Dictionary _engineServices = + engineServices.ToDictionary(es => es.WordAlignmentEngine); + + public override async Task Create(CreateRequest request, ServerCallContext context) + { + IWordAlignmentEngineService engineService = GetEngineService(request.EngineType); + await engineService.CreateAsync( + request.EngineId, + request.HasEngineName ? request.EngineName : null, + request.SourceLanguage, + request.TargetLanguage, + isModelPersisted: true, + cancellationToken: context.CancellationToken + ); + return Empty; + } + + public override async Task Delete(DeleteRequest request, ServerCallContext context) + { + IWordAlignmentEngineService engineService = GetEngineService(request.EngineType); + await engineService.DeleteAsync(request.EngineId, context.CancellationToken); + return Empty; + } + + public override async Task GetWordAlignment( + GetWordAlignmentRequest request, + ServerCallContext context + ) + { + IWordAlignmentEngineService engineService = GetEngineService(request.EngineType); + TranslationResult result; + try + { + result = await engineService.GetBestPhraseAlignmentAsync( + request.SourceSegment, + request.TargetSegment, + context.CancellationToken + ); + } + catch (EngineNotBuiltException e) + { + throw new RpcException(new Status(StatusCode.Aborted, e.Message, e)); + } + + return new GetWordAlignmentResponse { Result = Map(result) }; + } + + public override async Task StartBuild(StartBuildRequest request, ServerCallContext context) + { + IWordAlignmentEngineService engineService = GetEngineService(request.EngineType); + Models.ParallelCorpus[] corpora = request.Corpora.Select(Map).ToArray(); + try + { + await engineService.StartBuildAsync( + request.EngineId, + request.BuildId, + request.HasOptions ? request.Options : null, + corpora, + context.CancellationToken + ); + } + catch (InvalidOperationException e) + { + throw new RpcException(new Status(StatusCode.Aborted, e.Message, e)); + } + return Empty; + } + + public override async Task CancelBuild(CancelBuildRequest request, ServerCallContext context) + { + IWordAlignmentEngineService engineService = GetEngineService(request.EngineType); + try + { + await engineService.CancelBuildAsync(request.EngineId, context.CancellationToken); + } + catch (InvalidOperationException e) + { + throw new RpcException(new Status(StatusCode.Aborted, e.Message, e)); + } + return Empty; + } + + public override Task GetQueueSize(GetQueueSizeRequest request, ServerCallContext context) + { + IWordAlignmentEngineService engineService = GetEngineService(request.EngineType); + return Task.FromResult(new GetQueueSizeResponse { Size = engineService.GetQueueSize() }); + } + + private IWordAlignmentEngineService GetEngineService(string engineTypeStr) + { + if (_engineServices.TryGetValue(GetEngineType(engineTypeStr), out IWordAlignmentEngineService? service)) + return service; + throw new RpcException(new Status(StatusCode.InvalidArgument, "The engine type is invalid.")); + } + + private static WordAlignmentEngineType GetEngineType(string engineTypeStr) + { + engineTypeStr = engineTypeStr[0].ToString().ToUpperInvariant() + engineTypeStr[1..]; + if (System.Enum.TryParse(engineTypeStr, out WordAlignmentEngineType engineType)) + return engineType; + throw new RpcException(new Status(StatusCode.InvalidArgument, "The engine type is invalid.")); + } + + private static WordAlignmentResult Map(TranslationResult source) + { + return new WordAlignmentResult + { + SourceTokens = { source.SourceTokens }, + TargetTokens = { source.TargetTokens }, + Confidences = { source.Confidences }, + Alignment = { Map(source.Alignment) }, + }; + } + + private static IEnumerable Map(WordAlignmentMatrix source) + { + for (int i = 0; i < source.RowCount; i++) + { + for (int j = 0; j < source.ColumnCount; j++) + { + if (source[i, j]) + yield return new WordAlignment.V1.AlignedWordPair { SourceIndex = i, TargetIndex = j }; + } + } + } + + private static Models.ParallelCorpus Map(WordAlignment.V1.ParallelCorpus source) + { + return new Models.ParallelCorpus + { + Id = source.Id, + SourceCorpora = source.SourceCorpora.Select(Map).ToList(), + TargetCorpora = source.TargetCorpora.Select(Map).ToList() + }; + } + + private static Models.MonolingualCorpus Map(WordAlignment.V1.MonolingualCorpus source) + { + var trainOnChapters = source.TrainOnChapters.ToDictionary( + kvp => kvp.Key, + kvp => kvp.Value.Chapters.ToHashSet() + ); + var trainOnTextIds = source.TrainOnTextIds.ToHashSet(); + FilterChoice trainingFilter = GetFilterChoice(trainOnChapters, trainOnTextIds); + + var pretranslateChapters = source.WordAlignOnChapters.ToDictionary( + kvp => kvp.Key, + kvp => kvp.Value.Chapters.ToHashSet() + ); + var pretranslateTextIds = source.WordAlignOnTextIds.ToHashSet(); + FilterChoice pretranslateFilter = GetFilterChoice(pretranslateChapters, pretranslateTextIds); + + return new Models.MonolingualCorpus + { + Id = source.Id, + Language = source.Language, + Files = source.Files.Select(Map).ToList(), + TrainOnChapters = trainingFilter == FilterChoice.Chapters ? trainOnChapters : null, + TrainOnTextIds = trainingFilter == FilterChoice.TextIds ? trainOnTextIds : null, + PretranslateChapters = pretranslateFilter == FilterChoice.Chapters ? pretranslateChapters : null, + PretranslateTextIds = pretranslateFilter == FilterChoice.TextIds ? pretranslateTextIds : null + }; + } + + private static Models.CorpusFile Map(WordAlignment.V1.CorpusFile source) + { + return new Models.CorpusFile + { + Location = source.Location, + Format = (Models.FileFormat)source.Format, + TextId = source.TextId + }; + } + + private enum FilterChoice + { + Chapters, + TextIds, + None + } + + private static FilterChoice GetFilterChoice( + IReadOnlyDictionary> chapters, + HashSet textIds + ) + { + // Only either textIds or Scripture Range will be used at a time + // TextIds may be an empty array, so prefer that if both are empty (which applies to both scripture and text) + if (chapters is null && textIds is null) + return FilterChoice.None; + if (chapters is null || chapters.Count == 0) + return FilterChoice.TextIds; + return FilterChoice.Chapters; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs index 7c4f10b4..60e8c9a8 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs @@ -9,7 +9,7 @@ public class SmtTransferEngineService( SmtTransferEngineStateService stateService, IBuildJobService buildJobService, IClearMLQueueService clearMLQueueService -) : ITranslationEngineService +) : ITranslationEngineService, IWordAlignmentEngineService { private readonly IDistributedReaderWriterLockFactory _lockFactory = lockFactory; private readonly IPlatformService _platformService = platformService; @@ -21,6 +21,7 @@ IClearMLQueueService clearMLQueueService private readonly IClearMLQueueService _clearMLQueueService = clearMLQueueService; public TranslationEngineType Type => TranslationEngineType.SmtTransfer; + public WordAlignmentEngineType WordAlignmentType => WordAlignmentEngineType.Statistical; public async Task CreateAsync( string engineId, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/StatisticalEngineService.cs b/src/Machine/src/Serval.Machine.Shared/Services/StatisticalEngineService.cs new file mode 100644 index 00000000..63fcf111 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/StatisticalEngineService.cs @@ -0,0 +1,183 @@ +namespace Serval.Machine.Shared.Services; + +public class StatisticalEngineService( + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + SmtTransferEngineStateService stateService, + IBuildJobService buildJobService, + IClearMLQueueService clearMLQueueService +) : IWordAlignmentEngineService +{ + private readonly IDistributedReaderWriterLockFactory _lockFactory = lockFactory; + private readonly IPlatformService _platformService = platformService; + private readonly IDataAccessContext _dataAccessContext = dataAccessContext; + private readonly IRepository _engines = engines; + private readonly SmtTransferEngineStateService _stateService = stateService; + private readonly IBuildJobService _buildJobService = buildJobService; + private readonly IClearMLQueueService _clearMLQueueService = clearMLQueueService; + + public WordAlignmentEngineType Type => WordAlignmentEngineType.Statistical; + + public async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + WordAlignmentEngine wordAlignmentEngine = await _dataAccessContext.WithTransactionAsync( + async ct => + { + var waEngine = new WordAlignmentEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage, + Type = WordAlignmentEngineType.Statistical, + }; + await _engines.InsertAsync(waEngine, ct); + await _buildJobService.CreateEngineAsync(engineId, engineName, ct); + return waEngine; + }, + cancellationToken: cancellationToken + ); + + SmtTransferEngineState state = _stateService.Get(engineId); + state.InitNew(); + return wordAlignmentEngine; + } + + public async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await CancelBuildJobAsync(engineId, cancellationToken); + + await _dataAccessContext.WithTransactionAsync( + async ct => + { + await _engines.DeleteAsync(e => e.EngineId == engineId, ct); + }, + cancellationToken: cancellationToken + ); + await _buildJobService.DeleteEngineAsync(engineId, CancellationToken.None); + + SmtTransferEngineState state = _stateService.Get(engineId); + _stateService.Remove(engineId); + // there is no way to cancel this call + state.DeleteData(); + state.Dispose(); + await _lockFactory.DeleteAsync(engineId, CancellationToken.None); + } + + public async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + SmtTransferEngineState state = _stateService.Get(engineId); + + IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); + IReadOnlyList results = await @lock.ReaderLockAsync( + async ct => + { + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision, ct); + // there is no way to cancel this call + return hybridEngine.Translate(n, segment); + }, + cancellationToken: cancellationToken + ); + + state.Touch(); + return results; + } + + public async Task StartBuildAsync( + string engineId, + string buildId, + string? buildOptions, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + bool building = !await _buildJobService.StartBuildJobAsync( + BuildJobRunnerType.Hangfire, + TranslationEngineType.SmtTransfer, + engineId, + buildId, + BuildStage.Preprocess, + corpora, + buildOptions, + cancellationToken + ); + // If there is a pending/running build, then no need to start a new one. + if (building) + throw new InvalidOperationException("The engine is already building or in the process of canceling."); + + SmtTransferEngineState state = _stateService.Get(engineId); + state.Touch(); + } + + public async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + bool building = await CancelBuildJobAsync(engineId, cancellationToken); + if (!building) + throw new InvalidOperationException("The engine is not currently building."); + + SmtTransferEngineState state = _stateService.Get(engineId); + state.Touch(); + } + + public int GetQueueSize() + { + return _clearMLQueueService.GetQueueSize(Type); + } + + public bool IsLanguageNativeToModel(string language, out string internalCode) + { + throw new NotSupportedException("SMT transfer engines do not support language info."); + } + + private async Task CancelBuildJobAsync(string engineId, CancellationToken cancellationToken) + { + string? buildId = null; + await _dataAccessContext.WithTransactionAsync( + async ct => + { + (buildId, BuildJobState jobState) = await _buildJobService.CancelBuildJobAsync(engineId, ct); + if (buildId is not null && jobState is BuildJobState.None) + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + }, + cancellationToken: cancellationToken + ); + return buildId is not null; + } + + public Task GetModelDownloadUrlAsync( + string engineId, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await _engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException($"The engine {engineId} does not exist."); + return engine; + } + + private async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildRevision == 0) + throw new EngineNotBuiltException("The engine must be built first."); + return engine; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentEngineType.cs b/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentEngineType.cs new file mode 100644 index 00000000..a8ed74cb --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentEngineType.cs @@ -0,0 +1,6 @@ +namespace Serval.Machine.Shared.Services; + +public enum WordAlignmentEngineType +{ + Statistical, +} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs index 67145c01..f0d131a1 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs @@ -132,14 +132,14 @@ public TestEnvironment() [ new ClearMLBuildQueue() { - TranslationEngineType = TranslationEngineType.Nmt, + EngineType = TranslationEngineType.Nmt.ToString(), ModelType = "huggingface", DockerImage = "default", Queue = "default" }, new ClearMLBuildQueue() { - TranslationEngineType = TranslationEngineType.SmtTransfer, + EngineType = TranslationEngineType.SmtTransfer.ToString(), ModelType = "thot", DockerImage = "default", Queue = "default" diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs index d29f2213..022add57 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs @@ -719,14 +719,14 @@ public TestEnvironment() [ new ClearMLBuildQueue() { - TranslationEngineType = TranslationEngineType.Nmt, + EngineType = TranslationEngineType.Nmt.ToString(), ModelType = "huggingface", DockerImage = "default", Queue = "default" }, new ClearMLBuildQueue() { - TranslationEngineType = TranslationEngineType.SmtTransfer, + EngineType = TranslationEngineType.SmtTransfer.ToString(), ModelType = "thot", DockerImage = "default", Queue = "default" diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs index 6b888794..f5fabd6e 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs @@ -277,14 +277,14 @@ public TestEnvironment(BuildJobRunnerType trainJobRunnerType = BuildJobRunnerTyp [ new ClearMLBuildQueue() { - TranslationEngineType = TranslationEngineType.Nmt, + EngineType = TranslationEngineType.Nmt.ToString().ToString(), ModelType = "huggingface", DockerImage = "default", Queue = "default" }, new ClearMLBuildQueue() { - TranslationEngineType = TranslationEngineType.SmtTransfer, + EngineType = TranslationEngineType.SmtTransfer.ToString(), ModelType = "thot", DockerImage = "default", Queue = "default"