diff --git a/src/Machine/src/Serval.Machine.Shared/Models/Corpus.cs b/src/Machine/src/Serval.Machine.Shared/Models/TranslationCorpus.cs similarity index 95% rename from src/Machine/src/Serval.Machine.Shared/Models/Corpus.cs rename to src/Machine/src/Serval.Machine.Shared/Models/TranslationCorpus.cs index 9145e90d..4643eee3 100644 --- a/src/Machine/src/Serval.Machine.Shared/Models/Corpus.cs +++ b/src/Machine/src/Serval.Machine.Shared/Models/TranslationCorpus.cs @@ -1,6 +1,6 @@ namespace Serval.Machine.Shared.Models; -public record Corpus +public record TranslationCorpus { public required string Id { get; init; } public required string SourceLanguage { get; init; } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ITranslationEngineService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ITranslationEngineService.cs index 71ed5d94..37ccc157 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ITranslationEngineService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ITranslationEngineService.cs @@ -35,7 +35,7 @@ Task StartBuildAsync( string engineId, string buildId, string? buildOptions, - IReadOnlyList corpora, + IReadOnlyList corpora, CancellationToken cancellationToken = default ); diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtEngineService.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtEngineService.cs index fc1c2c95..1f752da6 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/NmtEngineService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/NmtEngineService.cs @@ -69,7 +69,7 @@ public async Task StartBuildAsync( string engineId, string buildId, string? buildOptions, - IReadOnlyList corpora, + IReadOnlyList corpora, CancellationToken cancellationToken = default ) { diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs index a8b3d52f..9c7d7a7c 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs @@ -11,7 +11,7 @@ public Job CreateJob(string engineId, string buildId, BuildStage stage, object? return stage switch { BuildStage.Preprocess - => CreateJob>( + => CreateJob>( engineId, buildId, "nmt", diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs index 97e5fc77..595e3a8c 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs @@ -1,6 +1,6 @@ namespace Serval.Machine.Shared.Services; -public class PreprocessBuildJob : HangfireBuildJob> +public class PreprocessBuildJob : HangfireBuildJob> { private static readonly JsonWriterOptions PretranslateWriterOptions = new() { Indented = true }; @@ -43,7 +43,7 @@ internal int Seed protected override async Task DoWorkAsync( string engineId, string buildId, - IReadOnlyList data, + IReadOnlyList data, string? buildOptions, CancellationToken cancellationToken ) @@ -99,7 +99,7 @@ CancellationToken cancellationToken private async Task<(int TrainCount, int PretranslateCount)> WriteDataFilesAsync( string buildId, - IReadOnlyList corpora, + IReadOnlyList corpora, string? buildOptions, CancellationToken cancellationToken ) @@ -121,7 +121,7 @@ CancellationToken cancellationToken int trainCount = 0; int pretranslateCount = 0; pretranslateWriter.WriteStartArray(); - foreach (Corpus corpus in corpora) + foreach (TranslationCorpus corpus in corpora) { ITextCorpus[] sourceTextCorpora = _corpusService.CreateTextCorpora(corpus.SourceFiles).ToArray(); ITextCorpus targetTextCorpus = @@ -204,7 +204,7 @@ CancellationToken cancellationToken protected override async Task CleanupAsync( string engineId, string buildId, - IReadOnlyList data, + IReadOnlyList data, JobCompletionStatus completionStatus ) { @@ -221,12 +221,12 @@ JobCompletionStatus completionStatus } } - private static bool IsInTrain(Row row, Corpus corpus) + private static bool IsInTrain(Row row, TranslationCorpus corpus) { return IsIncluded(row, corpus.TrainOnTextIds, corpus.TrainOnChapters); } - private static bool IsInPretranslate(Row row, Corpus corpus) + private static bool IsInPretranslate(Row row, TranslationCorpus corpus) { return IsIncluded(row, corpus.PretranslateTextIds, corpus.PretranslateChapters); } @@ -255,7 +255,7 @@ private static bool IsInChapters(IReadOnlyDictionary> bookC } private static IEnumerable AlignTrainCorpus( - Corpus corpus, + TranslationCorpus corpus, IReadOnlyList srcCorpora, ITextCorpus trgCorpus ) @@ -379,7 +379,11 @@ ITextCorpus trgCorpus } } - private static IEnumerable AlignPretranslateCorpus(Corpus corpus, ITextCorpus srcCorpus, ITextCorpus trgCorpus) + private static IEnumerable AlignPretranslateCorpus( + TranslationCorpus corpus, + ITextCorpus srcCorpus, + ITextCorpus trgCorpus + ) { IEnumerable? textIds = corpus.PretranslateChapters is not null ? corpus.PretranslateChapters.Keys diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs index 049889b9..54449a9c 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs @@ -91,7 +91,7 @@ await engineService.TrainSegmentPairAsync( public override async Task StartBuild(StartBuildRequest request, ServerCallContext context) { ITranslationEngineService engineService = GetEngineService(request.EngineType); - Models.Corpus[] corpora = request.Corpora.Select(Map).ToArray(); + Models.TranslationCorpus[] corpora = request.Corpora.Select(Map).ToArray(); try { await engineService.StartBuildAsync( @@ -269,7 +269,7 @@ private static Translation.V1.Phrase Map(SIL.Machine.Translation.Phrase source) }; } - private static Models.Corpus Map(Translation.V1.Corpus source) + private static Models.TranslationCorpus Map(Translation.V1.Corpus source) { var pretranslateChapters = source.PretranslateChapters.ToDictionary( kvp => kvp.Key, @@ -283,7 +283,7 @@ private static Models.Corpus Map(Translation.V1.Corpus source) ); FilterChoice trainingFilter = GetFilterChoice(source.TrainOnAll, trainOnChapters); - return new Models.Corpus + return new Models.TranslationCorpus { Id = source.Id, SourceLanguage = source.SourceLanguage, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs index 5789d67d..3980f92e 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs @@ -180,7 +180,7 @@ public async Task StartBuildAsync( string engineId, string buildId, string? buildOptions, - IReadOnlyList corpora, + IReadOnlyList corpora, CancellationToken cancellationToken = default ) { diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferHangfireBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferHangfireBuildJobFactory.cs index 2d9bf00c..dbdb42f8 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferHangfireBuildJobFactory.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferHangfireBuildJobFactory.cs @@ -11,7 +11,7 @@ public Job CreateJob(string engineId, string buildId, BuildStage stage, object? return stage switch { BuildStage.Preprocess - => CreateJob>( + => CreateJob>( engineId, buildId, "smt_transfer", diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPreprocessBuildJob.cs index 9e14037a..9d9785cb 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPreprocessBuildJob.cs @@ -27,7 +27,7 @@ IRepository trainSegmentPairs protected override async Task InitializeAsync( string engineId, string buildId, - IReadOnlyList data, + IReadOnlyList data, CancellationToken cancellationToken ) { diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs index 19a15a25..040f05e8 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs @@ -9,7 +9,7 @@ public async Task StartBuildAsync() using var env = new TestEnvironment(); TranslationEngine engine = env.Engines.Get("engine1"); Assert.That(engine.BuildRevision, Is.EqualTo(1)); - await env.Service.StartBuildAsync("engine1", "build1", "{}", Array.Empty()); + await env.Service.StartBuildAsync("engine1", "build1", "{}", Array.Empty()); await env.WaitForBuildToFinishAsync(); engine = env.Engines.Get("engine1"); Assert.Multiple(() => @@ -28,7 +28,7 @@ public async Task CancelBuildAsync_Building() TranslationEngine engine = env.Engines.Get("engine1"); Assert.That(engine.BuildRevision, Is.EqualTo(1)); - await env.Service.StartBuildAsync("engine1", "build1", "{}", Array.Empty()); + await env.Service.StartBuildAsync("engine1", "build1", "{}", Array.Empty()); await env.WaitForBuildToStartAsync(); engine = env.Engines.Get("engine1"); Assert.That(engine.CurrentBuild, Is.Not.Null); @@ -55,7 +55,7 @@ public async Task DeleteAsync_WhileBuilding() TranslationEngine engine = env.Engines.Get("engine1"); Assert.That(engine.BuildRevision, Is.EqualTo(1)); - await env.Service.StartBuildAsync("engine1", "build1", "{}", Array.Empty()); + await env.Service.StartBuildAsync("engine1", "build1", "{}", Array.Empty()); await env.WaitForBuildToStartAsync(); engine = env.Engines.Get("engine1"); Assert.That(engine.CurrentBuild, Is.Not.Null); diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs index df7498ee..83534cca 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs @@ -7,7 +7,7 @@ public class PreprocessBuildJobTests public async Task RunAsync_FilterOutEverything() { using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { }; + TranslationCorpus corpus1 = env.DefaultTextFileCorpus with { }; await env.RunBuildJobAsync(corpus1); @@ -25,7 +25,7 @@ public async Task RunAsync_FilterOutEverything() public async Task RunAsync_TrainOnAll() { using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { TrainOnTextIds = null }; + TranslationCorpus corpus1 = env.DefaultTextFileCorpus with { TrainOnTextIds = null }; await env.RunBuildJobAsync(corpus1); @@ -43,7 +43,7 @@ public async Task RunAsync_TrainOnAll() public async Task RunAsync_TrainOnTextIds() { using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { TrainOnTextIds = ["textId1"] }; + TranslationCorpus corpus1 = env.DefaultTextFileCorpus with { TrainOnTextIds = ["textId1"] }; await env.RunBuildJobAsync(corpus1); @@ -61,7 +61,11 @@ public async Task RunAsync_TrainOnTextIds() public async Task RunAsync_TrainAndPretranslateAll() { using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { PretranslateTextIds = null, TrainOnTextIds = null }; + TranslationCorpus corpus1 = env.DefaultTextFileCorpus with + { + PretranslateTextIds = null, + TrainOnTextIds = null + }; await env.RunBuildJobAsync(corpus1); @@ -72,7 +76,7 @@ public async Task RunAsync_TrainAndPretranslateAll() public async Task RunAsync_PretranslateAll() { using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { PretranslateTextIds = null }; + TranslationCorpus corpus1 = env.DefaultTextFileCorpus with { PretranslateTextIds = null }; await env.RunBuildJobAsync(corpus1); @@ -83,7 +87,11 @@ public async Task RunAsync_PretranslateAll() public async Task RunAsync_PretranslateTextIds() { using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { PretranslateTextIds = ["textId1"], TrainOnTextIds = null }; + TranslationCorpus corpus1 = env.DefaultTextFileCorpus with + { + PretranslateTextIds = ["textId1"], + TrainOnTextIds = null + }; await env.RunBuildJobAsync(corpus1); @@ -94,7 +102,7 @@ public async Task RunAsync_PretranslateTextIds() public async Task RunAsync_EnableKeyTerms() { using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultParatextCorpus with { }; + TranslationCorpus corpus1 = env.DefaultParatextCorpus with { }; await env.RunBuildJobAsync(corpus1, useKeyTerms: true); @@ -112,7 +120,7 @@ public async Task RunAsync_EnableKeyTerms() public async Task RunAsync_DisableKeyTerms() { using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultParatextCorpus with { }; + TranslationCorpus corpus1 = env.DefaultParatextCorpus with { }; await env.RunBuildJobAsync(corpus1, useKeyTerms: false); @@ -130,7 +138,7 @@ public async Task RunAsync_DisableKeyTerms() public async Task RunAsync_PretranslateChapters() { using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultParatextCorpus with + TranslationCorpus corpus1 = env.DefaultParatextCorpus with { PretranslateChapters = new Dictionary> { @@ -150,7 +158,7 @@ public async Task RunAsync_PretranslateChapters() public async Task RunAsync_TrainOnChapters() { using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultParatextCorpus with + TranslationCorpus corpus1 = env.DefaultParatextCorpus with { TrainOnChapters = new Dictionary> { @@ -177,7 +185,7 @@ public async Task RunAsync_TrainOnChapters() public async Task RunAsync_MixedSource_Paratext() { using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultMixedSourceParatextCorpus with + TranslationCorpus corpus1 = env.DefaultMixedSourceParatextCorpus with { TrainOnTextIds = null, PretranslateTextIds = null @@ -200,7 +208,7 @@ public async Task RunAsync_MixedSource_Paratext() public async Task RunAsync_MixedSource_Text() { using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultMixedSourceTextFileCorpus with + TranslationCorpus corpus1 = env.DefaultMixedSourceTextFileCorpus with { TrainOnTextIds = null, PretranslateTextIds = null, @@ -225,7 +233,7 @@ public async Task RunAsync_MixedSource_Text() public void RunAsync_UnknownLanguageTagsNoData() { using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { SourceLanguage = "xxx", TargetLanguage = "zzz" }; + TranslationCorpus corpus1 = env.DefaultTextFileCorpus with { SourceLanguage = "xxx", TargetLanguage = "zzz" }; Assert.ThrowsAsync(async () => { @@ -237,7 +245,7 @@ public void RunAsync_UnknownLanguageTagsNoData() public async Task RunAsync_UnknownLanguageTagsNoDataSmtTransfer() { using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { SourceLanguage = "xxx", TargetLanguage = "zzz" }; + TranslationCorpus corpus1 = env.DefaultTextFileCorpus with { SourceLanguage = "xxx", TargetLanguage = "zzz" }; await env.RunBuildJobAsync(corpus1, engineId: "engine2", engineType: TranslationEngineType.SmtTransfer); } @@ -246,7 +254,7 @@ public async Task RunAsync_UnknownLanguageTagsNoDataSmtTransfer() public async Task RunAsync_RemoveFreestandingEllipses() { using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultParatextCorpus with + TranslationCorpus corpus1 = env.DefaultParatextCorpus with { TrainOnChapters = new Dictionary> { @@ -286,7 +294,7 @@ public async Task RunAsync_RemoveFreestandingEllipses() public void RunAsync_OnlyParseSelectedBooks_NoBadBooks() { using TestEnvironment env = new(); - Corpus corpus = env.DefaultParatextCorpus with + TranslationCorpus corpus = env.DefaultParatextCorpus with { TrainOnTextIds = new() { "LEV" }, PretranslateTextIds = new() { "MRK" } @@ -310,7 +318,7 @@ public void RunAsync_OnlyParseSelectedBooks_NoBadBooks() public void RunAsync_OnlyParseSelectedBooks_TrainOnBadBook() { using TestEnvironment env = new(); - Corpus corpus = env.DefaultParatextCorpus with + TranslationCorpus corpus = env.DefaultParatextCorpus with { TrainOnTextIds = new() { "MAT" }, PretranslateTextIds = new() { "MRK" } @@ -333,7 +341,7 @@ public void RunAsync_OnlyParseSelectedBooks_TrainOnBadBook() public void RunAsync_OnlyParseSelectedBooks_PretranslateOnBadBook() { using TestEnvironment env = new(); - Corpus corpus = env.DefaultParatextCorpus with + TranslationCorpus corpus = env.DefaultParatextCorpus with { TrainOnTextIds = new() { "LEV" }, PretranslateTextIds = new() { "MAT" } @@ -375,10 +383,10 @@ private class TestEnvironment : DisposableBase public IClearMLService ClearMLService { get; } public IOptionsMonitor BuildJobOptions { get; } - public Corpus DefaultTextFileCorpus { get; } - public Corpus DefaultMixedSourceTextFileCorpus { get; } - public Corpus DefaultParatextCorpus { get; } - public Corpus DefaultMixedSourceParatextCorpus { get; } + public TranslationCorpus DefaultTextFileCorpus { get; } + public TranslationCorpus DefaultMixedSourceTextFileCorpus { get; } + public TranslationCorpus DefaultParatextCorpus { get; } + public TranslationCorpus DefaultMixedSourceParatextCorpus { get; } public TestEnvironment() { @@ -614,7 +622,7 @@ public PreprocessBuildJob GetBuildJob(TranslationEngineType engineType) } public Task RunBuildJobAsync( - Corpus corpus, + TranslationCorpus corpus, bool useKeyTerms = true, string engineId = "engine1", TranslationEngineType engineType = TranslationEngineType.Nmt diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs index 1a798d97..56a8e17e 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs @@ -40,7 +40,7 @@ await env.Service.StartBuildAsync( BuildId1, null, [ - new Corpus() + new TranslationCorpus() { Id = CorpusId1, SourceLanguage = "es", @@ -103,7 +103,7 @@ public async Task StartBuildAsync_RestartUnfinishedBuild() using var env = new TestEnvironment(BuildJobRunnerType.Hangfire); env.UseInfiniteTrainJob(); - await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); + await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); await env.WaitForTrainingToStartAsync(); TranslationEngine engine = env.Engines.Get(EngineId1); Assert.That(engine.CurrentBuild, Is.Not.Null); @@ -128,7 +128,7 @@ public async Task DeleteAsync_WhileBuilding(BuildJobRunnerType trainJobRunnerTyp using var env = new TestEnvironment(trainJobRunnerType); env.UseInfiniteTrainJob(); - await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); + await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); await env.WaitForTrainingToStartAsync(); TranslationEngine engine = env.Engines.Get(EngineId1); Assert.That(engine.CurrentBuild, Is.Not.Null); @@ -148,7 +148,7 @@ public async Task TrainSegmentPairAsync(BuildJobRunnerType trainJobRunnerType) using var env = new TestEnvironment(trainJobRunnerType); env.UseInfiniteTrainJob(); - await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); + await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); await env.WaitForBuildToStartAsync(); TranslationEngine engine = env.Engines.Get(EngineId1); Assert.That(engine.CurrentBuild, Is.Not.Null); diff --git a/src/Serval/src/Serval.Assessment/Models/AssessmentEngine.cs b/src/Serval/src/Serval.Assessment/Models/AssessmentEngine.cs new file mode 100644 index 00000000..970d0be1 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Models/AssessmentEngine.cs @@ -0,0 +1,7 @@ +namespace Serval.Assessment.Models; + +public record AssessmentEngine : Engine +{ + public required Corpus Corpus { get; init; } + public Corpus? ReferenceCorpus { get; init; } +} diff --git a/src/Serval/src/Serval.Translation/Models/Build.cs b/src/Serval/src/Serval.Assessment/Models/AssessmentJob.cs similarity index 61% rename from src/Serval/src/Serval.Translation/Models/Build.cs rename to src/Serval/src/Serval.Assessment/Models/AssessmentJob.cs index 2c67ba79..c6e2e1e0 100644 --- a/src/Serval/src/Serval.Translation/Models/Build.cs +++ b/src/Serval/src/Serval.Assessment/Models/AssessmentJob.cs @@ -1,17 +1,15 @@ -namespace Serval.Translation.Models; +namespace Serval.Assessment.Models; -public record Build : IEntity +public record AssessmentJob : Job { public string Id { get; set; } = ""; public int Revision { get; set; } = 1; public string? Name { get; init; } public required string EngineRef { get; init; } - public IReadOnlyList? TrainOn { get; init; } - public IReadOnlyList? Pretranslate { get; init; } - public int Step { get; init; } + public IReadOnlyList? TextIds { get; set; } + public string? ScriptureRange { get; set; } public double? PercentCompleted { get; init; } public string? Message { get; init; } - public int? QueueDepth { get; init; } public JobState State { get; init; } = JobState.Pending; public DateTime? DateFinished { get; init; } public IReadOnlyDictionary? Options { get; init; } diff --git a/src/Serval/src/Serval.Assessment/Models/CorpusFile.cs b/src/Serval/src/Serval.Assessment/Models/CorpusFile.cs deleted file mode 100644 index fa491558..00000000 --- a/src/Serval/src/Serval.Assessment/Models/CorpusFile.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Serval.Assessment.Models; - -public record CorpusFile -{ - public required string Id { get; init; } - public required string Filename { get; init; } - public required FileFormat Format { get; init; } - public required string TextId { get; init; } -} diff --git a/src/Serval/src/Serval.Assessment/Services/EngineService.cs b/src/Serval/src/Serval.Assessment/Services/EngineService.cs index 01ced93a..fb6bc8b8 100644 --- a/src/Serval/src/Serval.Assessment/Services/EngineService.cs +++ b/src/Serval/src/Serval.Assessment/Services/EngineService.cs @@ -65,9 +65,9 @@ await _dataAccessContext.WithTransactionAsync( ); } - public async Task ReplaceCorpusAsync( + public async Task ReplaceCorpusAsync( string id, - Models.Corpus corpus, + Shared.Models.Corpus corpus, CancellationToken cancellationToken = default ) { @@ -81,9 +81,9 @@ await _dataAccessContext.WithTransactionAsync( return engine.Corpus; } - public async Task ReplaceReferenceCorpusAsync( + public async Task ReplaceReferenceCorpusAsync( string id, - Models.Corpus referenceCorpus, + Shared.Models.Corpus referenceCorpus, CancellationToken cancellationToken = default ) { diff --git a/src/Serval/src/Serval.Translation/Contracts/AlignedWordPairDto.cs b/src/Serval/src/Serval.Shared/Contracts/AlignedWordPairDto.cs similarity index 76% rename from src/Serval/src/Serval.Translation/Contracts/AlignedWordPairDto.cs rename to src/Serval/src/Serval.Shared/Contracts/AlignedWordPairDto.cs index 4cd0dd66..5b9c63f6 100644 --- a/src/Serval/src/Serval.Translation/Contracts/AlignedWordPairDto.cs +++ b/src/Serval/src/Serval.Shared/Contracts/AlignedWordPairDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Shared.Contracts; public record AlignedWordPairDto { diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusFileConfigDto.cs b/src/Serval/src/Serval.Shared/Contracts/CorpusFileConfigDto.cs similarity index 52% rename from src/Serval/src/Serval.Translation/Contracts/TranslationCorpusFileConfigDto.cs rename to src/Serval/src/Serval.Shared/Contracts/CorpusFileConfigDto.cs index e9b967ec..fc7591e2 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusFileConfigDto.cs +++ b/src/Serval/src/Serval.Shared/Contracts/CorpusFileConfigDto.cs @@ -1,6 +1,6 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Shared.Contracts; -public record TranslationCorpusFileConfigDto +public record CorpusFileConfigDto { public required string FileId { get; init; } diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusFileDto.cs b/src/Serval/src/Serval.Shared/Contracts/CorpusFileDto.cs similarity index 55% rename from src/Serval/src/Serval.Translation/Contracts/TranslationCorpusFileDto.cs rename to src/Serval/src/Serval.Shared/Contracts/CorpusFileDto.cs index f5787408..5ccab8f9 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusFileDto.cs +++ b/src/Serval/src/Serval.Shared/Contracts/CorpusFileDto.cs @@ -1,6 +1,6 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Shared.Contracts; -public record TranslationCorpusFileDto +public record CorpusFileDto { public required ResourceLinkDto File { get; init; } public string? TextId { get; init; } diff --git a/src/Serval/src/Serval.Translation/Contracts/TrainingCorpusConfigDto.cs b/src/Serval/src/Serval.Shared/Contracts/CorpusFilterConfigDto.cs similarity index 84% rename from src/Serval/src/Serval.Translation/Contracts/TrainingCorpusConfigDto.cs rename to src/Serval/src/Serval.Shared/Contracts/CorpusFilterConfigDto.cs index db933fc3..d2efb239 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TrainingCorpusConfigDto.cs +++ b/src/Serval/src/Serval.Shared/Contracts/CorpusFilterConfigDto.cs @@ -1,6 +1,6 @@ namespace Serval.Translation.Contracts; -public record TrainingCorpusConfigDto +public record CorpusFilterConfigDto { public required string CorpusId { get; init; } public IReadOnlyList? TextIds { get; init; } diff --git a/src/Serval/src/Serval.Translation/Contracts/TrainingCorpusDto.cs b/src/Serval/src/Serval.Shared/Contracts/CorpusFilterDto.cs similarity index 86% rename from src/Serval/src/Serval.Translation/Contracts/TrainingCorpusDto.cs rename to src/Serval/src/Serval.Shared/Contracts/CorpusFilterDto.cs index 563f4fed..bf8eb3d8 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TrainingCorpusDto.cs +++ b/src/Serval/src/Serval.Shared/Contracts/CorpusFilterDto.cs @@ -1,6 +1,6 @@ namespace Serval.Translation.Contracts; -public record TrainingCorpusDto +public record CorpusFilterDto { public required ResourceLinkDto Corpus { get; init; } diff --git a/src/Serval/src/Serval.Translation/Contracts/QueueDto.cs b/src/Serval/src/Serval.Shared/Contracts/QueueDto.cs similarity index 75% rename from src/Serval/src/Serval.Translation/Contracts/QueueDto.cs rename to src/Serval/src/Serval.Shared/Contracts/QueueDto.cs index 51c75357..ade49773 100644 --- a/src/Serval/src/Serval.Translation/Contracts/QueueDto.cs +++ b/src/Serval/src/Serval.Shared/Contracts/QueueDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Shared.Contracts; public record QueueDto { diff --git a/src/Serval/src/Serval.Shared/Contracts/TrainingCorpusConfigDto.cs b/src/Serval/src/Serval.Shared/Contracts/TrainingCorpusConfigDto.cs new file mode 100644 index 00000000..9c39161d --- /dev/null +++ b/src/Serval/src/Serval.Shared/Contracts/TrainingCorpusConfigDto.cs @@ -0,0 +1,17 @@ +namespace Serval.Shared.Contracts; + +public record TrainingCorpusConfigDto +{ + /// + /// The corpus name. + /// + public string? Name { get; init; } + + public required string SourceLanguage { get; init; } + + public required string TargetLanguage { get; init; } + + public required IReadOnlyList SourceFiles { get; init; } + + public required IReadOnlyList TargetFiles { get; init; } +} diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusDto.cs b/src/Serval/src/Serval.Shared/Contracts/TrainingCorpusDto.cs similarity index 54% rename from src/Serval/src/Serval.Translation/Contracts/TranslationCorpusDto.cs rename to src/Serval/src/Serval.Shared/Contracts/TrainingCorpusDto.cs index da770f53..9054de28 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusDto.cs +++ b/src/Serval/src/Serval.Shared/Contracts/TrainingCorpusDto.cs @@ -1,6 +1,6 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Shared.Contracts; -public record TranslationCorpusDto +public record TrainingCorpusDto { public required string Id { get; init; } public required string Url { get; init; } @@ -8,6 +8,6 @@ public record TranslationCorpusDto public string? Name { get; init; } public required string SourceLanguage { get; init; } public required string TargetLanguage { get; init; } - public required IReadOnlyList SourceFiles { get; init; } - public required IReadOnlyList TargetFiles { get; init; } + public required IReadOnlyList SourceFiles { get; init; } + public required IReadOnlyList TargetFiles { get; init; } } diff --git a/src/Serval/src/Serval.Translation/Models/AlignedWordPair.cs b/src/Serval/src/Serval.Shared/Models/AlignedWordPair.cs similarity index 77% rename from src/Serval/src/Serval.Translation/Models/AlignedWordPair.cs rename to src/Serval/src/Serval.Shared/Models/AlignedWordPair.cs index 5e367495..550abeb0 100644 --- a/src/Serval/src/Serval.Translation/Models/AlignedWordPair.cs +++ b/src/Serval/src/Serval.Shared/Models/AlignedWordPair.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Models; +namespace Serval.Shared.Models; public record AlignedWordPair { diff --git a/src/Serval/src/Serval.Assessment/Models/Corpus.cs b/src/Serval/src/Serval.Shared/Models/Corpus.cs similarity index 82% rename from src/Serval/src/Serval.Assessment/Models/Corpus.cs rename to src/Serval/src/Serval.Shared/Models/Corpus.cs index 33ef0981..8ade27d0 100644 --- a/src/Serval/src/Serval.Assessment/Models/Corpus.cs +++ b/src/Serval/src/Serval.Shared/Models/Corpus.cs @@ -1,4 +1,4 @@ -namespace Serval.Assessment.Models; +namespace Serval.Shared.Models; public record Corpus { diff --git a/src/Serval/src/Serval.Translation/Models/CorpusFile.cs b/src/Serval/src/Serval.Shared/Models/CorpusFile.cs similarity index 84% rename from src/Serval/src/Serval.Translation/Models/CorpusFile.cs rename to src/Serval/src/Serval.Shared/Models/CorpusFile.cs index 2672ba56..2739e605 100644 --- a/src/Serval/src/Serval.Translation/Models/CorpusFile.cs +++ b/src/Serval/src/Serval.Shared/Models/CorpusFile.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Models; +namespace Serval.Shared.Models; public record CorpusFile { diff --git a/src/Serval/src/Serval.Assessment/Models/Engine.cs b/src/Serval/src/Serval.Shared/Models/Engine.cs similarity index 65% rename from src/Serval/src/Serval.Assessment/Models/Engine.cs rename to src/Serval/src/Serval.Shared/Models/Engine.cs index 337b8875..4336cbf9 100644 --- a/src/Serval/src/Serval.Assessment/Models/Engine.cs +++ b/src/Serval/src/Serval.Shared/Models/Engine.cs @@ -1,12 +1,12 @@ -namespace Serval.Assessment.Models; +namespace Serval.Shared.Models; public record Engine : IOwnedEntity { public string Id { get; set; } = ""; public int Revision { get; set; } = 1; - public required string Owner { get; init; } public string? Name { get; init; } public required string Type { get; init; } - public required Corpus Corpus { get; init; } - public Corpus? ReferenceCorpus { get; init; } + public required string Owner { get; init; } + public bool IsBuilding { get; init; } + public int ModelRevision { get; init; } } diff --git a/src/Serval/src/Serval.Translation/Models/TrainingCorpus.cs b/src/Serval/src/Serval.Shared/Models/FilteredCorpus.cs similarity index 72% rename from src/Serval/src/Serval.Translation/Models/TrainingCorpus.cs rename to src/Serval/src/Serval.Shared/Models/FilteredCorpus.cs index e91fb1ea..82eb7f19 100644 --- a/src/Serval/src/Serval.Translation/Models/TrainingCorpus.cs +++ b/src/Serval/src/Serval.Shared/Models/FilteredCorpus.cs @@ -1,6 +1,6 @@ -namespace Serval.Translation.Models; +namespace Serval.Shared.Models; -public record TrainingCorpus +public record FilteredCorpus { public required string CorpusRef { get; set; } = default!; public IReadOnlyList? TextIds { get; set; } diff --git a/src/Serval/src/Serval.Assessment/Models/Job.cs b/src/Serval/src/Serval.Shared/Models/Job.cs similarity index 77% rename from src/Serval/src/Serval.Assessment/Models/Job.cs rename to src/Serval/src/Serval.Shared/Models/Job.cs index c1863e46..362a94ed 100644 --- a/src/Serval/src/Serval.Assessment/Models/Job.cs +++ b/src/Serval/src/Serval.Shared/Models/Job.cs @@ -1,4 +1,4 @@ -namespace Serval.Assessment.Models; +namespace Serval.Shared.Models; public record Job : IEntity { @@ -6,8 +6,6 @@ public record Job : IEntity public int Revision { get; set; } = 1; public string? Name { get; init; } public required string EngineRef { get; init; } - public IReadOnlyList? TextIds { get; set; } - public string? ScriptureRange { get; set; } public double? PercentCompleted { get; init; } public string? Message { get; init; } public JobState State { get; init; } = JobState.Pending; diff --git a/src/Serval/src/Serval.Translation/Models/Queue.cs b/src/Serval/src/Serval.Shared/Models/Queue.cs similarity index 76% rename from src/Serval/src/Serval.Translation/Models/Queue.cs rename to src/Serval/src/Serval.Shared/Models/Queue.cs index 3e269b2c..7a3272a8 100644 --- a/src/Serval/src/Serval.Translation/Models/Queue.cs +++ b/src/Serval/src/Serval.Shared/Models/Queue.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Models; +namespace Serval.Shared.Models; public record Queue { diff --git a/src/Serval/src/Serval.Shared/Models/TrainingBuildJob.cs b/src/Serval/src/Serval.Shared/Models/TrainingBuildJob.cs new file mode 100644 index 00000000..0016f1c2 --- /dev/null +++ b/src/Serval/src/Serval.Shared/Models/TrainingBuildJob.cs @@ -0,0 +1,8 @@ +namespace Serval.Shared.Models; + +public record TrainingBuildJob : Job +{ + public IReadOnlyList? TrainOn { get; init; } + public int Step { get; init; } + public int? QueueDepth { get; init; } +} diff --git a/src/Serval/src/Serval.Translation/Models/Corpus.cs b/src/Serval/src/Serval.Shared/Models/TrainingCorpus.cs similarity index 84% rename from src/Serval/src/Serval.Translation/Models/Corpus.cs rename to src/Serval/src/Serval.Shared/Models/TrainingCorpus.cs index e8e3a4d4..d9d1d115 100644 --- a/src/Serval/src/Serval.Translation/Models/Corpus.cs +++ b/src/Serval/src/Serval.Shared/Models/TrainingCorpus.cs @@ -1,6 +1,6 @@ -namespace Serval.Translation.Models; +namespace Serval.Shared.Models; -public record Corpus +public record TrainingCorpus { public string Id { get; set; } = ""; public string? Name { get; set; } diff --git a/src/Serval/src/Serval.Shared/Models/TrainingEngine.cs b/src/Serval/src/Serval.Shared/Models/TrainingEngine.cs new file mode 100644 index 00000000..d602f578 --- /dev/null +++ b/src/Serval/src/Serval.Shared/Models/TrainingEngine.cs @@ -0,0 +1,11 @@ +namespace Serval.Shared.Models; + +public record TrainingEngine : Engine +{ + public required string SourceLanguage { get; init; } + public required string TargetLanguage { get; init; } + public required IReadOnlyList Corpora { get; init; } + public bool? IsModelPersisted { get; init; } + public double Confidence { get; init; } + public int CorpusSize { get; init; } +} diff --git a/src/Serval/src/Serval.Shared/Services/IEngineService.cs b/src/Serval/src/Serval.Shared/Services/IEngineService.cs new file mode 100644 index 00000000..0570cf37 --- /dev/null +++ b/src/Serval/src/Serval.Shared/Services/IEngineService.cs @@ -0,0 +1,15 @@ +namespace Serval.Shared.Services; + +public interface IEngineService +{ + Task> GetAllAsync(string owner, CancellationToken cancellationToken = default); + Task GetAsync(string engineId, CancellationToken cancellationToken = default); + + Task CreateAsync(TEngine engine, CancellationToken cancellationToken = default); + Task DeleteAsync(string engineId, CancellationToken cancellationToken = default); + Task StartJobAsync(TJob build, CancellationToken cancellationToken = default); + + Task CancelJobAsync(string engineId, CancellationToken cancellationToken = default); + + Task GetQueueAsync(string engineType, CancellationToken cancellationToken = default); +} diff --git a/src/Serval/src/Serval.Shared/Services/IJobService.cs b/src/Serval/src/Serval.Shared/Services/IJobService.cs new file mode 100644 index 00000000..cf025820 --- /dev/null +++ b/src/Serval/src/Serval.Shared/Services/IJobService.cs @@ -0,0 +1,18 @@ +namespace Serval.Shared.Services; + +public interface IJobService +{ + Task> GetAllAsync(string parentId, CancellationToken cancellationToken = default); + Task GetAsync(string id, CancellationToken cancellationToken = default); + Task GetActiveAsync(string parentId, CancellationToken cancellationToken = default); + Task> GetNewerRevisionAsync( + string id, + long minRevision, + CancellationToken cancellationToken = default + ); + Task> GetActiveNewerRevisionAsync( + string parentId, + long minRevision, + CancellationToken cancellationToken = default + ); +} diff --git a/src/Serval/src/Serval.Shared/Services/ITrainingEngineService.cs b/src/Serval/src/Serval.Shared/Services/ITrainingEngineService.cs new file mode 100644 index 00000000..b54deda6 --- /dev/null +++ b/src/Serval/src/Serval.Shared/Services/ITrainingEngineService.cs @@ -0,0 +1,21 @@ +namespace Serval.Shared.Services; + +public interface ITrainingEngineService : IEngineService +{ + Task AddCorpusAsync(string engineId, TrainingCorpus corpus, CancellationToken cancellationToken = default); + Task UpdateCorpusAsync( + string engineId, + string corpusId, + IReadOnlyList? sourceFiles, + IReadOnlyList? targetFiles, + CancellationToken cancellationToken = default + ); + Task DeleteCorpusAsync( + string engineId, + string corpusId, + bool deleteFiles, + CancellationToken cancellationToken = default + ); + + Task DeleteAllCorpusFilesAsync(string dataFileId, CancellationToken cancellationToken = default); +} diff --git a/src/Serval/src/Serval.Shared/Usings.cs b/src/Serval/src/Serval.Shared/Usings.cs index 3e84144f..fb72d2d0 100644 --- a/src/Serval/src/Serval.Shared/Usings.cs +++ b/src/Serval/src/Serval.Shared/Usings.cs @@ -12,6 +12,7 @@ global using Microsoft.Extensions.Logging; global using Microsoft.Extensions.Options; global using Serval.Shared.Configuration; +global using Serval.Shared.Contracts; global using Serval.Shared.Models; global using Serval.Shared.Services; global using Serval.Shared.Utils; diff --git a/src/Serval/src/Serval.Translation/Configuration/IMemoryDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.Translation/Configuration/IMemoryDataAccessConfiguratorExtensions.cs index fa858640..f24d3afc 100644 --- a/src/Serval/src/Serval.Translation/Configuration/IMemoryDataAccessConfiguratorExtensions.cs +++ b/src/Serval/src/Serval.Translation/Configuration/IMemoryDataAccessConfiguratorExtensions.cs @@ -7,7 +7,7 @@ this IMemoryDataAccessConfigurator configurator ) { configurator.AddRepository(); - configurator.AddRepository(); + configurator.AddRepository(); configurator.AddRepository(); return configurator; } diff --git a/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs index ea016c0e..7545c880 100644 --- a/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs +++ b/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs @@ -17,11 +17,13 @@ await c.Indexes.CreateOrUpdateAsync( ); } ); - configurator.AddRepository( + configurator.AddRepository( "translation.builds", init: c => c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(b => b.EngineRef)) + new CreateIndexModel( + Builders.IndexKeys.Ascending(b => b.EngineRef) + ) ) ); configurator.AddRepository( diff --git a/src/Serval/src/Serval.Translation/Configuration/IServalBuilderExtensions.cs b/src/Serval/src/Serval.Translation/Configuration/IServalBuilderExtensions.cs index 190d627f..b84a18d8 100644 --- a/src/Serval/src/Serval.Translation/Configuration/IServalBuilderExtensions.cs +++ b/src/Serval/src/Serval.Translation/Configuration/IServalBuilderExtensions.cs @@ -23,7 +23,7 @@ public static IServalBuilder AddTranslation( builder.Services.AddScoped(); builder.Services.AddScoped(); - builder.Services.AddScoped(); + builder.Services.AddScoped(); var translationOptions = new TranslationOptions(); builder.Configuration?.GetSection(TranslationOptions.Key).Bind(translationOptions); diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationBuildConfigDto.cs b/src/Serval/src/Serval.Translation/Contracts/TranslationBuildConfigDto.cs index e8024216..b77dd988 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationBuildConfigDto.cs +++ b/src/Serval/src/Serval.Translation/Contracts/TranslationBuildConfigDto.cs @@ -3,7 +3,7 @@ public record TranslationBuildConfigDto { public string? Name { get; init; } - public IReadOnlyList? TrainOn { get; init; } + public IReadOnlyList? TrainOn { get; init; } public IReadOnlyList? Pretranslate { get; init; } /// diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs b/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs index 741ff4ba..2470819d 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs +++ b/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs @@ -7,7 +7,7 @@ public record TranslationBuildDto public required int Revision { get; init; } public string? Name { get; init; } public required ResourceLinkDto Engine { get; init; } - public IReadOnlyList? TrainOn { get; init; } + public IReadOnlyList? TrainOn { get; init; } public IReadOnlyList? Pretranslate { get; init; } public required int Step { get; init; } public double? PercentCompleted { get; init; } diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusConfigDto.cs b/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusConfigDto.cs deleted file mode 100644 index d865a4c9..00000000 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusConfigDto.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace Serval.Translation.Contracts; - -public record TranslationCorpusConfigDto -{ - /// - /// The corpus name. - /// - public string? Name { get; init; } - - public required string SourceLanguage { get; init; } - - public required string TargetLanguage { get; init; } - - public required IReadOnlyList SourceFiles { get; init; } - - public required IReadOnlyList TargetFiles { get; init; } -} diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusUpdateConfigDto.cs b/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusUpdateConfigDto.cs index 8dd96311..766252a0 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusUpdateConfigDto.cs +++ b/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusUpdateConfigDto.cs @@ -4,9 +4,9 @@ namespace Serval.Translation.Contracts; public record TranslationCorpusUpdateConfigDto : IValidatableObject { - public IReadOnlyList? SourceFiles { get; init; } + public IReadOnlyList? SourceFiles { get; init; } - public IReadOnlyList? TargetFiles { get; init; } + public IReadOnlyList? TargetFiles { get; init; } public IEnumerable Validate( ValidationContext validationContext diff --git a/src/Serval/src/Serval.Translation/Controllers/TranslationEngineTypesController.cs b/src/Serval/src/Serval.Translation/Controllers/TranslationEngineTypesController.cs index 4e5458d1..89288a13 100644 --- a/src/Serval/src/Serval.Translation/Controllers/TranslationEngineTypesController.cs +++ b/src/Serval/src/Serval.Translation/Controllers/TranslationEngineTypesController.cs @@ -3,10 +3,12 @@ [ApiVersion(1.0)] [Route("api/v{version:apiVersion}/translation/engine-types")] [OpenApiTag("Translation Engines")] -public class TranslationEngineTypesController(IAuthorizationService authService, IEngineService engineService) - : ServalControllerBase(authService) +public class TranslationEngineTypesController( + IAuthorizationService authService, + ITranslationEngineService engineService +) : ServalControllerBase(authService) { - private readonly IEngineService _engineService = engineService; + private readonly ITranslationEngineService _engineService = engineService; /// /// Get queue information for a given engine type diff --git a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs b/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs index 31d69846..587ba56a 100644 --- a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs +++ b/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs @@ -5,7 +5,7 @@ [OpenApiTag("Translation Engines")] public class TranslationEnginesController( IAuthorizationService authService, - IEngineService engineService, + ITranslationEngineService engineService, IBuildService buildService, IPretranslationService pretranslationService, IOptionsMonitor apiOptions, @@ -16,7 +16,7 @@ ILogger logger private static readonly JsonSerializerOptions ObjectJsonSerializerOptions = new() { Converters = { new ObjectToInferredTypesConverter() } }; - private readonly IEngineService _engineService = engineService; + private readonly ITranslationEngineService _engineService = engineService; private readonly IBuildService _buildService = buildService; private readonly IPretranslationService _pretranslationService = pretranslationService; private readonly IOptionsMonitor _apiOptions = apiOptions; @@ -352,9 +352,9 @@ await _engineService.TrainSegmentPairAsync( [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task> AddCorpusAsync( + public async Task> AddCorpusAsync( [NotNull] string id, - [FromBody] TranslationCorpusConfigDto corpusConfig, + [FromBody] TrainingCorpusConfigDto corpusConfig, [FromServices] IRequestClient getDataFileClient, [FromServices] IIdGenerator idGenerator, CancellationToken cancellationToken @@ -362,9 +362,14 @@ CancellationToken cancellationToken { Engine engine = await _engineService.GetAsync(id, cancellationToken); await AuthorizeAsync(engine); - Corpus corpus = await MapAsync(getDataFileClient, idGenerator.GenerateId(), corpusConfig, cancellationToken); + TrainingCorpus corpus = await MapAsync( + getDataFileClient, + idGenerator.GenerateId(), + corpusConfig, + cancellationToken + ); await _engineService.AddCorpusAsync(id, corpus, cancellationToken); - TranslationCorpusDto dto = Map(id, corpus); + TrainingCorpusDto dto = Map(id, corpus); return Created(dto.Url, dto); } @@ -394,7 +399,7 @@ CancellationToken cancellationToken [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task> UpdateCorpusAsync( + public async Task> UpdateCorpusAsync( [NotNull] string id, [NotNull] string corpusId, [FromBody] TranslationCorpusUpdateConfigDto corpusConfig, @@ -403,7 +408,7 @@ CancellationToken cancellationToken ) { await AuthorizeAsync(id, cancellationToken); - Corpus corpus = await _engineService.UpdateCorpusAsync( + TrainingCorpus corpus = await _engineService.UpdateCorpusAsync( id, corpusId, corpusConfig.SourceFiles is null @@ -434,12 +439,12 @@ corpusConfig.TargetFiles is null [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task>> GetAllCorporaAsync( + public async Task>> GetAllCorporaAsync( [NotNull] string id, CancellationToken cancellationToken ) { - Engine engine = await _engineService.GetAsync(id, cancellationToken); + TrainingEngine engine = await _engineService.GetAsync(id, cancellationToken); await AuthorizeAsync(engine); return Ok(engine.Corpora.Select(c => Map(id, c))); } @@ -462,7 +467,7 @@ CancellationToken cancellationToken [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task> GetCorpusAsync( + public async Task> GetCorpusAsync( [NotNull] string id, [NotNull] string corpusId, CancellationToken cancellationToken @@ -780,7 +785,7 @@ CancellationToken cancellationToken await AuthorizeAsync(id, cancellationToken); if (minRevision != null) { - (_, EntityChange change) = await TaskEx.Timeout( + (_, EntityChange change) = await TaskEx.Timeout( ct => _buildService.GetNewerRevisionAsync(buildId, minRevision.Value, ct), _apiOptions.CurrentValue.LongPollTimeout, cancellationToken: cancellationToken @@ -794,7 +799,7 @@ CancellationToken cancellationToken } else { - Build build = await _buildService.GetAsync(buildId, cancellationToken); + TrainingBuildJob build = await _buildService.GetAsync(buildId, cancellationToken); return Ok(Map(build)); } } @@ -847,7 +852,7 @@ CancellationToken cancellationToken { Engine engine = await _engineService.GetAsync(id, cancellationToken); await AuthorizeAsync(engine); - Build build = Map(engine, buildConfig); + TrainingBuildJob build = Map(engine, buildConfig); await _engineService.StartBuildAsync(build, cancellationToken); TranslationBuildDto dto = Map(build); @@ -890,7 +895,7 @@ CancellationToken cancellationToken await AuthorizeAsync(id, cancellationToken); if (minRevision != null) { - (_, EntityChange change) = await TaskEx.Timeout( + (_, EntityChange change) = await TaskEx.Timeout( ct => _buildService.GetActiveNewerRevisionAsync(id, minRevision.Value, ct), _apiOptions.CurrentValue.LongPollTimeout, cancellationToken: cancellationToken @@ -904,7 +909,7 @@ CancellationToken cancellationToken } else { - Build? build = await _buildService.GetActiveAsync(id, cancellationToken); + TrainingBuildJob? build = await _buildService.GetActiveAsync(id, cancellationToken); if (build == null) return NoContent(); @@ -989,14 +994,14 @@ private async Task AuthorizeAsync(string id, CancellationToken cancellationToken await AuthorizeAsync(engine); } - private async Task MapAsync( + private async Task MapAsync( IRequestClient getDataFileClient, string corpusId, - TranslationCorpusConfigDto source, + TrainingCorpusConfigDto source, CancellationToken cancellationToken ) { - return new Corpus + return new TrainingCorpus { Id = corpusId, Name = source.Name, @@ -1009,12 +1014,12 @@ CancellationToken cancellationToken private async Task> MapAsync( IRequestClient getDataFileClient, - IEnumerable fileConfigs, + IEnumerable fileConfigs, CancellationToken cancellationToken ) { var files = new List(); - foreach (TranslationCorpusFileConfigDto fileConfig in fileConfigs) + foreach (CorpusFileConfigDto fileConfig in fileConfigs) { Response response = await getDataFileClient.GetResponse< DataFileResult, @@ -1054,9 +1059,9 @@ private Engine Map(TranslationEngineConfigDto source) }; } - private static Build Map(Engine engine, TranslationBuildConfigDto source) + private static TrainingBuildJob Map(Engine engine, TranslationBuildConfigDto source) { - return new Build + return new TrainingBuildJob { EngineRef = engine.Id, Name = source.Name, @@ -1099,14 +1104,14 @@ private static Build Map(Engine engine, TranslationBuildConfigDto source) return pretranslateCorpora; } - private static List? Map(Engine engine, IReadOnlyList? source) + private static List? Map(Engine engine, IReadOnlyList? source) { if (source is null) return null; var corpusIds = new HashSet(engine.Corpora.Select(c => c.Id)); - var trainOnCorpora = new List(); - foreach (TrainingCorpusConfigDto tcc in source) + var trainOnCorpora = new List(); + foreach (CorpusFilterConfigDto tcc in source) { if (!corpusIds.Contains(tcc.CorpusId)) { @@ -1121,7 +1126,7 @@ private static Build Map(Engine engine, TranslationBuildConfigDto source) ); } trainOnCorpora.Add( - new TrainingCorpus + new FilteredCorpus { CorpusRef = tcc.CorpusId, TextIds = tcc.TextIds?.ToList(), @@ -1165,7 +1170,7 @@ private TranslationEngineDto Map(Engine source) }; } - private TranslationBuildDto Map(Build source) + private TranslationBuildDto Map(TrainingBuildJob source) { return new TranslationBuildDto { @@ -1207,7 +1212,7 @@ private PretranslateCorpusDto Map(string engineId, PretranslateCorpus source) }; } - private TrainingCorpusDto Map(string engineId, TrainingCorpus source) + private TrainingCorpusDto Map(string engineId, FilteredCorpus source) { return new TrainingCorpusDto { @@ -1290,9 +1295,9 @@ private static PretranslationDto Map(Pretranslation source) }; } - private TranslationCorpusDto Map(string engineId, Corpus source) + private TrainingCorpusDto Map(string engineId, TrainingCorpus source) { - return new TranslationCorpusDto + return new TrainingCorpusDto { Id = source.Id, Url = _urlService.GetUrl(Endpoints.GetTranslationCorpus, new { id = engineId, corpusId = source.Id }), @@ -1309,9 +1314,9 @@ private TranslationCorpusDto Map(string engineId, Corpus source) }; } - private TranslationCorpusFileDto Map(CorpusFile source) + private CorpusFileDto Map(CorpusFile source) { - return new TranslationCorpusFileDto + return new CorpusFileDto { File = new ResourceLinkDto { diff --git a/src/Serval/src/Serval.Translation/Models/Engine.cs b/src/Serval/src/Serval.Translation/Models/Engine.cs deleted file mode 100644 index 9548ad17..00000000 --- a/src/Serval/src/Serval.Translation/Models/Engine.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace Serval.Translation.Models; - -public record Engine : IOwnedEntity -{ - public string Id { get; set; } = ""; - public int Revision { get; set; } = 1; - public string? Name { get; init; } - public required string SourceLanguage { get; init; } - public required string TargetLanguage { get; init; } - public required string Type { get; init; } - public required string Owner { get; init; } - public required IReadOnlyList Corpora { get; init; } - public bool? IsModelPersisted { get; init; } - public bool IsBuilding { get; init; } - public int ModelRevision { get; init; } - public double Confidence { get; init; } - public int CorpusSize { get; init; } -} diff --git a/src/Serval/src/Serval.Translation/Models/TranslationBuildJob.cs b/src/Serval/src/Serval.Translation/Models/TranslationBuildJob.cs new file mode 100644 index 00000000..7a1aaa3f --- /dev/null +++ b/src/Serval/src/Serval.Translation/Models/TranslationBuildJob.cs @@ -0,0 +1,6 @@ +namespace Serval.Translation.Models; + +public record TranslationBuildJob : TrainingBuildJob +{ + public IReadOnlyList? Pretranslate { get; init; } +} diff --git a/src/Serval/src/Serval.Translation/Services/BuildService.cs b/src/Serval/src/Serval.Translation/Services/BuildService.cs index e3d652d3..2210f082 100644 --- a/src/Serval/src/Serval.Translation/Services/BuildService.cs +++ b/src/Serval/src/Serval.Translation/Services/BuildService.cs @@ -1,13 +1,18 @@ namespace Serval.Translation.Services; -public class BuildService(IRepository builds) : EntityServiceBase(builds), IBuildService +public class BuildService(IRepository builds) + : EntityServiceBase(builds), + IBuildService { - public async Task> GetAllAsync(string parentId, CancellationToken cancellationToken = default) + public async Task> GetAllAsync( + string parentId, + CancellationToken cancellationToken = default + ) { return await Entities.GetAllAsync(e => e.EngineRef == parentId, cancellationToken); } - public Task GetActiveAsync(string parentId, CancellationToken cancellationToken = default) + public Task GetActiveAsync(string parentId, CancellationToken cancellationToken = default) { return Entities.GetAsync( b => b.EngineRef == parentId && (b.State == JobState.Active || b.State == JobState.Pending), @@ -15,7 +20,7 @@ public async Task> GetAllAsync(string parentId, CancellationT ); } - public Task> GetNewerRevisionAsync( + public Task> GetNewerRevisionAsync( string id, long minRevision, CancellationToken cancellationToken = default @@ -24,7 +29,7 @@ public Task> GetNewerRevisionAsync( return GetNewerRevisionAsync(e => e.Id == id, minRevision, cancellationToken); } - public Task> GetActiveNewerRevisionAsync( + public Task> GetActiveNewerRevisionAsync( string parentId, long minRevision, CancellationToken cancellationToken = default @@ -37,14 +42,14 @@ public Task> GetActiveNewerRevisionAsync( ); } - private async Task> GetNewerRevisionAsync( - Expression> filter, + private async Task> GetNewerRevisionAsync( + Expression> filter, long minRevision, CancellationToken cancellationToken = default ) { - using ISubscription subscription = await Entities.SubscribeAsync(filter, cancellationToken); - EntityChange curChange = subscription.Change; + using ISubscription subscription = await Entities.SubscribeAsync(filter, cancellationToken); + EntityChange curChange = subscription.Change; if (curChange.Type == EntityChangeType.Delete && minRevision > 1) return curChange; while (true) diff --git a/src/Serval/src/Serval.Translation/Services/IBuildService.cs b/src/Serval/src/Serval.Translation/Services/IBuildService.cs index 4fc13dee..f365138c 100644 --- a/src/Serval/src/Serval.Translation/Services/IBuildService.cs +++ b/src/Serval/src/Serval.Translation/Services/IBuildService.cs @@ -2,15 +2,15 @@ public interface IBuildService { - Task> GetAllAsync(string parentId, CancellationToken cancellationToken = default); - Task GetAsync(string id, CancellationToken cancellationToken = default); - Task GetActiveAsync(string parentId, CancellationToken cancellationToken = default); - Task> GetNewerRevisionAsync( + Task> GetAllAsync(string parentId, CancellationToken cancellationToken = default); + Task GetAsync(string id, CancellationToken cancellationToken = default); + Task GetActiveAsync(string parentId, CancellationToken cancellationToken = default); + Task> GetNewerRevisionAsync( string id, long minRevision, CancellationToken cancellationToken = default ); - Task> GetActiveNewerRevisionAsync( + Task> GetActiveNewerRevisionAsync( string parentId, long minRevision, CancellationToken cancellationToken = default diff --git a/src/Serval/src/Serval.Translation/Services/IEngineService.cs b/src/Serval/src/Serval.Translation/Services/IEngineService.cs deleted file mode 100644 index ce8b1765..00000000 --- a/src/Serval/src/Serval.Translation/Services/IEngineService.cs +++ /dev/null @@ -1,64 +0,0 @@ -namespace Serval.Translation.Services; - -public interface IEngineService -{ - Task> GetAllAsync(string owner, CancellationToken cancellationToken = default); - Task GetAsync(string engineId, CancellationToken cancellationToken = default); - - Task CreateAsync(Engine engine, CancellationToken cancellationToken = default); - Task DeleteAsync(string engineId, CancellationToken cancellationToken = default); - - Task TranslateAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ); - - Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ); - - Task GetWordGraphAsync(string engineId, string segment, CancellationToken cancellationToken = default); - - Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ); - - Task StartBuildAsync(Build build, CancellationToken cancellationToken = default); - - Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default); - - Task GetModelDownloadUrlAsync(string engineId, CancellationToken cancellationToken = default); - - Task AddCorpusAsync(string engineId, Corpus corpus, CancellationToken cancellationToken = default); - Task UpdateCorpusAsync( - string engineId, - string corpusId, - IReadOnlyList? sourceFiles, - IReadOnlyList? targetFiles, - CancellationToken cancellationToken = default - ); - Task DeleteCorpusAsync( - string engineId, - string corpusId, - bool deleteFiles, - CancellationToken cancellationToken = default - ); - - Task DeleteAllCorpusFilesAsync(string dataFileId, CancellationToken cancellationToken = default); - - Task GetQueueAsync(string engineType, CancellationToken cancellationToken = default); - - Task GetLanguageInfoAsync( - string engineType, - string language, - CancellationToken cancellationToken = default - ); -} diff --git a/src/Serval/src/Serval.Translation/Services/ITranslationEngineService.cs b/src/Serval/src/Serval.Translation/Services/ITranslationEngineService.cs new file mode 100644 index 00000000..6fdd5455 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Services/ITranslationEngineService.cs @@ -0,0 +1,33 @@ +namespace Serval.Translation.Services; + +public interface ITranslationEngineService : ITrainingEngineService +{ + Task TranslateAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ); + + Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ); + + Task GetWordGraphAsync(string engineId, string segment, CancellationToken cancellationToken = default); + + Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ); + + Task GetLanguageInfoAsync( + string engineType, + string language, + CancellationToken cancellationToken = default + ); +} diff --git a/src/Serval/src/Serval.Translation/Services/EngineService.cs b/src/Serval/src/Serval.Translation/Services/TranslationEngineService.cs similarity index 89% rename from src/Serval/src/Serval.Translation/Services/EngineService.cs rename to src/Serval/src/Serval.Translation/Services/TranslationEngineService.cs index bfee7000..c4a785ae 100644 --- a/src/Serval/src/Serval.Translation/Services/EngineService.cs +++ b/src/Serval/src/Serval.Translation/Services/TranslationEngineService.cs @@ -3,9 +3,9 @@ namespace Serval.Translation.Services; -public class EngineService( - IRepository engines, - IRepository builds, +public class TranslationEngineService( + IRepository engines, + IRepository builds, IRepository pretranslations, IScopedMediator mediator, GrpcClientFactory grpcClientFactory, @@ -13,15 +13,15 @@ public class EngineService( IDataAccessContext dataAccessContext, ILoggerFactory loggerFactory, IScriptureDataFileService scriptureDataFileService -) : OwnedEntityServiceBase(engines), IEngineService +) : OwnedEntityServiceBase(engines), ITranslationEngineService { - private readonly IRepository _builds = builds; + private readonly IRepository _builds = builds; private readonly IRepository _pretranslations = pretranslations; private readonly IScopedMediator _mediator = mediator; private readonly GrpcClientFactory _grpcClientFactory = grpcClientFactory; private readonly IOptionsMonitor _dataFileOptions = dataFileOptions; private readonly IDataAccessContext _dataAccessContext = dataAccessContext; - private readonly ILogger _logger = loggerFactory.CreateLogger(); + private readonly ILogger _logger = loggerFactory.CreateLogger(); private readonly IScriptureDataFileService _scriptureDataFileService = scriptureDataFileService; public async Task TranslateAsync( @@ -30,7 +30,7 @@ IScriptureDataFileService scriptureDataFileService CancellationToken cancellationToken = default ) { - Engine engine = await GetAsync(engineId, cancellationToken); + TrainingEngine engine = await GetAsync(engineId, cancellationToken); TranslationEngineApi.TranslationEngineApiClient client = _grpcClientFactory.CreateClient(engine.Type); @@ -54,7 +54,7 @@ IScriptureDataFileService scriptureDataFileService CancellationToken cancellationToken = default ) { - Engine engine = await GetAsync(engineId, cancellationToken); + TrainingEngine engine = await GetAsync(engineId, cancellationToken); TranslationEngineApi.TranslationEngineApiClient client = _grpcClientFactory.CreateClient(engine.Type); @@ -77,7 +77,7 @@ IScriptureDataFileService scriptureDataFileService CancellationToken cancellationToken = default ) { - Engine engine = await GetAsync(engineId, cancellationToken); + TrainingEngine engine = await GetAsync(engineId, cancellationToken); TranslationEngineApi.TranslationEngineApiClient client = _grpcClientFactory.CreateClient(engine.Type); @@ -101,7 +101,7 @@ public async Task TrainSegmentPairAsync( CancellationToken cancellationToken = default ) { - Engine engine = await GetAsync(engineId, cancellationToken); + TrainingEngine engine = await GetAsync(engineId, cancellationToken); TranslationEngineApi.TranslationEngineApiClient client = _grpcClientFactory.CreateClient(engine.Type); @@ -118,7 +118,10 @@ await client.TrainSegmentPairAsync( ); } - public override async Task CreateAsync(Engine engine, CancellationToken cancellationToken = default) + public override async Task CreateAsync( + TrainingEngine engine, + CancellationToken cancellationToken = default + ) { bool updateIsModelPersisted = engine.IsModelPersisted is null; try @@ -177,7 +180,7 @@ await Entities.UpdateAsync( public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) { - Engine? engine = await Entities.GetAsync(engineId, cancellationToken); + TrainingEngine? engine = await Entities.GetAsync(engineId, cancellationToken); if (engine is null) throw new EntityNotFoundException($"Could not find the Engine '{engineId}'."); @@ -199,9 +202,9 @@ await _dataAccessContext.WithTransactionAsync( ); } - public async Task StartBuildAsync(Build build, CancellationToken cancellationToken = default) + public async Task StartJobAsync(TranslationBuildJob build, CancellationToken cancellationToken = default) { - Engine engine = await GetAsync(build.EngineRef, cancellationToken); + TrainingEngine engine = await GetAsync(build.EngineRef, cancellationToken); await _builds.InsertAsync(build, cancellationToken); try @@ -275,18 +278,18 @@ Dictionary> GetChapters(V1.Corpus corpus, string scriptureRang ); } } - if (trainOn?.TryGetValue(c.Id, out TrainingCorpus? trainingCorpus) ?? false) + if (trainOn?.TryGetValue(c.Id, out FilteredCorpus? filteredCorpus) ?? false) { - corpus.TrainOnAll = trainingCorpus.TextIds is null && trainingCorpus.ScriptureRange is null; - if (trainingCorpus.TextIds is not null && trainingCorpus.ScriptureRange is not null) + corpus.TrainOnAll = filteredCorpus.TextIds is null && filteredCorpus.ScriptureRange is null; + if (filteredCorpus.TextIds is not null && filteredCorpus.ScriptureRange is not null) { throw new InvalidOperationException( $"The corpus {c.Id} cannot specify both 'textIds' and 'scriptureRange' for trainOn" ); } - if (trainingCorpus.TextIds is not null) - corpus.TrainOnTextIds.Add(trainingCorpus.TextIds); - if (!string.IsNullOrEmpty(trainingCorpus.ScriptureRange)) + if (filteredCorpus.TextIds is not null) + corpus.TrainOnTextIds.Add(filteredCorpus.TextIds); + if (!string.IsNullOrEmpty(filteredCorpus.ScriptureRange)) { if ( c.TargetFiles.Count > 1 @@ -298,7 +301,7 @@ Dictionary> GetChapters(V1.Corpus corpus, string scriptureRang ); } corpus.TrainOnChapters.Add( - GetChapters(corpus, trainingCorpus.ScriptureRange) + GetChapters(corpus, filteredCorpus.ScriptureRange) .Select( (kvp) => { @@ -359,9 +362,9 @@ Dictionary> GetChapters(V1.Corpus corpus, string scriptureRang } } - public async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + public async Task CancelJobAsync(string engineId, CancellationToken cancellationToken = default) { - Engine? engine = await GetAsync(engineId, cancellationToken); + TrainingEngine? engine = await GetAsync(engineId, cancellationToken); if (engine is null) throw new EntityNotFoundException($"Could not find the Engine '{engineId}'."); @@ -388,7 +391,7 @@ public async Task GetModelDownloadUrlAsync( CancellationToken cancellationToken = default ) { - Engine? engine = await GetAsync(engineId, cancellationToken); + TrainingEngine? engine = await GetAsync(engineId, cancellationToken); if (engine is null) throw new EntityNotFoundException($"Could not find the Engine '{engineId}'."); @@ -406,20 +409,20 @@ public async Task GetModelDownloadUrlAsync( }; } - public Task AddCorpusAsync(string engineId, Models.Corpus corpus, CancellationToken cancellationToken = default) + public Task AddCorpusAsync(string engineId, TrainingCorpus corpus, CancellationToken cancellationToken = default) { return Entities.UpdateAsync(engineId, u => u.Add(e => e.Corpora, corpus), cancellationToken: cancellationToken); } - public async Task UpdateCorpusAsync( + public async Task UpdateCorpusAsync( string engineId, string corpusId, - IReadOnlyList? sourceFiles, - IReadOnlyList? targetFiles, + IReadOnlyList? sourceFiles, + IReadOnlyList? targetFiles, CancellationToken cancellationToken = default ) { - Engine? engine = await Entities.UpdateAsync( + TrainingEngine? engine = await Entities.UpdateAsync( e => e.Id == engineId && e.Corpora.Any(c => c.Id == corpusId), u => { @@ -442,7 +445,7 @@ public async Task DeleteCorpusAsync( CancellationToken cancellationToken = default ) { - Engine? originalEngine = null; + TrainingEngine? originalEngine = null; await _dataAccessContext.WithTransactionAsync( async (ct) => { @@ -539,9 +542,9 @@ private Models.TranslationResult Map(V1.TranslationResult source) return source.Values.Cast().ToHashSet(); } - private Models.AlignedWordPair Map(V1.AlignedWordPair source) + private Shared.Models.AlignedWordPair Map(V1.AlignedWordPair source) { - return new Models.AlignedWordPair { SourceIndex = source.SourceIndex, TargetIndex = source.TargetIndex }; + return new Shared.Models.AlignedWordPair { SourceIndex = source.SourceIndex, TargetIndex = source.TargetIndex }; } private Models.Phrase Map(V1.Phrase source) @@ -581,7 +584,7 @@ private Models.WordGraphArc Map(V1.WordGraphArc source) }; } - private V1.Corpus Map(Models.Corpus source) + private V1.Corpus Map(TrainingCorpus source) { return new V1.Corpus { @@ -593,7 +596,7 @@ private V1.Corpus Map(Models.Corpus source) }; } - private V1.CorpusFile Map(Models.CorpusFile source) + private V1.CorpusFile Map(Shared.Models.CorpusFile source) { return new V1.CorpusFile { diff --git a/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs b/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs index 615e8c89..d450d341 100644 --- a/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs +++ b/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs @@ -4,7 +4,7 @@ namespace Serval.Translation.Services; public class TranslationPlatformServiceV1( - IRepository builds, + IRepository builds, IRepository engines, IRepository pretranslations, IDataAccessContext dataAccessContext, @@ -14,7 +14,7 @@ IPublishEndpoint publishEndpoint private const int PretranslationInsertBatchSize = 128; private static readonly Empty Empty = new(); - private readonly IRepository _builds = builds; + private readonly IRepository _builds = builds; private readonly IRepository _engines = engines; private readonly IRepository _pretranslations = pretranslations; private readonly IDataAccessContext _dataAccessContext = dataAccessContext; @@ -25,7 +25,7 @@ public override async Task BuildStarted(BuildStartedRequest request, Serv await _dataAccessContext.WithTransactionAsync( async (ct) => { - Build? build = await _builds.UpdateAsync( + TrainingBuildJob? build = await _builds.UpdateAsync( request.BuildId, u => u.Set(b => b.State, JobState.Active), cancellationToken: ct @@ -61,7 +61,7 @@ public override async Task BuildCompleted(BuildCompletedRequest request, await _dataAccessContext.WithTransactionAsync( async (ct) => { - Build? build = await _builds.UpdateAsync( + TrainingBuildJob? build = await _builds.UpdateAsync( request.BuildId, u => u.Set(b => b.State, JobState.Completed) @@ -114,7 +114,7 @@ public override async Task BuildCanceled(BuildCanceledRequest request, Se await _dataAccessContext.WithTransactionAsync( async (ct) => { - Build? build = await _builds.UpdateAsync( + TrainingBuildJob? build = await _builds.UpdateAsync( request.BuildId, u => u.Set(b => b.Message, "Canceled") @@ -163,7 +163,7 @@ public override async Task BuildFaulted(BuildFaultedRequest request, Serv await _dataAccessContext.WithTransactionAsync( async (ct) => { - Build? build = await _builds.UpdateAsync( + TrainingBuildJob? build = await _builds.UpdateAsync( request.BuildId, u => u.Set(b => b.State, JobState.Faulted) @@ -212,7 +212,7 @@ public override async Task BuildRestarting(BuildRestartingRequest request await _dataAccessContext.WithTransactionAsync( async (ct) => { - Build? build = await _builds.UpdateAsync( + TrainingBuildJob? build = await _builds.UpdateAsync( request.BuildId, u => u.Set(b => b.Message, "Restarting") diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs b/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs index 2be669b2..05fdbaa7 100644 --- a/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs +++ b/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs @@ -325,7 +325,11 @@ string engineId { case 200: await _env.Builds.InsertAsync( - new Build { EngineRef = engineId, State = Shared.Contracts.JobState.Completed } + new Translation.Models.TranslationBuildJob + { + EngineRef = engineId, + State = Shared.Contracts.JobState.Completed + } ); Client.TranslationResult result = await client.TranslateAsync(engineId, "This is a test ."); Assert.That(result.Translation, Is.EqualTo("This is a test .")); @@ -378,7 +382,11 @@ string engineId { case 200: await _env.Builds.InsertAsync( - new Build { EngineRef = engineId, State = Shared.Contracts.JobState.Completed } + new Translation.Models.TranslationBuildJob + { + EngineRef = engineId, + State = Shared.Contracts.JobState.Completed + } ); ICollection results = await client.TranslateNAsync( engineId, @@ -437,7 +445,11 @@ string engineId case 200: TranslationCorpus addedCorpus = await client.AddCorpusAsync(engineId, TestCorpusConfig); await _env.Builds.InsertAsync( - new Build { EngineRef = engineId, State = Shared.Contracts.JobState.Completed } + new Translation.Models.TranslationBuildJob + { + EngineRef = engineId, + State = Shared.Contracts.JobState.Completed + } ); Client.WordGraph wg = await client.GetWordGraphAsync(engineId, "This is a test ."); Assert.Multiple(() => @@ -501,7 +513,11 @@ string engineId case 200: TranslationCorpus addedCorpus = await client.AddCorpusAsync(engineId, TestCorpusConfig); await _env.Builds.InsertAsync( - new Build { EngineRef = engineId, State = Shared.Contracts.JobState.Completed } + new Translation.Models.TranslationBuildJob + { + EngineRef = engineId, + State = Shared.Contracts.JobState.Completed + } ); await client.TrainSegmentAsync(engineId, sp); break; @@ -922,16 +938,16 @@ public async Task GetAllBuildsForEngineByIdAsync( ) { TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); - Build? build = null; + Translation.Models.TranslationBuildJob? build = null; if (addBuild) { - build = new Build { EngineRef = engineId }; + build = new Translation.Models.TranslationBuildJob { EngineRef = engineId }; await _env.Builds.InsertAsync(build); } switch (expectedStatusCode) { case 200: - ICollection results = await client.GetAllBuildsAsync(engineId); + ICollection results = await client.GetAllBuildsAsync(engineId); Assert.That(results, Is.Not.Empty); Assert.Multiple(() => { @@ -967,10 +983,10 @@ public async Task GetBuildByIdForEngineByIdAsync( ) { TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); - Build? build = null; + Translation.Models.TranslationBuildJob? build = null; if (addBuild) { - build = new Build { EngineRef = engineId }; + build = new Translation.Models.TranslationBuildJob { EngineRef = engineId }; await _env.Builds.InsertAsync(build); } @@ -979,7 +995,7 @@ public async Task GetBuildByIdForEngineByIdAsync( case 200: { Assert.That(build, Is.Not.Null); - TranslationBuild result = await client.GetBuildAsync(engineId, build.Id); + Client.TranslationBuild result = await client.GetBuildAsync(engineId, build.Id); Assert.Multiple(() => { Assert.That(result.Revision, Is.EqualTo(1)); @@ -1055,13 +1071,13 @@ public async Task StartBuildForEngineByIdAsync(IEnumerable scope, int ex "some_string":"string"} """ }; - TranslationBuild resultAfterStart; + Client.TranslationBuild resultAfterStart; Assert.ThrowsAsync(async () => { resultAfterStart = await client.GetCurrentBuildAsync(engineId); }); - TranslationBuild build = await client.StartBuildAsync(engineId, tbc); + Client.TranslationBuild build = await client.StartBuildAsync(engineId, tbc); Assert.That(build, Is.Not.Null); build = await client.GetCurrentBuildAsync(engineId); @@ -1098,7 +1114,7 @@ public async Task StartBuildForEngineAsync_UnparsableOptions() TrainOn = [tcc], Options = "unparsable json" }; - TranslationBuild resultAfterStart; + Client.TranslationBuild resultAfterStart; Assert.ThrowsAsync(async () => { resultAfterStart = await client.GetCurrentBuildAsync(ECHO_ENGINE1_ID); @@ -1158,10 +1174,10 @@ public async Task GetCurrentBuildForEngineByIdAsync( ) { TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); - Build? build = null; + Translation.Models.TranslationBuildJob? build = null; if (addBuild) { - build = new Build { EngineRef = engineId }; + build = new Translation.Models.TranslationBuildJob { EngineRef = engineId }; await _env.Builds.InsertAsync(build); } @@ -1170,7 +1186,7 @@ public async Task GetCurrentBuildForEngineByIdAsync( case 200: { Assert.That(build, Is.Not.Null); - TranslationBuild result = await client.GetCurrentBuildAsync(engineId); + Client.TranslationBuild result = await client.GetCurrentBuildAsync(engineId); Assert.That(result.Id, Is.EqualTo(build.Id)); break; } @@ -1215,7 +1231,7 @@ public async Task CancelCurrentBuildForEngineByIdAsync( TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); if (!addBuild) { - var build = new Build { EngineRef = engineId }; + var build = new Translation.Models.TranslationBuildJob { EngineRef = engineId }; await _env.Builds.InsertAsync(build); _env.NmtClient.CancelBuildAsync(Arg.Any(), null, null, Arg.Any()) .Returns(CreateAsyncUnaryCall(StatusCode.Aborted)); @@ -1250,7 +1266,7 @@ public async Task TryToQueueMultipleBuildsPerSingleUser() TranslationCorpus addedCorpus = await client.AddCorpusAsync(engineId, TestCorpusConfigNonEcho); var ptcc = new PretranslateCorpusConfig { CorpusId = addedCorpus.Id, TextIds = ["all"] }; var tbc = new TranslationBuildConfig { Pretranslate = [ptcc] }; - TranslationBuild build = await client.StartBuildAsync(engineId, tbc); + Client.TranslationBuild build = await client.StartBuildAsync(engineId, tbc); _env.NmtClient.StartBuildAsync(Arg.Any(), null, null, Arg.Any()) .Returns(CreateAsyncUnaryCall(StatusCode.Aborted)); ServalApiException? ex = Assert.ThrowsAsync(async () => @@ -1378,7 +1394,7 @@ public TestEnvironment() Pretranslations = _scope.ServiceProvider.GetRequiredService< IRepository >(); - Builds = _scope.ServiceProvider.GetRequiredService>(); + Builds = _scope.ServiceProvider.GetRequiredService>(); EchoClient = Substitute.For(); EchoClient @@ -1550,7 +1566,7 @@ public TestEnvironment() public IRepository Engines { get; } public IRepository DataFiles { get; } public IRepository Pretranslations { get; } - public IRepository Builds { get; } + public IRepository Builds { get; } public TranslationEngineApi.TranslationEngineApiClient EchoClient { get; } public TranslationEngineApi.TranslationEngineApiClient NmtClient { get; } diff --git a/src/Serval/test/Serval.Translation.Tests/Services/BuildServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/BuildServiceTests.cs index d2e1e4e6..c45281e1 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/BuildServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/BuildServiceTests.cs @@ -8,17 +8,17 @@ public class BuildServiceTests [Test] public async Task GetActiveNewerRevisionAsync_Insert() { - var builds = new MemoryRepository(); + var builds = new MemoryRepository(); var service = new BuildService(builds); - Task> task = service.GetActiveNewerRevisionAsync("engine1", 1); - var build = new Build + Task> task = service.GetActiveNewerRevisionAsync("engine1", 1); + var build = new TranslationBuildJob { Id = BUILD1_ID, EngineRef = "engine1", PercentCompleted = 0.1 }; await builds.InsertAsync(build); - EntityChange change = await task; + EntityChange change = await task; Assert.That(change.Type, Is.EqualTo(EntityChangeType.Insert)); Assert.That(change.Entity!.Revision, Is.EqualTo(1)); Assert.That(change.Entity.PercentCompleted, Is.EqualTo(0.1)); @@ -27,13 +27,13 @@ public async Task GetActiveNewerRevisionAsync_Insert() [Test] public async Task GetNewerRevisionAsync_Update() { - var builds = new MemoryRepository(); + var builds = new MemoryRepository(); var service = new BuildService(builds); - var build = new Build { Id = BUILD1_ID, EngineRef = "engine1" }; + var build = new TranslationBuildJob { Id = BUILD1_ID, EngineRef = "engine1" }; await builds.InsertAsync(build); - Task> task = service.GetNewerRevisionAsync(build.Id, 2); + Task> task = service.GetNewerRevisionAsync(build.Id, 2); await builds.UpdateAsync(build, u => u.Set(b => b.PercentCompleted, 0.1)); - EntityChange change = await task; + EntityChange change = await task; Assert.That(change.Type, Is.EqualTo(EntityChangeType.Update)); Assert.That(change.Entity!.Revision, Is.EqualTo(2)); Assert.That(change.Entity.PercentCompleted, Is.EqualTo(0.1)); @@ -42,21 +42,21 @@ public async Task GetNewerRevisionAsync_Update() [Test] public async Task GetNewerRevisionAsync_Delete() { - var builds = new MemoryRepository(); + var builds = new MemoryRepository(); var service = new BuildService(builds); - var build = new Build { Id = BUILD1_ID, EngineRef = "engine1" }; + var build = new TranslationBuildJob { Id = BUILD1_ID, EngineRef = "engine1" }; await builds.InsertAsync(build); - Task> task = service.GetNewerRevisionAsync(build.Id, 2); + Task> task = service.GetNewerRevisionAsync(build.Id, 2); await builds.DeleteAsync(build); - EntityChange change = await task; + EntityChange change = await task; Assert.That(change.Type, Is.EqualTo(EntityChangeType.Delete)); } [Test] public async Task GetNewerRevisionAsync_DoesNotExist() { - var service = new BuildService(new MemoryRepository()); - EntityChange change = await service.GetNewerRevisionAsync("build1", 2); + var service = new BuildService(new MemoryRepository()); + EntityChange change = await service.GetNewerRevisionAsync("build1", 2); Assert.That(change.Type, Is.EqualTo(EntityChangeType.Delete)); } } diff --git a/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs index 49e114cb..b76f28c2 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs @@ -108,7 +108,7 @@ public async Task StartBuildAsync_TrainOnNotSpecified() { var env = new TestEnvironment(); string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; - await env.Service.StartBuildAsync(new Build { Id = BUILD1_ID, EngineRef = engineId }); + await env.Service.StartJobAsync(new TranslationBuildJob { Id = BUILD1_ID, EngineRef = engineId }); _ = env.TranslationServiceClient.Received() .StartBuildAsync( new StartBuildRequest @@ -153,8 +153,8 @@ public async Task StartBuildAsync_TextIdsEmpty() { var env = new TestEnvironment(); string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; - await env.Service.StartBuildAsync( - new Build + await env.Service.StartJobAsync( + new TranslationBuildJob { Id = BUILD1_ID, EngineRef = engineId, @@ -206,8 +206,8 @@ public async Task StartBuildAsync_TextIdsPopulated() { var env = new TestEnvironment(); string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; - await env.Service.StartBuildAsync( - new Build + await env.Service.StartJobAsync( + new TranslationBuildJob { Id = BUILD1_ID, EngineRef = engineId, @@ -259,8 +259,8 @@ public async Task StartBuildAsync_TextIdsNotSpecified() { var env = new TestEnvironment(); string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; - await env.Service.StartBuildAsync( - new Build + await env.Service.StartJobAsync( + new TranslationBuildJob { Id = BUILD1_ID, EngineRef = engineId, @@ -313,8 +313,8 @@ public async Task StartBuildAsync_TextFilesScriptureRangeSpecified() string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; Assert.ThrowsAsync( () => - env.Service.StartBuildAsync( - new Build + env.Service.StartJobAsync( + new TranslationBuildJob { Id = BUILD1_ID, EngineRef = engineId, @@ -329,8 +329,8 @@ public async Task StartBuildAsync_ScriptureRangeSpecified() { var env = new TestEnvironment(); string engineId = (await env.CreateEngineWithParatextProjectAsync()).Id; - await env.Service.StartBuildAsync( - new Build + await env.Service.StartJobAsync( + new TranslationBuildJob { Id = BUILD1_ID, EngineRef = engineId, @@ -392,8 +392,8 @@ public async Task StartBuildAsync_ScriptureRangeEmptyString() { var env = new TestEnvironment(); string engineId = (await env.CreateEngineWithParatextProjectAsync()).Id; - await env.Service.StartBuildAsync( - new Build + await env.Service.StartJobAsync( + new TranslationBuildJob { Id = BUILD1_ID, EngineRef = engineId, @@ -444,7 +444,7 @@ public async Task CancelBuildAsync_EngineExistsNotBuilding() { var env = new TestEnvironment(); string engineId = (await env.CreateEngineWithTextFilesAsync()).Id; - await env.Service.CancelBuildAsync(engineId); + await env.Service.CancelJobAsync(engineId); } [Test] @@ -454,7 +454,7 @@ public async Task UpdateCorpusAsync() Engine engine = await env.CreateEngineWithTextFilesAsync(); string corpusId = engine.Corpora[0].Id; - Models.Corpus? corpus = await env.Service.UpdateCorpusAsync( + Shared.Models.Corpus? corpus = await env.Service.UpdateCorpusAsync( engine.Id, corpusId, sourceFiles: @@ -626,7 +626,7 @@ public TestEnvironment() Service = new EngineService( Engines, - new MemoryRepository(), + new MemoryRepository(), new MemoryRepository(), Substitute.For(), grpcClientFactory, @@ -637,7 +637,7 @@ public TestEnvironment() ); } - public EngineService Service { get; } + public TranslationEngineService Service { get; } public IRepository Engines { get; } public TranslationEngineApi.TranslationEngineApiClient TranslationServiceClient { get; } @@ -650,7 +650,7 @@ public async Task CreateEngineWithTextFilesAsync() SourceLanguage = "es", TargetLanguage = "en", Type = "Smt", - Corpora = new Models.Corpus[] + Corpora = new Shared.Models.Corpus[] { new() { @@ -693,7 +693,7 @@ public async Task CreateEngineWithParatextProjectAsync() SourceLanguage = "es", TargetLanguage = "en", Type = "Smt", - Corpora = new Models.Corpus[] + Corpora = new TrainingCorpus[] { new() { diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs index 10f3ca14..bbf6952b 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs @@ -20,7 +20,7 @@ await env.Engines.InsertAsync( Corpora = [] } ); - await env.Builds.InsertAsync(new Build() { Id = "b0", EngineRef = "e0" }); + await env.Builds.InsertAsync(new TranslationBuildJob() { Id = "b0", EngineRef = "e0" }); await env.PlatformService.BuildStarted(new BuildStartedRequest() { BuildId = "b0" }, env.ServerCallContext); Assert.That(env.Builds.Get("b0").State, Is.EqualTo(Shared.Contracts.JobState.Active)); Assert.That(env.Engines.Get("e0").IsBuilding, Is.True); @@ -74,7 +74,7 @@ await env.Engines.InsertAsync( Corpora = [] } ); - await env.Builds.InsertAsync(new Build() { Id = "b0", EngineRef = "e0" }); + await env.Builds.InsertAsync(new TranslationBuildJob() { Id = "b0", EngineRef = "e0" }); Assert.That(env.Builds.Get("b0").QueueDepth, Is.Null); Assert.That(env.Builds.Get("b0").PercentCompleted, Is.Null); await env.PlatformService.UpdateBuildStatus( @@ -117,7 +117,7 @@ private class TestEnvironment { public TestEnvironment() { - Builds = new MemoryRepository(); + Builds = new MemoryRepository(); Engines = new MemoryRepository(); Pretranslations = new MemoryRepository(); DataAccessContext = Substitute.For(); @@ -146,7 +146,7 @@ public TestEnvironment() ); } - public MemoryRepository Builds { get; } + public MemoryRepository Builds { get; } public MemoryRepository Engines { get; } public MemoryRepository Pretranslations { get; } public IDataAccessContext DataAccessContext { get; }