From 0601c9661e20e23a9265af09f5459e979aa8a172 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Thu, 15 Aug 2024 15:40:27 -0400 Subject: [PATCH 01/15] Add ability to use localizations for key terms --- .../Services/CorpusService.cs | 4 +-- .../Services/ICorpusService.cs | 2 +- .../Services/PreprocessBuildJob.cs | 31 ++++++++++++------- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs index 17d562ad..ff71e7c6 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs @@ -36,14 +36,14 @@ public IEnumerable CreateTextCorpora(IReadOnlyList file return corpora; } - public IEnumerable CreateTermCorpora(IReadOnlyList files) + public IEnumerable CreateTermCorpora(IReadOnlyList files, string? languageCode = null) { foreach (CorpusFile file in files) { switch (file.Format) { case FileFormat.Paratext: - yield return new ParatextBackupTermsCorpus(file.Location, ["PN"]); + yield return new ParatextBackupTermsCorpus(file.Location, ["PN"], languageCode: languageCode); break; } } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs index bbcc9de3..6e62e7d9 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs @@ -3,5 +3,5 @@ public interface ICorpusService { IEnumerable CreateTextCorpora(IReadOnlyList files); - IEnumerable CreateTermCorpora(IReadOnlyList files); + IEnumerable CreateTermCorpora(IReadOnlyList files, string? languageCode = null); } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs index 850d1b68..b11a68e4 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs @@ -50,13 +50,20 @@ protected override async Task DoWorkAsync( CancellationToken cancellationToken ) { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new OperationCanceledException($"Engine {engineId} does not exist. Build canceled."); + + bool sourceTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.SourceLanguage, out string srcLang); + bool targetTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.TargetLanguage, out string trgLang); + (int trainCount, int pretranslateCount) = await WriteDataFilesAsync( buildId, data, buildOptions, + (engine.SourceLanguage, engine.TargetLanguage), cancellationToken ); - // Log summary of build data JsonObject buildPreprocessSummary = new() @@ -65,16 +72,10 @@ CancellationToken cancellationToken { "EngineId", engineId }, { "BuildId", buildId }, { "NumTrainRows", trainCount }, - { "NumPretranslateRows", pretranslateCount } + { "NumPretranslateRows", pretranslateCount }, + { "SourceLanguageResolved", srcLang }, + { "TargetLanguageResolved", trgLang } }; - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new OperationCanceledException($"Engine {engineId} does not exist. Build canceled."); - - bool sourceTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.SourceLanguage, out string srcLang); - buildPreprocessSummary.Add("SourceLanguageResolved", srcLang); - bool targetTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.TargetLanguage, out string trgLang); - buildPreprocessSummary.Add("TargetLanguageResolved", trgLang); Logger.LogInformation("{summary}", buildPreprocessSummary.ToJsonString()); if (trainCount == 0 && (!sourceTagInBaseModel || !targetTagInBaseModel)) @@ -105,6 +106,7 @@ CancellationToken cancellationToken string buildId, IReadOnlyList corpora, string? buildOptions, + (string, string) languageCodes, CancellationToken cancellationToken ) { @@ -164,8 +166,13 @@ CancellationToken cancellationToken if ((bool?)buildOptionsObject?["use_key_terms"] ?? true) { - ITextCorpus? sourceTermCorpus = _corpusService.CreateTermCorpora(corpus.SourceFiles).FirstOrDefault(); - ITextCorpus? targetTermCorpus = _corpusService.CreateTermCorpora(corpus.TargetFiles).FirstOrDefault(); + (string sourceLanguageCode, string targetLanguageCode) = languageCodes; + ITextCorpus? sourceTermCorpus = _corpusService + .CreateTermCorpora(corpus.SourceFiles, sourceLanguageCode) + .FirstOrDefault(); + ITextCorpus? targetTermCorpus = _corpusService + .CreateTermCorpora(corpus.TargetFiles, targetLanguageCode) + .FirstOrDefault(); if (sourceTermCorpus is not null && targetTermCorpus is not null) { IParallelTextCorpus parallelKeyTermsCorpus = sourceTermCorpus.AlignRows(targetTermCorpus); From 0ab2ca769dbb75ebac1ab5e12ca97467a6b345be Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Fri, 16 Aug 2024 09:55:01 -0400 Subject: [PATCH 02/15] Use language code from settings file --- .../Serval.Machine.Shared/Services/CorpusService.cs | 4 ++-- .../Serval.Machine.Shared/Services/ICorpusService.cs | 2 +- .../Services/PreprocessBuildJob.cs | 11 ++--------- .../Services/EngineServiceTests.cs | 3 ++- .../Services/PretranslationServiceTests.cs | 3 ++- 5 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs index ff71e7c6..17d562ad 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs @@ -36,14 +36,14 @@ public IEnumerable CreateTextCorpora(IReadOnlyList file return corpora; } - public IEnumerable CreateTermCorpora(IReadOnlyList files, string? languageCode = null) + public IEnumerable CreateTermCorpora(IReadOnlyList files) { foreach (CorpusFile file in files) { switch (file.Format) { case FileFormat.Paratext: - yield return new ParatextBackupTermsCorpus(file.Location, ["PN"], languageCode: languageCode); + yield return new ParatextBackupTermsCorpus(file.Location, ["PN"]); break; } } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs index 6e62e7d9..bbcc9de3 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs @@ -3,5 +3,5 @@ public interface ICorpusService { IEnumerable CreateTextCorpora(IReadOnlyList files); - IEnumerable CreateTermCorpora(IReadOnlyList files, string? languageCode = null); + IEnumerable CreateTermCorpora(IReadOnlyList files); } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs index b11a68e4..4e5f94b1 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs @@ -61,7 +61,6 @@ CancellationToken cancellationToken buildId, data, buildOptions, - (engine.SourceLanguage, engine.TargetLanguage), cancellationToken ); // Log summary of build data @@ -106,7 +105,6 @@ CancellationToken cancellationToken string buildId, IReadOnlyList corpora, string? buildOptions, - (string, string) languageCodes, CancellationToken cancellationToken ) { @@ -166,13 +164,8 @@ CancellationToken cancellationToken if ((bool?)buildOptionsObject?["use_key_terms"] ?? true) { - (string sourceLanguageCode, string targetLanguageCode) = languageCodes; - ITextCorpus? sourceTermCorpus = _corpusService - .CreateTermCorpora(corpus.SourceFiles, sourceLanguageCode) - .FirstOrDefault(); - ITextCorpus? targetTermCorpus = _corpusService - .CreateTermCorpora(corpus.TargetFiles, targetLanguageCode) - .FirstOrDefault(); + ITextCorpus? sourceTermCorpus = _corpusService.CreateTermCorpora(corpus.SourceFiles).FirstOrDefault(); + ITextCorpus? targetTermCorpus = _corpusService.CreateTermCorpora(corpus.TargetFiles).FirstOrDefault(); if (sourceTermCorpus is not null && targetTermCorpus is not null) { IParallelTextCorpus parallelKeyTermsCorpus = sourceTermCorpus.AlignRows(targetTermCorpus); diff --git a/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs index e6df88c6..49e114cb 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs @@ -619,7 +619,8 @@ public TestEnvironment() fileNameSuffix: ".USFM", biblicalTermsListType: "BiblicalTerms", biblicalTermsProjectName: "", - biblicalTermsFileName: "BiblicalTerms.xml" + biblicalTermsFileName: "BiblicalTerms.xml", + languageCode: "en" ) ); diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs index b2f08824..cbdcb6ff 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs @@ -402,7 +402,8 @@ private static ParatextProjectSettings CreateProjectSettings(string name) fileNameSuffix: $"{name}.SFM", biblicalTermsListType: "Major", biblicalTermsProjectName: "", - biblicalTermsFileName: "BiblicalTerms.xml" + biblicalTermsFileName: "BiblicalTerms.xml", + languageCode: "en" ); } } From c5841b61f0f3b582fa15e7be6eed0adf52e853d8 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Fri, 16 Aug 2024 16:22:59 -0400 Subject: [PATCH 03/15] Accommodate change in class name/structure --- .../src/Serval.Machine.Shared/Services/CorpusService.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs index 17d562ad..1555bf71 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs @@ -43,7 +43,10 @@ public IEnumerable CreateTermCorpora(IReadOnlyList file switch (file.Format) { case FileFormat.Paratext: - yield return new ParatextBackupTermsCorpus(file.Location, ["PN"]); + using (var archive = ZipFile.OpenRead(file.Location)) + { + yield return new ZipParatextProjectTermsCorpus(archive, ["PN"]); + } break; } } From 7de44c44cae55a8af5d371d90ef6c137440d64f1 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Tue, 20 Aug 2024 14:36:56 -0400 Subject: [PATCH 04/15] Revert name of machine class --- src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs index 1555bf71..940227d7 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs @@ -45,7 +45,7 @@ public IEnumerable CreateTermCorpora(IReadOnlyList file case FileFormat.Paratext: using (var archive = ZipFile.OpenRead(file.Location)) { - yield return new ZipParatextProjectTermsCorpus(archive, ["PN"]); + yield return new ParatextBackupTermsCorpus(archive, ["PN"]); } break; } From 20b005969534eecedaaf7919893370eff8c16971 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Wed, 21 Aug 2024 15:47:22 -0400 Subject: [PATCH 05/15] Revert to using filepath in constructor --- .../src/Serval.Machine.Shared/Services/CorpusService.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs index 940227d7..17d562ad 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs @@ -43,10 +43,7 @@ public IEnumerable CreateTermCorpora(IReadOnlyList file switch (file.Format) { case FileFormat.Paratext: - using (var archive = ZipFile.OpenRead(file.Location)) - { - yield return new ParatextBackupTermsCorpus(archive, ["PN"]); - } + yield return new ParatextBackupTermsCorpus(file.Location, ["PN"]); break; } } From b998b8b095881038ede169ce38457b0ee771dd2d Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Thu, 15 Aug 2024 15:40:27 -0400 Subject: [PATCH 06/15] Add ability to use localizations for key terms --- .../Services/CorpusService.cs | 4 +-- .../Services/ICorpusService.cs | 2 +- .../Services/PreprocessBuildJob.cs | 31 ++++++++++++------- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs index 17d562ad..ff71e7c6 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs @@ -36,14 +36,14 @@ public IEnumerable CreateTextCorpora(IReadOnlyList file return corpora; } - public IEnumerable CreateTermCorpora(IReadOnlyList files) + public IEnumerable CreateTermCorpora(IReadOnlyList files, string? languageCode = null) { foreach (CorpusFile file in files) { switch (file.Format) { case FileFormat.Paratext: - yield return new ParatextBackupTermsCorpus(file.Location, ["PN"]); + yield return new ParatextBackupTermsCorpus(file.Location, ["PN"], languageCode: languageCode); break; } } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs index bbcc9de3..6e62e7d9 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs @@ -3,5 +3,5 @@ public interface ICorpusService { IEnumerable CreateTextCorpora(IReadOnlyList files); - IEnumerable CreateTermCorpora(IReadOnlyList files); + IEnumerable CreateTermCorpora(IReadOnlyList files, string? languageCode = null); } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs index 850d1b68..b11a68e4 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs @@ -50,13 +50,20 @@ protected override async Task DoWorkAsync( CancellationToken cancellationToken ) { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new OperationCanceledException($"Engine {engineId} does not exist. Build canceled."); + + bool sourceTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.SourceLanguage, out string srcLang); + bool targetTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.TargetLanguage, out string trgLang); + (int trainCount, int pretranslateCount) = await WriteDataFilesAsync( buildId, data, buildOptions, + (engine.SourceLanguage, engine.TargetLanguage), cancellationToken ); - // Log summary of build data JsonObject buildPreprocessSummary = new() @@ -65,16 +72,10 @@ CancellationToken cancellationToken { "EngineId", engineId }, { "BuildId", buildId }, { "NumTrainRows", trainCount }, - { "NumPretranslateRows", pretranslateCount } + { "NumPretranslateRows", pretranslateCount }, + { "SourceLanguageResolved", srcLang }, + { "TargetLanguageResolved", trgLang } }; - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new OperationCanceledException($"Engine {engineId} does not exist. Build canceled."); - - bool sourceTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.SourceLanguage, out string srcLang); - buildPreprocessSummary.Add("SourceLanguageResolved", srcLang); - bool targetTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.TargetLanguage, out string trgLang); - buildPreprocessSummary.Add("TargetLanguageResolved", trgLang); Logger.LogInformation("{summary}", buildPreprocessSummary.ToJsonString()); if (trainCount == 0 && (!sourceTagInBaseModel || !targetTagInBaseModel)) @@ -105,6 +106,7 @@ CancellationToken cancellationToken string buildId, IReadOnlyList corpora, string? buildOptions, + (string, string) languageCodes, CancellationToken cancellationToken ) { @@ -164,8 +166,13 @@ CancellationToken cancellationToken if ((bool?)buildOptionsObject?["use_key_terms"] ?? true) { - ITextCorpus? sourceTermCorpus = _corpusService.CreateTermCorpora(corpus.SourceFiles).FirstOrDefault(); - ITextCorpus? targetTermCorpus = _corpusService.CreateTermCorpora(corpus.TargetFiles).FirstOrDefault(); + (string sourceLanguageCode, string targetLanguageCode) = languageCodes; + ITextCorpus? sourceTermCorpus = _corpusService + .CreateTermCorpora(corpus.SourceFiles, sourceLanguageCode) + .FirstOrDefault(); + ITextCorpus? targetTermCorpus = _corpusService + .CreateTermCorpora(corpus.TargetFiles, targetLanguageCode) + .FirstOrDefault(); if (sourceTermCorpus is not null && targetTermCorpus is not null) { IParallelTextCorpus parallelKeyTermsCorpus = sourceTermCorpus.AlignRows(targetTermCorpus); From 4aaeedcb7299310f93d593217aca58fb5ddc1ffe Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Fri, 16 Aug 2024 09:55:01 -0400 Subject: [PATCH 07/15] Use language code from settings file --- .../Serval.Machine.Shared/Services/CorpusService.cs | 4 ++-- .../Serval.Machine.Shared/Services/ICorpusService.cs | 2 +- .../Services/PreprocessBuildJob.cs | 11 ++--------- .../Services/EngineServiceTests.cs | 3 ++- .../Services/PretranslationServiceTests.cs | 3 ++- 5 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs index ff71e7c6..17d562ad 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs @@ -36,14 +36,14 @@ public IEnumerable CreateTextCorpora(IReadOnlyList file return corpora; } - public IEnumerable CreateTermCorpora(IReadOnlyList files, string? languageCode = null) + public IEnumerable CreateTermCorpora(IReadOnlyList files) { foreach (CorpusFile file in files) { switch (file.Format) { case FileFormat.Paratext: - yield return new ParatextBackupTermsCorpus(file.Location, ["PN"], languageCode: languageCode); + yield return new ParatextBackupTermsCorpus(file.Location, ["PN"]); break; } } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs index 6e62e7d9..bbcc9de3 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs @@ -3,5 +3,5 @@ public interface ICorpusService { IEnumerable CreateTextCorpora(IReadOnlyList files); - IEnumerable CreateTermCorpora(IReadOnlyList files, string? languageCode = null); + IEnumerable CreateTermCorpora(IReadOnlyList files); } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs index b11a68e4..4e5f94b1 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs @@ -61,7 +61,6 @@ CancellationToken cancellationToken buildId, data, buildOptions, - (engine.SourceLanguage, engine.TargetLanguage), cancellationToken ); // Log summary of build data @@ -106,7 +105,6 @@ CancellationToken cancellationToken string buildId, IReadOnlyList corpora, string? buildOptions, - (string, string) languageCodes, CancellationToken cancellationToken ) { @@ -166,13 +164,8 @@ CancellationToken cancellationToken if ((bool?)buildOptionsObject?["use_key_terms"] ?? true) { - (string sourceLanguageCode, string targetLanguageCode) = languageCodes; - ITextCorpus? sourceTermCorpus = _corpusService - .CreateTermCorpora(corpus.SourceFiles, sourceLanguageCode) - .FirstOrDefault(); - ITextCorpus? targetTermCorpus = _corpusService - .CreateTermCorpora(corpus.TargetFiles, targetLanguageCode) - .FirstOrDefault(); + ITextCorpus? sourceTermCorpus = _corpusService.CreateTermCorpora(corpus.SourceFiles).FirstOrDefault(); + ITextCorpus? targetTermCorpus = _corpusService.CreateTermCorpora(corpus.TargetFiles).FirstOrDefault(); if (sourceTermCorpus is not null && targetTermCorpus is not null) { IParallelTextCorpus parallelKeyTermsCorpus = sourceTermCorpus.AlignRows(targetTermCorpus); diff --git a/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs index e6df88c6..49e114cb 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs @@ -619,7 +619,8 @@ public TestEnvironment() fileNameSuffix: ".USFM", biblicalTermsListType: "BiblicalTerms", biblicalTermsProjectName: "", - biblicalTermsFileName: "BiblicalTerms.xml" + biblicalTermsFileName: "BiblicalTerms.xml", + languageCode: "en" ) ); diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs index b2f08824..cbdcb6ff 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs @@ -402,7 +402,8 @@ private static ParatextProjectSettings CreateProjectSettings(string name) fileNameSuffix: $"{name}.SFM", biblicalTermsListType: "Major", biblicalTermsProjectName: "", - biblicalTermsFileName: "BiblicalTerms.xml" + biblicalTermsFileName: "BiblicalTerms.xml", + languageCode: "en" ); } } From 6f196946284b8302b45590bd66a642cb0003a572 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Fri, 16 Aug 2024 16:22:59 -0400 Subject: [PATCH 08/15] Accommodate change in class name/structure --- .../src/Serval.Machine.Shared/Services/CorpusService.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs index 17d562ad..1555bf71 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs @@ -43,7 +43,10 @@ public IEnumerable CreateTermCorpora(IReadOnlyList file switch (file.Format) { case FileFormat.Paratext: - yield return new ParatextBackupTermsCorpus(file.Location, ["PN"]); + using (var archive = ZipFile.OpenRead(file.Location)) + { + yield return new ZipParatextProjectTermsCorpus(archive, ["PN"]); + } break; } } From 66a0601885a8d94b44bee7dcc080b8aca5279936 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Tue, 20 Aug 2024 14:36:56 -0400 Subject: [PATCH 09/15] Revert name of machine class --- src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs index 1555bf71..940227d7 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs @@ -45,7 +45,7 @@ public IEnumerable CreateTermCorpora(IReadOnlyList file case FileFormat.Paratext: using (var archive = ZipFile.OpenRead(file.Location)) { - yield return new ZipParatextProjectTermsCorpus(archive, ["PN"]); + yield return new ParatextBackupTermsCorpus(archive, ["PN"]); } break; } From 621bd57c3068a81edc2e64be0981b8b85fd86290 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Wed, 21 Aug 2024 15:47:22 -0400 Subject: [PATCH 10/15] Revert to using filepath in constructor --- .../src/Serval.Machine.Shared/Services/CorpusService.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs index 940227d7..17d562ad 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs @@ -43,10 +43,7 @@ public IEnumerable CreateTermCorpora(IReadOnlyList file switch (file.Format) { case FileFormat.Paratext: - using (var archive = ZipFile.OpenRead(file.Location)) - { - yield return new ParatextBackupTermsCorpus(archive, ["PN"]); - } + yield return new ParatextBackupTermsCorpus(file.Location, ["PN"]); break; } } From 2d9e750d1f0c8c447b493aea03bdde426a4f5004 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Thu, 22 Aug 2024 12:40:46 -0400 Subject: [PATCH 11/15] Update Machine version --- .../src/Serval.Machine.Shared/Serval.Machine.Shared.csproj | 6 +++--- src/Serval/src/Serval.Shared/Serval.Shared.csproj | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj b/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj index 08281b26..6064b202 100644 --- a/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj +++ b/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj @@ -37,9 +37,9 @@ - - - + + + diff --git a/src/Serval/src/Serval.Shared/Serval.Shared.csproj b/src/Serval/src/Serval.Shared/Serval.Shared.csproj index 0270f7df..7e46a9ee 100644 --- a/src/Serval/src/Serval.Shared/Serval.Shared.csproj +++ b/src/Serval/src/Serval.Shared/Serval.Shared.csproj @@ -19,7 +19,7 @@ - + From 2e237c605cfaecea8d8d0d2e96d9bd6b92b8d95f Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Thu, 22 Aug 2024 12:49:29 -0400 Subject: [PATCH 12/15] Merge branch 'key_terms' of https://github.com/sillsdev/serval into key_terms --- .github/workflows/ci.yml | 9 +- Serval.sln | 19 + deploy/qa-ext-values.yaml | 4 +- .../src/SIL.DataAccess/ObjectRefConvention.cs | 2 +- .../EchoTranslationEngine/HealthServiceV1.cs | 15 + src/Echo/src/EchoTranslationEngine/Program.cs | 1 + .../TranslationEngineServiceV1.cs | 22 +- .../IEndpointRouteBuilderExtensions.cs | 1 + .../IMachineBuilderExtensions.cs | 2 +- .../Serval.Machine.Shared.csproj | 3 +- .../Services/LanguageTagService.cs | 149 +- .../Services/ServalHealthServiceV1.cs | 16 + .../ServalPlatformOutboxMessageHandler.cs | 2 +- .../ServalTranslationEngineServiceV1.cs | 15 +- .../src/Serval.Machine.Shared/Usings.cs | 3 +- .../Services/LanguageTagServiceTests.cs | 34 +- ...ServalPlatformOutboxMessageHandlerTests.cs | 6 +- .../Serval.ApiServer/Serval.ApiServer.csproj | 1 + src/Serval/src/Serval.ApiServer/Startup.cs | 91 +- src/Serval/src/Serval.ApiServer/Usings.cs | 3 +- .../appsettings.Development.json | 3 + .../src/Serval.ApiServer/appsettings.json | 3 + .../Configuration/AssessmentOptions.cs | 14 + .../IEndpointRouteBuilderExtensions.cs | 11 + ...iatorRegistrationConfiguratorExtensions.cs | 12 + ...IMemoryDataAccessConfiguratorExtensions.cs | 14 + .../IMongoDataAccessConfiguratorExtensions.cs | 42 + .../Configuration/IServalBuilderExtensions.cs | 45 + .../Consumers/DataFileDeletedConsumer.cs | 11 + .../Contracts/AssessmentCorpusConfigDto.cs | 19 + .../Contracts/AssessmentCorpusDto.cs | 9 + .../AssessmentCorpusFileConfigDto.cs | 8 + .../Contracts/AssessmentCorpusFileDto.cs | 7 + .../Contracts/AssessmentEngineConfigDto.cs | 24 + .../Contracts/AssessmentEngineDto.cs | 11 + .../Contracts/AssessmentJobConfigDto.cs | 15 + .../Contracts/AssessmentJobDto.cs | 27 + .../Contracts/AssessmentResultDto.cs | 9 + .../AssessmentEnginesController.cs | 673 +++++ .../src/Serval.Assessment/Models/Corpus.cs | 8 + .../Serval.Assessment/Models/CorpusFile.cs | 9 + .../src/Serval.Assessment/Models/Engine.cs | 12 + .../src/Serval.Assessment/Models/Job.cs | 16 + .../src/Serval.Assessment/Models/Result.cs | 13 + .../Serval.Assessment.csproj | 26 + .../Services/AssessmentPlatformServiceV1.cs | 257 ++ .../Services/EngineService.cs | 239 ++ .../Services/IEngineService.cs | 22 + .../Serval.Assessment/Services/IJobService.cs | 13 + .../Services/IResultService.cs | 11 + .../Serval.Assessment/Services/JobService.cs | 62 + .../Services/ResultService.cs | 17 + src/Serval/src/Serval.Assessment/Usings.cs | 31 + src/Serval/src/Serval.Client/Client.g.cs | 2251 ++++++++++++++++- .../Controllers/DataFilesController.cs | 4 +- .../Services/DataFileService.cs | 7 +- .../Services/DeletedFileCleaner.cs | 5 +- .../Protos/serval/assessment/v1/engine.proto | 62 + .../serval/assessment/v1/platform.proto | 51 + .../Protos/serval/health/v1/health.proto | 21 + .../Protos/serval/translation/v1/engine.proto | 13 - .../serval/translation/v1/platform.proto | 8 +- .../Utils/WriteGrpcHealthCheckResponse.cs | 2 +- .../Contracts/AssessmentJobFinished.cs | 11 + .../Contracts/AssessmentJobFinishedDto.cs | 10 + .../Contracts/AssessmentJobStarted.cs | 8 + .../Contracts/AssessmentJobStartedDto.cs | 7 + .../Contracts/DataFileUpdated.cs | 6 + .../Contracts/TranslationBuildFinishedDto.cs | 1 + .../Serval.Shared/Controllers/Endpoints.cs | 17 + .../src/Serval.Shared/Controllers/Scopes.cs | 14 +- .../src/Serval.Shared/Serval.Shared.csproj | 1 + .../Services/EntityServiceBase.cs | 14 +- .../Services/GrpcServiceHealthCheck.cs | 8 +- .../Services/OwnedEntityServiceBase.cs | 10 + src/Serval/src/Serval.Shared/Usings.cs | 2 + .../Configuration/IServalBuilderExtensions.cs | 7 +- .../TranslationCorpusUpdateConfigDto.cs | 2 +- .../TranslationEnginesController.cs | 28 +- .../Services/EngineService.cs | 7 +- .../Services/TranslationPlatformServiceV1.cs | 4 +- ...iatorRegistrationConfiguratorExtensions.cs | 2 + .../AssessmentJobFinishedConsumer.cs | 36 + .../Consumers/AssessmentJobStartedConsumer.cs | 33 + .../TranslationBuildFinishedConsumer.cs | 5 +- .../TranslationBuildStartedConsumer.cs | 4 +- .../Serval.Webhooks/Contracts/WebhookEvent.cs | 5 +- .../Controllers/WebhooksController.cs | 4 +- .../Services/IWebhookService.cs | 2 +- .../Services/WebhookService.cs | 7 +- .../AssessmentEngineTests.cs | 223 ++ .../TranslationEngineTests.cs | 25 +- .../Usings.cs | 1 - .../Utils.cs | 27 + .../Services/ScriptureDataFileServiceTests.cs | 9 +- src/Serval/test/Serval.Shared.Tests/Usings.cs | 2 + .../Utils}/NUnitExtensions.cs | 2 +- .../Serval.Translation.Tests.csproj | 1 + .../Services/PlatformServiceTests.cs | 4 +- .../test/Serval.Translation.Tests/Usings.cs | 1 - .../IHealthChecksBuilderExtensions.cs | 12 + .../SIL.ServiceToolkit.csproj | 20 + .../Services/CancellationInterceptor.cs | 2 +- .../Services/HangfireHealthCheck.cs | 2 +- .../Services}/RecurrentTask.cs | 2 +- .../src/SIL.ServiceToolkit/Usings.cs | 12 + .../Utils/LanguageTagParser.cs | 132 + 107 files changed, 4853 insertions(+), 394 deletions(-) create mode 100644 src/Echo/src/EchoTranslationEngine/HealthServiceV1.cs create mode 100644 src/Machine/src/Serval.Machine.Shared/Services/ServalHealthServiceV1.cs create mode 100644 src/Serval/src/Serval.Assessment/Configuration/AssessmentOptions.cs create mode 100644 src/Serval/src/Serval.Assessment/Configuration/IEndpointRouteBuilderExtensions.cs create mode 100644 src/Serval/src/Serval.Assessment/Configuration/IMediatorRegistrationConfiguratorExtensions.cs create mode 100644 src/Serval/src/Serval.Assessment/Configuration/IMemoryDataAccessConfiguratorExtensions.cs create mode 100644 src/Serval/src/Serval.Assessment/Configuration/IMongoDataAccessConfiguratorExtensions.cs create mode 100644 src/Serval/src/Serval.Assessment/Configuration/IServalBuilderExtensions.cs create mode 100644 src/Serval/src/Serval.Assessment/Consumers/DataFileDeletedConsumer.cs create mode 100644 src/Serval/src/Serval.Assessment/Contracts/AssessmentCorpusConfigDto.cs create mode 100644 src/Serval/src/Serval.Assessment/Contracts/AssessmentCorpusDto.cs create mode 100644 src/Serval/src/Serval.Assessment/Contracts/AssessmentCorpusFileConfigDto.cs create mode 100644 src/Serval/src/Serval.Assessment/Contracts/AssessmentCorpusFileDto.cs create mode 100644 src/Serval/src/Serval.Assessment/Contracts/AssessmentEngineConfigDto.cs create mode 100644 src/Serval/src/Serval.Assessment/Contracts/AssessmentEngineDto.cs create mode 100644 src/Serval/src/Serval.Assessment/Contracts/AssessmentJobConfigDto.cs create mode 100644 src/Serval/src/Serval.Assessment/Contracts/AssessmentJobDto.cs create mode 100644 src/Serval/src/Serval.Assessment/Contracts/AssessmentResultDto.cs create mode 100644 src/Serval/src/Serval.Assessment/Controllers/AssessmentEnginesController.cs create mode 100644 src/Serval/src/Serval.Assessment/Models/Corpus.cs create mode 100644 src/Serval/src/Serval.Assessment/Models/CorpusFile.cs create mode 100644 src/Serval/src/Serval.Assessment/Models/Engine.cs create mode 100644 src/Serval/src/Serval.Assessment/Models/Job.cs create mode 100644 src/Serval/src/Serval.Assessment/Models/Result.cs create mode 100644 src/Serval/src/Serval.Assessment/Serval.Assessment.csproj create mode 100644 src/Serval/src/Serval.Assessment/Services/AssessmentPlatformServiceV1.cs create mode 100644 src/Serval/src/Serval.Assessment/Services/EngineService.cs create mode 100644 src/Serval/src/Serval.Assessment/Services/IEngineService.cs create mode 100644 src/Serval/src/Serval.Assessment/Services/IJobService.cs create mode 100644 src/Serval/src/Serval.Assessment/Services/IResultService.cs create mode 100644 src/Serval/src/Serval.Assessment/Services/JobService.cs create mode 100644 src/Serval/src/Serval.Assessment/Services/ResultService.cs create mode 100644 src/Serval/src/Serval.Assessment/Usings.cs create mode 100644 src/Serval/src/Serval.Grpc/Protos/serval/assessment/v1/engine.proto create mode 100644 src/Serval/src/Serval.Grpc/Protos/serval/assessment/v1/platform.proto create mode 100644 src/Serval/src/Serval.Grpc/Protos/serval/health/v1/health.proto create mode 100644 src/Serval/src/Serval.Shared/Contracts/AssessmentJobFinished.cs create mode 100644 src/Serval/src/Serval.Shared/Contracts/AssessmentJobFinishedDto.cs create mode 100644 src/Serval/src/Serval.Shared/Contracts/AssessmentJobStarted.cs create mode 100644 src/Serval/src/Serval.Shared/Contracts/AssessmentJobStartedDto.cs create mode 100644 src/Serval/src/Serval.Shared/Contracts/DataFileUpdated.cs create mode 100644 src/Serval/src/Serval.Shared/Controllers/Endpoints.cs rename src/Serval/src/{Serval.Translation => Serval.Shared}/Services/GrpcServiceHealthCheck.cs (84%) create mode 100644 src/Serval/src/Serval.Shared/Services/OwnedEntityServiceBase.cs create mode 100644 src/Serval/src/Serval.Webhooks/Consumers/AssessmentJobFinishedConsumer.cs create mode 100644 src/Serval/src/Serval.Webhooks/Consumers/AssessmentJobStartedConsumer.cs create mode 100644 src/Serval/test/Serval.ApiServer.IntegrationTests/AssessmentEngineTests.cs create mode 100644 src/Serval/test/Serval.ApiServer.IntegrationTests/Utils.cs rename src/Serval/test/{Serval.Translation.Tests/Services => Serval.Shared.Tests/Utils}/NUnitExtensions.cs (87%) create mode 100644 src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/IHealthChecksBuilderExtensions.cs create mode 100644 src/ServiceToolkit/src/SIL.ServiceToolkit/SIL.ServiceToolkit.csproj rename src/{Machine/src/Serval.Machine.Shared => ServiceToolkit/src/SIL.ServiceToolkit}/Services/CancellationInterceptor.cs (95%) rename src/{Machine/src/Serval.Machine.Shared => ServiceToolkit/src/SIL.ServiceToolkit}/Services/HangfireHealthCheck.cs (94%) rename src/{Machine/src/Serval.Machine.Shared/Utils => ServiceToolkit/src/SIL.ServiceToolkit/Services}/RecurrentTask.cs (96%) create mode 100644 src/ServiceToolkit/src/SIL.ServiceToolkit/Usings.cs create mode 100644 src/ServiceToolkit/src/SIL.ServiceToolkit/Utils/LanguageTagParser.cs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e335e757..80a5130b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,14 @@ jobs: - name: Pre-Test run: sudo mkdir -p /var/lib/serval && sudo chmod 777 /var/lib/serval - name: Test - run: dotnet test --verbosity normal --filter "TestCategory!=E2E&TestCategory!=E2EMissingServices" --collect:"Xplat Code Coverage" + run: dotnet test --verbosity normal --filter "TestCategory!=E2E&TestCategory!=E2EMissingServices" --collect:"Xplat Code Coverage" --logger "trx;LogFileName=test-results.trx" + - name: Test report + uses: dorny/test-reporter@v1 + if: success() || failure() + with: + name: NUnit Tests + path: src/**/TestResults/test-results.trx + reporter: dotnet-trx - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 env: diff --git a/Serval.sln b/Serval.sln index 8188624d..edd3f075 100644 --- a/Serval.sln +++ b/Serval.sln @@ -78,6 +78,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Machine.JobServer", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Machine.Shared.Tests", "src\Machine\test\Serval.Machine.Shared.Tests\Serval.Machine.Shared.Tests.csproj", "{B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Assessment", "src\Serval\src\Serval.Assessment\Serval.Assessment.csproj", "{10657805-48F1-4205-B8F5-79447F6EF620}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ServiceToolkit", "ServiceToolkit", "{EA69B41C-49EF-4017-A687-44B9DF37FF98}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{C3A14577-A654-4604-818C-4E683DD45A51}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.ServiceToolkit", "src\ServiceToolkit\src\SIL.ServiceToolkit\SIL.ServiceToolkit.csproj", "{0E40F959-C641-40A2-9750-B17A4F9F9E55}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -164,6 +172,14 @@ Global {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Debug|Any CPU.Build.0 = Debug|Any CPU {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Release|Any CPU.ActiveCfg = Release|Any CPU {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Release|Any CPU.Build.0 = Release|Any CPU + {10657805-48F1-4205-B8F5-79447F6EF620}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {10657805-48F1-4205-B8F5-79447F6EF620}.Debug|Any CPU.Build.0 = Debug|Any CPU + {10657805-48F1-4205-B8F5-79447F6EF620}.Release|Any CPU.ActiveCfg = Release|Any CPU + {10657805-48F1-4205-B8F5-79447F6EF620}.Release|Any CPU.Build.0 = Release|Any CPU + {0E40F959-C641-40A2-9750-B17A4F9F9E55}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0E40F959-C641-40A2-9750-B17A4F9F9E55}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0E40F959-C641-40A2-9750-B17A4F9F9E55}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0E40F959-C641-40A2-9750-B17A4F9F9E55}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -196,6 +212,9 @@ Global {C02494FB-663E-4430-9F2D-41F1A740B271} = {D808D2BE-ED26-4E60-A409-AE58F7C1CB8F} {BC766753-E560-4ADF-9923-C7A96076EA47} = {D808D2BE-ED26-4E60-A409-AE58F7C1CB8F} {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D} = {40C225C2-1EEF-4D1D-9D14-1CBB86C8A1CB} + {10657805-48F1-4205-B8F5-79447F6EF620} = {25CDB05B-4E24-4A6E-933E-1E0BEC97D74D} + {C3A14577-A654-4604-818C-4E683DD45A51} = {EA69B41C-49EF-4017-A687-44B9DF37FF98} + {0E40F959-C641-40A2-9750-B17A4F9F9E55} = {C3A14577-A654-4604-818C-4E683DD45A51} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9F18C25E-E140-43C3-B177-D562E1628370} diff --git a/deploy/qa-ext-values.yaml b/deploy/qa-ext-values.yaml index 2c237e5b..fcc786a6 100644 --- a/deploy/qa-ext-values.yaml +++ b/deploy/qa-ext-values.yaml @@ -1,6 +1,6 @@ externalHost: qa.serval-api.org environment: Production -deploymentVersion: '1.5.QA5.1' +deploymentVersion: '1.5.QA6' alertEmail: ext-qa-serval-alerts@languagetechnology.org emailsToAlert: john_lambert@sil.org enableTls: true @@ -8,7 +8,7 @@ namespace: serval auth0Domain: dev-sillsdev.auth0.com lokiTenent: serval-tenant lokiUrl: http://loki-distributed-gateway.loki.svc.cluster.local -servalImage: ghcr.io/sillsdev/serval:1.5.5.1 +servalImage: ghcr.io/sillsdev/serval:1.5.6 ClearMLDockerImage: ghcr.io/sillsdev/machine.py:1.5.4 ClearMLQueue: production MongoConnectionPrefix: qa_ diff --git a/src/DataAccess/src/SIL.DataAccess/ObjectRefConvention.cs b/src/DataAccess/src/SIL.DataAccess/ObjectRefConvention.cs index 9110d05d..70cefe21 100644 --- a/src/DataAccess/src/SIL.DataAccess/ObjectRefConvention.cs +++ b/src/DataAccess/src/SIL.DataAccess/ObjectRefConvention.cs @@ -4,7 +4,7 @@ public class ObjectRefConvention : ConventionBase, IMemberMapConvention { public void Apply(BsonMemberMap memberMap) { - if (memberMap.MemberName.EndsWith("Ref")) + if (memberMap.MemberName.EndsWith("Ref") && memberMap.MemberName.Length > 3) memberMap.SetSerializer(new StringSerializer(BsonType.ObjectId)); } } diff --git a/src/Echo/src/EchoTranslationEngine/HealthServiceV1.cs b/src/Echo/src/EchoTranslationEngine/HealthServiceV1.cs new file mode 100644 index 00000000..05bc98c1 --- /dev/null +++ b/src/Echo/src/EchoTranslationEngine/HealthServiceV1.cs @@ -0,0 +1,15 @@ +using Serval.Health.V1; + +namespace EchoTranslationEngine; + +public class HealthServiceV1(HealthCheckService healthCheckService) : HealthApi.HealthApiBase +{ + private readonly HealthCheckService _healthCheckService = healthCheckService; + + public override async Task HealthCheck(Empty request, ServerCallContext context) + { + HealthReport healthReport = await _healthCheckService.CheckHealthAsync(); + HealthCheckResponse healthCheckResponse = WriteGrpcHealthCheckResponse.Generate(healthReport); + return healthCheckResponse; + } +} diff --git a/src/Echo/src/EchoTranslationEngine/Program.cs b/src/Echo/src/EchoTranslationEngine/Program.cs index 11a3323d..a931cd7e 100644 --- a/src/Echo/src/EchoTranslationEngine/Program.cs +++ b/src/Echo/src/EchoTranslationEngine/Program.cs @@ -18,5 +18,6 @@ app.UseHttpsRedirection(); app.MapGrpcService(); +app.MapGrpcService(); app.Run(); diff --git a/src/Echo/src/EchoTranslationEngine/TranslationEngineServiceV1.cs b/src/Echo/src/EchoTranslationEngine/TranslationEngineServiceV1.cs index 4db7af78..8a348c8d 100644 --- a/src/Echo/src/EchoTranslationEngine/TranslationEngineServiceV1.cs +++ b/src/Echo/src/EchoTranslationEngine/TranslationEngineServiceV1.cs @@ -1,13 +1,10 @@ namespace EchoTranslationEngine; -public class TranslationEngineServiceV1(BackgroundTaskQueue taskQueue, HealthCheckService healthCheckService) - : TranslationEngineApi.TranslationEngineApiBase +public class TranslationEngineServiceV1(BackgroundTaskQueue taskQueue) : TranslationEngineApi.TranslationEngineApiBase { private static readonly Empty Empty = new(); private readonly BackgroundTaskQueue _taskQueue = taskQueue; - private readonly HealthCheckService _healthCheckService = healthCheckService; - public override Task Create(CreateRequest request, ServerCallContext context) { if (request.SourceLanguage != request.TargetLanguage) @@ -79,7 +76,7 @@ await client.BuildStartedAsync( try { using ( - AsyncClientStreamingCall call = + AsyncClientStreamingCall call = client.InsertPretranslations(cancellationToken: cancellationToken) ) { @@ -124,7 +121,7 @@ await client.BuildStartedAsync( if (sourceLine.Length > 0 && targetLine.Length == 0) { await call.RequestStream.WriteAsync( - new InsertPretranslationRequest + new InsertPretranslationsRequest { EngineId = request.EngineId, CorpusId = corpus.Id, @@ -157,7 +154,7 @@ await call.RequestStream.WriteAsync( if (sourceLine.Length > 0 && targetLine.Length == 0) { await call.RequestStream.WriteAsync( - new InsertPretranslationRequest + new InsertPretranslationsRequest { EngineId = request.EngineId, CorpusId = corpus.Id, @@ -182,7 +179,7 @@ await call.RequestStream.WriteAsync( if (sourceLine.Length > 0) { await call.RequestStream.WriteAsync( - new InsertPretranslationRequest + new InsertPretranslationsRequest { EngineId = request.EngineId, CorpusId = corpus.Id, @@ -203,7 +200,7 @@ await call.RequestStream.WriteAsync( if (sourceLine.Length > 0) { await call.RequestStream.WriteAsync( - new InsertPretranslationRequest + new InsertPretranslationsRequest { EngineId = request.EngineId, CorpusId = corpus.Id, @@ -319,11 +316,4 @@ ServerCallContext context new GetLanguageInfoResponse { InternalCode = request.Language + "_echo", IsNative = true, } ); } - - public override async Task HealthCheck(Empty request, ServerCallContext context) - { - HealthReport healthReport = await _healthCheckService.CheckHealthAsync(); - HealthCheckResponse healthCheckResponse = WriteGrpcHealthCheckResponse.Generate(healthReport); - return healthCheckResponse; - } } diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IEndpointRouteBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IEndpointRouteBuilderExtensions.cs index 694dd67e..107de6c2 100644 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/IEndpointRouteBuilderExtensions.cs +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/IEndpointRouteBuilderExtensions.cs @@ -5,6 +5,7 @@ public static class IEndpointRouteBuilderExtensions public static IEndpointRouteBuilder MapServalTranslationEngineService(this IEndpointRouteBuilder builder) { builder.MapGrpcService(); + builder.MapGrpcService(); return builder; } diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs index 2495da6e..d67afb90 100644 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs @@ -192,7 +192,7 @@ public static IMachineBuilder AddMongoHangfireJobClient( .UseMongoStorage(connectionString, GetMongoStorageOptions()) .UseFilter(new AutomaticRetryAttribute { Attempts = 0 }) ); - builder.Services.AddHealthChecks().AddCheck(name: "Hangfire"); + builder.Services.AddHealthChecks().AddHangfire(); return builder; } diff --git a/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj b/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj index 6064b202..8397d98d 100644 --- a/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj +++ b/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj @@ -32,7 +32,7 @@ - + @@ -50,6 +50,7 @@ + diff --git a/src/Machine/src/Serval.Machine.Shared/Services/LanguageTagService.cs b/src/Machine/src/Serval.Machine.Shared/Services/LanguageTagService.cs index 30e065f5..01a19f71 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/LanguageTagService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/LanguageTagService.cs @@ -2,103 +2,18 @@ public class LanguageTagService : ILanguageTagService { - private static readonly Dictionary StandardLanguages = - new() - { - { "ar", "arb" }, - { "ms", "zsm" }, - { "lv", "lvs" }, - { "ne", "npi" }, - { "sw", "swh" }, - { "cmn", "zh" } - }; - - private static readonly Dictionary StandardScripts = new() { { "Kore", "Hang" } }; - - private readonly Dictionary _defaultScripts; - - private readonly Dictionary _flores200Languages; - - private static readonly Regex LangTagPattern = - new("(?'language'[a-zA-Z]{2,8})([_-](?'script'[a-zA-Z]{4}))?", RegexOptions.ExplicitCapture); - - public LanguageTagService() - { - // initialize SLDR language tags to retrieve latest langtags.json file - _defaultScripts = InitializeDefaultScripts(); - _flores200Languages = InitializeFlores200Languages(); - } - - protected virtual void InitializeSldrLanguageTags() - { - Sldr.InitializeLanguageTags(); - } - - private Dictionary InitializeDefaultScripts() - { - InitializeSldrLanguageTags(); - var cachedAllTagsPath = Path.Combine(Sldr.SldrCachePath, "langtags.json"); - JsonNode? json; - - if (!File.Exists(cachedAllTagsPath)) - { - using HttpClient client = new(); - using HttpResponseMessage response = client.Send( - new HttpRequestMessage( - HttpMethod.Get, - "https://raw.githubusercontent.com/silnrsi/langtags/master/pub/langtags.json" - ) - ); - response.EnsureSuccessStatusCode(); - using Stream responseStream = response.Content.ReadAsStream(); - using FileStream fileStream = new(cachedAllTagsPath, FileMode.Create); - responseStream.CopyTo(fileStream); - } - using FileStream stream = new(cachedAllTagsPath, FileMode.Open); - json = JsonNode.Parse(stream); - - Dictionary tempDefaultScripts = new(); - foreach (JsonNode? entry in json!.AsArray()) - { - if (entry is null) - continue; - - var script = (string?)entry["script"]; - if (script is null) - continue; - - JsonNode? tags = entry["tags"]; - if (tags is not null) - { - foreach (var t in tags.AsArray().Select(v => (string?)v)) - { - if ( - t is not null - && IetfLanguageTag.TryGetParts(t, out _, out string? s, out _, out _) - && s is null - ) - { - tempDefaultScripts[t] = script; - } - } - } - - var tag = (string?)entry["tag"]; - if (tag is not null) - tempDefaultScripts[tag] = script; - } - return tempDefaultScripts; - } + private readonly Dictionary _flores200Languages = InitializeFlores200Languages(); + private readonly LanguageTagParser _parser = new(); private static Dictionary InitializeFlores200Languages() { - var tempFlores200Languages = new Dictionary(); + Dictionary flores200Languages = []; using var floresStream = Assembly .GetExecutingAssembly() .GetManifestResourceStream("Serval.Machine.Shared.data.flores200languages.csv"); Debug.Assert(floresStream is not null); - var reader = new StreamReader(floresStream); - var firstLine = reader.ReadLine(); + StreamReader reader = new(floresStream); + string? firstLine = reader.ReadLine(); Debug.Assert(firstLine == "language, code"); while (!reader.EndOfStream) { @@ -106,9 +21,9 @@ private static Dictionary InitializeFlores200Languages() if (line is null) continue; string[] values = line.Split(','); - tempFlores200Languages[values[1].Trim()] = values[0].Trim(); + flores200Languages[values[1].Trim()] = values[0].Trim(); } - return tempFlores200Languages; + return flores200Languages; } /** @@ -119,52 +34,10 @@ private static Dictionary InitializeFlores200Languages() */ public bool ConvertToFlores200Code(string languageTag, out string flores200Code) { - flores200Code = ResolveLanguageTag(languageTag); - return _flores200Languages.ContainsKey(flores200Code); - } - - private string ResolveLanguageTag(string languageTag) - { - // Try to find a pattern of {language code}_{script} - Match langTagMatch = LangTagPattern.Match(languageTag); - if (!langTagMatch.Success) - return languageTag; - string parsedLanguage = langTagMatch.Groups["language"].Value; - string languageSubtag = parsedLanguage; - string iso639_3Code = parsedLanguage; - - // Best attempt to convert language to a registered ISO 639-3 code - // Uses https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry for mapping - - // If they gave us the ISO code, revert it to the 2 character code - if (StandardSubtags.TryGetLanguageFromIso3Code(languageSubtag, out LanguageSubtag tempSubtag)) - languageSubtag = tempSubtag.Code; - - // There are a few extra conversions not in SIL Writing Systems that we need to handle - if (StandardLanguages.TryGetValue(languageSubtag, out string? tempName)) - languageSubtag = tempName; - - if (StandardSubtags.RegisteredLanguages.TryGet(languageSubtag, out LanguageSubtag? languageSubtagObj)) - iso639_3Code = languageSubtagObj.Iso3Code; - - // Use default script unless there is one parsed out of the language tag - Group scriptGroup = langTagMatch.Groups["script"]; - string? script = null; - - if (scriptGroup.Success) - script = scriptGroup.Value; - else if (_defaultScripts.TryGetValue(languageTag, out string? tempScript2)) - script = tempScript2; - else if (_defaultScripts.TryGetValue(languageSubtag, out string? tempScript)) - script = tempScript; - - // There are a few extra conversions not in SIL Writing Systems that we need to handle - if (script is not null && StandardScripts.TryGetValue(script, out string? tempScript3)) - script = tempScript3; - - if (script is not null) - return $"{iso639_3Code}_{script}"; + if (_parser.TryParse(languageTag, out string? languageCode, out string? scriptCode)) + flores200Code = $"{languageCode}_{scriptCode}"; else - return languageTag; + flores200Code = languageTag; + return _flores200Languages.ContainsKey(flores200Code); } } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalHealthServiceV1.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalHealthServiceV1.cs new file mode 100644 index 00000000..57221e6d --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalHealthServiceV1.cs @@ -0,0 +1,16 @@ +using Google.Protobuf.WellKnownTypes; +using Serval.Health.V1; + +namespace Serval.Machine.Shared.Services; + +public class ServalHealthServiceV1(HealthCheckService healthCheckService) : HealthApi.HealthApiBase +{ + private readonly HealthCheckService _healthCheckService = healthCheckService; + + public override async Task HealthCheck(Empty request, ServerCallContext context) + { + HealthReport healthReport = await _healthCheckService.CheckHealthAsync(); + HealthCheckResponse healthCheckResponse = WriteGrpcHealthCheckResponse.Generate(healthReport); + return healthCheckResponse; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxMessageHandler.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxMessageHandler.cs index 9b1cf788..41132504 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxMessageHandler.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxMessageHandler.cs @@ -64,7 +64,7 @@ await _client.BuildRestartingAsync( await foreach (Pretranslation pretranslation in pretranslations) { await call.RequestStream.WriteAsync( - new InsertPretranslationRequest + new InsertPretranslationsRequest { EngineId = content!, CorpusId = pretranslation.CorpusId, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs index 863fd158..049889b9 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs @@ -3,18 +3,14 @@ namespace Serval.Machine.Shared.Services; -public class ServalTranslationEngineServiceV1( - IEnumerable engineServices, - HealthCheckService healthCheckService -) : TranslationEngineApi.TranslationEngineApiBase +public class ServalTranslationEngineServiceV1(IEnumerable engineServices) + : TranslationEngineApi.TranslationEngineApiBase { private static readonly Empty Empty = new(); private readonly Dictionary _engineServices = engineServices.ToDictionary(es => es.Type); - private readonly HealthCheckService _healthCheckService = healthCheckService; - public override async Task Create(CreateRequest request, ServerCallContext context) { ITranslationEngineService engineService = GetEngineService(request.EngineType); @@ -172,13 +168,6 @@ ServerCallContext context return Task.FromResult(new GetLanguageInfoResponse { InternalCode = internalCode, IsNative = isNative, }); } - public override async Task HealthCheck(Empty request, ServerCallContext context) - { - HealthReport healthReport = await _healthCheckService.CheckHealthAsync(); - HealthCheckResponse healthCheckResponse = WriteGrpcHealthCheckResponse.Generate(healthReport); - return healthCheckResponse; - } - private ITranslationEngineService GetEngineService(string engineTypeStr) { if (_engineServices.TryGetValue(GetEngineType(engineTypeStr), out ITranslationEngineService? service)) diff --git a/src/Machine/src/Serval.Machine.Shared/Usings.cs b/src/Machine/src/Serval.Machine.Shared/Usings.cs index 244e090b..159f4f01 100644 --- a/src/Machine/src/Serval.Machine.Shared/Usings.cs +++ b/src/Machine/src/Serval.Machine.Shared/Usings.cs @@ -16,7 +16,6 @@ global using System.Text.Json; global using System.Text.Json.Nodes; global using System.Text.Json.Serialization; -global using System.Text.RegularExpressions; global using Amazon; global using Amazon.Runtime; global using Amazon.S3; @@ -56,4 +55,6 @@ global using SIL.Machine.Translation.Thot; global using SIL.Machine.Utils; global using SIL.Scripture; +global using SIL.ServiceToolkit.Services; +global using SIL.ServiceToolkit.Utils; global using SIL.WritingSystems; diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/LanguageTagServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/LanguageTagServiceTests.cs index 008b13e0..f67f6293 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/LanguageTagServiceTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/LanguageTagServiceTests.cs @@ -3,6 +3,13 @@ [TestFixture] public class LanguageTagServiceTests { + [OneTimeSetUp] + public void OneTimeSetUp() + { + if (!Sldr.IsInitialized) + Sldr.Initialize(); + } + [Test] [TestCase("es", "spa_Latn", Description = "Iso639_1Code")] [TestCase("hne", "hne_Deva", Description = "Iso639_3Code")] @@ -21,8 +28,6 @@ public class LanguageTagServiceTests [TestCase("kor_Kore", "kor_Hang", Description = "KoreanScriptCorrection")] public void ConvertToFlores200CodeTest(string language, string internalCodeTruth) { - if (!Sldr.IsInitialized) - Sldr.Initialize(); new LanguageTagService().ConvertToFlores200Code(language, out string internalCode); Assert.That(internalCode, Is.EqualTo(internalCodeTruth)); } @@ -34,8 +39,6 @@ public void ConvertToFlores200CodeTest(string language, string internalCodeTruth [TestCase("xyz", "xyz", false)] public void GetLanguageInfoAsync(string languageCode, string? resolvedLanguageCode, bool nativeLanguageSupport) { - if (!Sldr.IsInitialized) - Sldr.Initialize(); bool isNative = new LanguageTagService().ConvertToFlores200Code(languageCode, out string internalCode); Assert.Multiple(() => { @@ -43,27 +46,4 @@ public void GetLanguageInfoAsync(string languageCode, string? resolvedLanguageCo Assert.That(isNative, Is.EqualTo(nativeLanguageSupport)); }); } - - public class TestLanguageTagService : LanguageTagService - { - // Don't call Sldr initialize to call - protected override void InitializeSldrLanguageTags() - { - // remove langtags.json to force download - var cachedAllTagsPath = Path.Combine(Sldr.SldrCachePath, "langtags.json"); - if (File.Exists(cachedAllTagsPath)) - File.Delete(cachedAllTagsPath); - Directory.CreateDirectory(Sldr.SldrCachePath); - } - } - - [Test] - public void BackupLangtagsJsonTest() - { - if (!Sldr.IsInitialized) - Sldr.Initialize(); - var service = new TestLanguageTagService(); - service.ConvertToFlores200Code("en", out string internalCode); - Assert.That(internalCode, Is.EqualTo("eng_Latn")); - } } diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/ServalPlatformOutboxMessageHandlerTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/ServalPlatformOutboxMessageHandlerTests.cs index f3667838..3bc63f98 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/ServalPlatformOutboxMessageHandlerTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/ServalPlatformOutboxMessageHandlerTests.cs @@ -54,7 +54,7 @@ await env.Handler.HandleMessageAsync( _ = env.Client.Received(1).InsertPretranslations(); _ = env.PretranslationWriter.Received(1) .WriteAsync( - new InsertPretranslationRequest + new InsertPretranslationsRequest { EngineId = "engine1", CorpusId = "corpus1", @@ -78,7 +78,7 @@ public TestEnvironment() Client .IncrementTranslationEngineCorpusSizeAsync(Arg.Any()) .Returns(CreateEmptyUnaryCall()); - PretranslationWriter = Substitute.For>(); + PretranslationWriter = Substitute.For>(); Client .InsertPretranslations(cancellationToken: Arg.Any()) .Returns( @@ -97,7 +97,7 @@ public TestEnvironment() public TranslationPlatformApi.TranslationPlatformApiClient Client { get; } public ServalPlatformOutboxMessageHandler Handler { get; } - public IClientStreamWriter PretranslationWriter { get; } + public IClientStreamWriter PretranslationWriter { get; } private static AsyncUnaryCall CreateEmptyUnaryCall() { diff --git a/src/Serval/src/Serval.ApiServer/Serval.ApiServer.csproj b/src/Serval/src/Serval.ApiServer/Serval.ApiServer.csproj index eb410d83..f9ad4210 100644 --- a/src/Serval/src/Serval.ApiServer/Serval.ApiServer.csproj +++ b/src/Serval/src/Serval.ApiServer/Serval.ApiServer.csproj @@ -44,6 +44,7 @@ + diff --git a/src/Serval/src/Serval.ApiServer/Startup.cs b/src/Serval/src/Serval.ApiServer/Startup.cs index cf01ac17..27e142e2 100644 --- a/src/Serval/src/Serval.ApiServer/Startup.cs +++ b/src/Serval/src/Serval.ApiServer/Startup.cs @@ -8,6 +8,7 @@ public class Startup(IConfiguration configuration, IWebHostEnvironment environme public void ConfigureServices(IServiceCollection services) { + services.AddFeatureManagement(); services.AddRouting(o => o.LowercaseUrls = true); services.AddOutputCache(options => { @@ -71,10 +72,12 @@ public void ConfigureServices(IServiceCollection services) .AddMongoDataAccess(cfg => { cfg.AddTranslationRepositories(); + cfg.AddAssessmentRepositories(); cfg.AddDataFilesRepositories(); cfg.AddWebhooksRepositories(); }) .AddTranslation() + .AddAssessment() .AddDataFiles() .AddWebhooks(); services.AddTransient(); @@ -102,6 +105,7 @@ public void ConfigureServices(IServiceCollection services) services.AddMediator(cfg => { cfg.AddTranslationConsumers(); + cfg.AddAssessmentConsumers(); cfg.AddDataFilesConsumers(); cfg.AddWebhooksConsumers(); }); @@ -128,49 +132,59 @@ public void ConfigureServices(IServiceCollection services) Version[] versions = [new Version(1, 0)]; foreach (Version version in versions) { - services.AddSwaggerDocument(o => - { - o.SchemaSettings.SchemaType = SchemaType.Swagger2; - o.Title = "Serval API"; - o.Description = "Natural language processing services for minority language Bible translation."; - o.DocumentName = "v" + version.Major; - o.ApiGroupNames = new[] { "v" + version.Major }; - o.Version = version.Major + "." + version.Minor; - - o.SchemaSettings.SchemaNameGenerator = new ServalSchemaNameGenerator(); - o.UseControllerSummaryAsTagDescription = true; - o.AddSecurity( - "bearer", - Enumerable.Empty(), - new OpenApiSecurityScheme + services.AddSwaggerDocument( + (o, sp) => + { + o.SchemaSettings.SchemaType = SchemaType.Swagger2; + o.Title = "Serval API"; + o.Description = "Natural language processing services for minority language Bible translation."; + o.DocumentName = "v" + version.Major; + o.ApiGroupNames = ["v" + version.Major]; + o.Version = version.Major + "." + version.Minor; + + var featureManager = sp.GetRequiredService(); + if (!featureManager.IsEnabledAsync("Assessment").WaitAndUnwrapException()) { - Type = OpenApiSecuritySchemeType.OAuth2, - Description = "Auth0 Client Credentials Flow", - Flow = OpenApiOAuth2Flow.Application, - Flows = new OpenApiOAuthFlows + o.AddOperationFilter(ctxt => + !(ctxt.ControllerType.Namespace?.StartsWith("Serval.Assessment") ?? true) + ); + } + + o.SchemaSettings.SchemaNameGenerator = new ServalSchemaNameGenerator(); + o.UseControllerSummaryAsTagDescription = true; + o.AddSecurity( + "bearer", + Enumerable.Empty(), + new OpenApiSecurityScheme { - ClientCredentials = new OpenApiOAuthFlow + Type = OpenApiSecuritySchemeType.OAuth2, + Description = "Auth0 Client Credentials Flow", + Flow = OpenApiOAuth2Flow.Application, + Flows = new OpenApiOAuthFlows { - AuthorizationUrl = $"{authority}authorize", - TokenUrl = $"{authority}oauth/token" - } - }, - } - ); - o.OperationProcessors.Add(new AspNetCoreOperationSecurityScopeProcessor("bearer")); + ClientCredentials = new OpenApiOAuthFlow + { + AuthorizationUrl = $"{authority}authorize", + TokenUrl = $"{authority}oauth/token" + } + }, + } + ); + o.OperationProcessors.Add(new AspNetCoreOperationSecurityScopeProcessor("bearer")); - o.SchemaSettings.AllowReferencesWithProperties = true; - o.PostProcess = document => - { - string prefix = "/api/v" + version.Major; - document.Servers.Add(new OpenApiServer { Url = prefix }); - foreach (KeyValuePair pair in document.Paths.ToArray()) + o.SchemaSettings.AllowReferencesWithProperties = true; + o.PostProcess = document => { - document.Paths.Remove(pair.Key); - document.Paths[pair.Key[prefix.Length..]] = pair.Value; - } - }; - }); + string prefix = "/api/v" + version.Major; + document.Servers.Add(new OpenApiServer { Url = prefix }); + foreach (KeyValuePair pair in document.Paths.ToArray()) + { + document.Paths.Remove(pair.Key); + document.Paths[pair.Key[prefix.Length..]] = pair.Value; + } + }; + } + ); } if (Environment.IsDevelopment()) { @@ -207,6 +221,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) { x.MapControllers(); x.MapServalTranslationServices(); + x.MapServalAssessmentServices(); x.MapHangfireDashboard(); }); diff --git a/src/Serval/src/Serval.ApiServer/Usings.cs b/src/Serval/src/Serval.ApiServer/Usings.cs index 47f8718c..377cd261 100644 --- a/src/Serval/src/Serval.ApiServer/Usings.cs +++ b/src/Serval/src/Serval.ApiServer/Usings.cs @@ -12,9 +12,10 @@ global using Microsoft.AspNetCore.Authorization; global using Microsoft.AspNetCore.Mvc; global using Microsoft.AspNetCore.OutputCaching; -global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Diagnostics.HealthChecks; +global using Microsoft.FeatureManagement; global using Microsoft.IdentityModel.Tokens; +global using Nito.AsyncEx.Synchronous; global using NJsonSchema; global using NJsonSchema.Generation; global using NSwag; diff --git a/src/Serval/src/Serval.ApiServer/appsettings.Development.json b/src/Serval/src/Serval.ApiServer/appsettings.Development.json index 2e7c98fc..8a5d5cd6 100644 --- a/src/Serval/src/Serval.ApiServer/appsettings.Development.json +++ b/src/Serval/src/Serval.ApiServer/appsettings.Development.json @@ -4,6 +4,9 @@ "Protocols": "Http2" } }, + "FeatureManagement": { + "Assessment": true + }, "ConnectionStrings": { "Mongo": "mongodb://localhost:27017/serval", "Hangfire": "mongodb://localhost:27017/serval_jobs" diff --git a/src/Serval/src/Serval.ApiServer/appsettings.json b/src/Serval/src/Serval.ApiServer/appsettings.json index de5cb333..cb35bd0f 100644 --- a/src/Serval/src/Serval.ApiServer/appsettings.json +++ b/src/Serval/src/Serval.ApiServer/appsettings.json @@ -1,5 +1,8 @@ { "AllowedHosts": "*", + "FeatureManagement": { + "Assessment": false + }, "Auth": { "Domain": "sil-appbuilder.auth0.com", "Audience": "https://serval-api.org/" diff --git a/src/Serval/src/Serval.Assessment/Configuration/AssessmentOptions.cs b/src/Serval/src/Serval.Assessment/Configuration/AssessmentOptions.cs new file mode 100644 index 00000000..d119df1f --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Configuration/AssessmentOptions.cs @@ -0,0 +1,14 @@ +namespace Serval.Assessment.Configuration; + +public class AssessmentOptions +{ + public const string Key = "Assessment"; + + public List Engines { get; set; } = new List(); +} + +public class EngineInfo +{ + public string Type { get; set; } = ""; + public string Address { get; set; } = ""; +} diff --git a/src/Serval/src/Serval.Assessment/Configuration/IEndpointRouteBuilderExtensions.cs b/src/Serval/src/Serval.Assessment/Configuration/IEndpointRouteBuilderExtensions.cs new file mode 100644 index 00000000..64ef9cad --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Configuration/IEndpointRouteBuilderExtensions.cs @@ -0,0 +1,11 @@ +namespace Microsoft.AspNetCore.Builder; + +public static class IEndpointRouteBuilderExtensions +{ + public static IEndpointRouteBuilder MapServalAssessmentServices(this IEndpointRouteBuilder builder) + { + builder.MapGrpcService(); + + return builder; + } +} diff --git a/src/Serval/src/Serval.Assessment/Configuration/IMediatorRegistrationConfiguratorExtensions.cs b/src/Serval/src/Serval.Assessment/Configuration/IMediatorRegistrationConfiguratorExtensions.cs new file mode 100644 index 00000000..8b9bd293 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Configuration/IMediatorRegistrationConfiguratorExtensions.cs @@ -0,0 +1,12 @@ +namespace Microsoft.Extensions.DependencyInjection; + +public static class IMediatorRegistrationConfiguratorExtensions +{ + public static IMediatorRegistrationConfigurator AddAssessmentConsumers( + this IMediatorRegistrationConfigurator configurator + ) + { + configurator.AddConsumer(); + return configurator; + } +} diff --git a/src/Serval/src/Serval.Assessment/Configuration/IMemoryDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.Assessment/Configuration/IMemoryDataAccessConfiguratorExtensions.cs new file mode 100644 index 00000000..047d4b48 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Configuration/IMemoryDataAccessConfiguratorExtensions.cs @@ -0,0 +1,14 @@ +namespace Microsoft.Extensions.DependencyInjection; + +public static class IMemoryDataAccessConfiguratorExtensions +{ + public static IMemoryDataAccessConfigurator AddAssessmentRepositories( + this IMemoryDataAccessConfigurator configurator + ) + { + configurator.AddRepository(); + configurator.AddRepository(); + configurator.AddRepository(); + return configurator; + } +} diff --git a/src/Serval/src/Serval.Assessment/Configuration/IMongoDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.Assessment/Configuration/IMongoDataAccessConfiguratorExtensions.cs new file mode 100644 index 00000000..8ef721b1 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Configuration/IMongoDataAccessConfiguratorExtensions.cs @@ -0,0 +1,42 @@ +using MongoDB.Driver; + +namespace Microsoft.Extensions.DependencyInjection; + +public static class IMongoDataAccessConfiguratorExtensions +{ + public static IMongoDataAccessConfigurator AddAssessmentRepositories(this IMongoDataAccessConfigurator configurator) + { + configurator.AddRepository( + "assessment.engines", + init: async c => + { + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(e => e.Owner)) + ); + } + ); + configurator.AddRepository( + "assessment.jobs", + init: c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(b => b.EngineRef)) + ) + ); + configurator.AddRepository( + "assessment.results", + init: async c => + { + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(pt => pt.EngineRef)) + ); + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(pt => pt.JobRef)) + ); + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(pt => pt.TextId)) + ); + } + ); + return configurator; + } +} diff --git a/src/Serval/src/Serval.Assessment/Configuration/IServalBuilderExtensions.cs b/src/Serval/src/Serval.Assessment/Configuration/IServalBuilderExtensions.cs new file mode 100644 index 00000000..d770433d --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Configuration/IServalBuilderExtensions.cs @@ -0,0 +1,45 @@ +using Serval.Assessment.V1; +using Serval.Health.V1; + +namespace Microsoft.Extensions.DependencyInjection; + +public static class IServalBuilderExtensions +{ + public static IServalBuilder AddAssessment(this IServalBuilder builder, Action? configure = null) + { + if (builder.Configuration is null) + { + builder.AddApiOptions(o => { }); + builder.AddDataFileOptions(o => { }); + } + else + { + builder.AddApiOptions(builder.Configuration.GetSection(ApiOptions.Key)); + builder.AddDataFileOptions(builder.Configuration.GetSection(DataFileOptions.Key)); + } + + builder.Services.AddScoped(); + builder.Services.AddScoped(); + builder.Services.AddScoped(); + + var assessmentOptions = new AssessmentOptions(); + builder.Configuration?.GetSection(AssessmentOptions.Key).Bind(assessmentOptions); + if (configure is not null) + configure(assessmentOptions); + + foreach (EngineInfo engine in assessmentOptions.Engines) + { + builder.Services.AddGrpcClient( + engine.Type, + o => o.Address = new Uri(engine.Address) + ); + builder.Services.AddGrpcClient( + $"{engine.Type}-Health", + o => o.Address = new Uri(engine.Address) + ); + builder.Services.AddHealthChecks().AddCheck(engine.Type); + } + + return builder; + } +} diff --git a/src/Serval/src/Serval.Assessment/Consumers/DataFileDeletedConsumer.cs b/src/Serval/src/Serval.Assessment/Consumers/DataFileDeletedConsumer.cs new file mode 100644 index 00000000..11dde5b4 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Consumers/DataFileDeletedConsumer.cs @@ -0,0 +1,11 @@ +namespace Serval.Assessment.Consumers; + +public class DataFileDeletedConsumer(IEngineService engineService) : IConsumer +{ + private readonly IEngineService _engineService = engineService; + + public Task Consume(ConsumeContext context) + { + return _engineService.DeleteAllCorpusFilesAsync(context.Message.DataFileId, context.CancellationToken); + } +} diff --git a/src/Serval/src/Serval.Assessment/Contracts/AssessmentCorpusConfigDto.cs b/src/Serval/src/Serval.Assessment/Contracts/AssessmentCorpusConfigDto.cs new file mode 100644 index 00000000..562d3568 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Contracts/AssessmentCorpusConfigDto.cs @@ -0,0 +1,19 @@ +namespace Serval.Assessment.Contracts; + +public record AssessmentCorpusConfigDto +{ + /// + /// The corpus name. + /// + public string? Name { get; init; } + + /// + /// The language tag. + /// + public required string Language { get; init; } + + /// + /// The corpus files. + /// + public required IReadOnlyList Files { get; init; } +} diff --git a/src/Serval/src/Serval.Assessment/Contracts/AssessmentCorpusDto.cs b/src/Serval/src/Serval.Assessment/Contracts/AssessmentCorpusDto.cs new file mode 100644 index 00000000..7be13e2b --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Contracts/AssessmentCorpusDto.cs @@ -0,0 +1,9 @@ +namespace Serval.Assessment.Contracts; + +public record AssessmentCorpusDto +{ + public required string Url { get; init; } + public string? Name { get; init; } + public required string Language { get; init; } + public required IReadOnlyList Files { get; init; } +} diff --git a/src/Serval/src/Serval.Assessment/Contracts/AssessmentCorpusFileConfigDto.cs b/src/Serval/src/Serval.Assessment/Contracts/AssessmentCorpusFileConfigDto.cs new file mode 100644 index 00000000..d539d562 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Contracts/AssessmentCorpusFileConfigDto.cs @@ -0,0 +1,8 @@ +namespace Serval.Assessment.Contracts; + +public record AssessmentCorpusFileConfigDto +{ + public required string FileId { get; init; } + + public string? TextId { get; init; } +} diff --git a/src/Serval/src/Serval.Assessment/Contracts/AssessmentCorpusFileDto.cs b/src/Serval/src/Serval.Assessment/Contracts/AssessmentCorpusFileDto.cs new file mode 100644 index 00000000..a9b57925 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Contracts/AssessmentCorpusFileDto.cs @@ -0,0 +1,7 @@ +namespace Serval.Assessment.Contracts; + +public record AssessmentCorpusFileDto +{ + public required ResourceLinkDto File { get; init; } + public string? TextId { get; init; } +} diff --git a/src/Serval/src/Serval.Assessment/Contracts/AssessmentEngineConfigDto.cs b/src/Serval/src/Serval.Assessment/Contracts/AssessmentEngineConfigDto.cs new file mode 100644 index 00000000..81851b79 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Contracts/AssessmentEngineConfigDto.cs @@ -0,0 +1,24 @@ +namespace Serval.Assessment.Contracts; + +public record AssessmentEngineConfigDto +{ + /// + /// The assessment engine name. + /// + public string? Name { get; init; } + + /// + /// The assessment engine type. + /// + public required string Type { get; init; } + + /// + /// The corpus. + /// + public required AssessmentCorpusConfigDto Corpus { get; init; } + + /// + /// The reference corpus. + /// + public AssessmentCorpusConfigDto? ReferenceCorpus { get; init; } +} diff --git a/src/Serval/src/Serval.Assessment/Contracts/AssessmentEngineDto.cs b/src/Serval/src/Serval.Assessment/Contracts/AssessmentEngineDto.cs new file mode 100644 index 00000000..aa3951f9 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Contracts/AssessmentEngineDto.cs @@ -0,0 +1,11 @@ +namespace Serval.Assessment.Contracts; + +public record AssessmentEngineDto +{ + public required string Id { get; init; } + public required string Url { get; init; } + public string? Name { get; init; } + public required string Type { get; init; } + public required AssessmentCorpusDto Corpus { get; init; } + public AssessmentCorpusDto? ReferenceCorpus { get; init; } +} diff --git a/src/Serval/src/Serval.Assessment/Contracts/AssessmentJobConfigDto.cs b/src/Serval/src/Serval.Assessment/Contracts/AssessmentJobConfigDto.cs new file mode 100644 index 00000000..c5e6c276 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Contracts/AssessmentJobConfigDto.cs @@ -0,0 +1,15 @@ +namespace Serval.Assessment.Contracts; + +public record AssessmentJobConfigDto +{ + public string? Name { get; init; } + public IReadOnlyList? TextIds { get; init; } + public string? ScriptureRange { get; init; } + + /// + /// { + /// "property" : "value" + /// } + /// + public object? Options { get; init; } +} diff --git a/src/Serval/src/Serval.Assessment/Contracts/AssessmentJobDto.cs b/src/Serval/src/Serval.Assessment/Contracts/AssessmentJobDto.cs new file mode 100644 index 00000000..296c6b32 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Contracts/AssessmentJobDto.cs @@ -0,0 +1,27 @@ +namespace Serval.Assessment.Contracts; + +public record AssessmentJobDto +{ + public required string Id { get; init; } + public required string Url { get; init; } + public required int Revision { get; init; } + public string? Name { get; init; } + public required ResourceLinkDto Engine { get; init; } + public IReadOnlyList? TextIds { get; init; } + public string? ScriptureRange { get; init; } + public double? PercentCompleted { get; init; } + public string? Message { get; init; } + + /// + /// The current job state. + /// + public required JobState State { get; init; } + public DateTime? DateFinished { get; init; } + + /// + /// { + /// "property" : "value" + /// } + /// + public object? Options { get; init; } +} diff --git a/src/Serval/src/Serval.Assessment/Contracts/AssessmentResultDto.cs b/src/Serval/src/Serval.Assessment/Contracts/AssessmentResultDto.cs new file mode 100644 index 00000000..c1289da1 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Contracts/AssessmentResultDto.cs @@ -0,0 +1,9 @@ +namespace Serval.Assessment.Contracts; + +public record AssessmentResultDto +{ + public required string TextId { get; init; } + public required string Ref { get; init; } + public double? Score { get; init; } + public string? Description { get; init; } +} diff --git a/src/Serval/src/Serval.Assessment/Controllers/AssessmentEnginesController.cs b/src/Serval/src/Serval.Assessment/Controllers/AssessmentEnginesController.cs new file mode 100644 index 00000000..3a139a36 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Controllers/AssessmentEnginesController.cs @@ -0,0 +1,673 @@ +namespace Serval.Assessment.Controllers; + +[ApiVersion(1.0)] +[Route("api/v{version:apiVersion}/assessment/engines")] +[OpenApiTag("Assessment")] +[FeatureGate("Assessment")] +public class AssessmentEnginesController( + IAuthorizationService authService, + IEngineService engineService, + IJobService jobService, + IResultService resultService, + IOptionsMonitor apiOptions, + IUrlService urlService +) : ServalControllerBase(authService) +{ + private static readonly JsonSerializerOptions ObjectJsonSerializerOptions = + new() { Converters = { new ObjectToInferredTypesConverter() } }; + + private readonly IEngineService _engineService = engineService; + private readonly IJobService _jobService = jobService; + private readonly IResultService _resultService = resultService; + private readonly IOptionsMonitor _apiOptions = apiOptions; + private readonly IUrlService _urlService = urlService; + + /// + /// Get all assessment engines. + /// + /// + /// The engines + /// The client is not authenticated. + /// The authenticated client cannot perform the operation. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadAssessmentEngines)] + [HttpGet] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> GetAllAsync(CancellationToken cancellationToken) + { + return (await _engineService.GetAllAsync(Owner, cancellationToken)).Select(Map); + } + + /// + /// Get an assessment engine. + /// + /// The engine id + /// + /// The engine + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the engine. + /// The engine does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. + + [Authorize(Scopes.ReadAssessmentEngines)] + [HttpGet("{id}", Name = Endpoints.GetAssessmentEngine)] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> GetAsync( + [NotNull] string id, + CancellationToken cancellationToken + ) + { + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); + return Ok(Map(engine)); + } + + /// + /// Create a new assessment engine. + /// + /// The engine configuration (see above) + /// + /// The new engine + /// Bad request. Is the engine type correct? + /// The client is not authenticated. + /// The authenticated client cannot perform the operation. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.CreateAssessmentEngines)] + [HttpPost] + [ProducesResponseType(StatusCodes.Status201Created)] + [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> CreateAsync( + [FromBody] AssessmentEngineConfigDto engineConfig, + [FromServices] IRequestClient getDataFileClient, + CancellationToken cancellationToken + ) + { + Engine engine = await MapAsync(getDataFileClient, engineConfig, cancellationToken); + Engine updatedEngine = await _engineService.CreateAsync(engine, cancellationToken); + AssessmentEngineDto dto = Map(updatedEngine); + return Created(dto.Url, dto); + } + + /// + /// Delete an assessment engine. + /// + /// The engine id + /// + /// The engine was successfully deleted. + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the engine. + /// The engine does not exist and therefore cannot be deleted. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.DeleteAssessmentEngines)] + [HttpDelete("{id}")] + [ProducesResponseType(typeof(void), StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task DeleteAsync([NotNull] string id, CancellationToken cancellationToken) + { + await AuthorizeAsync(id, cancellationToken); + await _engineService.DeleteAsync(id, cancellationToken); + return Ok(); + } + + /// + /// Get the configuration of the corpus for an assessment engine. + /// + /// The assessment engine id + /// + /// The corpus configuration + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the assessment engine. + /// The engine or corpus does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadAssessmentEngines)] + [HttpGet("{id}/corpus", Name = Endpoints.GetAssessmentCorpus)] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> GetCorpusAsync( + [NotNull] string id, + CancellationToken cancellationToken + ) + { + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); + return Ok(Map(id, Endpoints.GetAssessmentCorpus, engine.Corpus)); + } + + /// + /// Replace the corpus configuration for an assessment engine. + /// + /// The assessment engine id + /// The new corpus configuration + /// The data file client + /// + /// The corpus configuration + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the assessment engine. + /// The engine or corpus does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.UpdateAssessmentEngines)] + [HttpPut("{id}/corpus")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> ReplaceCorpusAsync( + [NotNull] string id, + [FromBody] AssessmentCorpusConfigDto corpusConfig, + [FromServices] IRequestClient getDataFileClient, + CancellationToken cancellationToken + ) + { + await AuthorizeAsync(id, cancellationToken); + Corpus newCorpus = await MapAsync(getDataFileClient, corpusConfig, cancellationToken); + Corpus updatedCorpus = await _engineService.ReplaceCorpusAsync(id, newCorpus, cancellationToken); + return Ok(Map(id, Endpoints.GetAssessmentCorpus, updatedCorpus)); + } + + /// + /// Get the configuration of the reference corpus for an assessment engine. + /// + /// The assessment engine id + /// + /// The corpus configuration + /// The engine does not have a reference corpus. + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the assessment engine. + /// The engine or corpus does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadTranslationEngines)] + [HttpGet("{id}/reference-corpus", Name = Endpoints.GetAssessmentReferenceCorpus)] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status204NoContent)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> GetReferenceCorpusAsync( + [NotNull] string id, + CancellationToken cancellationToken + ) + { + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); + if (engine.ReferenceCorpus is null) + return NoContent(); + return Ok(Map(id, Endpoints.GetAssessmentReferenceCorpus, engine.ReferenceCorpus)); + } + + /// + /// Replace the reference corpus configuration for an assessment engine. + /// + /// The assessment engine id + /// The corpus configuration + /// The data file client + /// + /// The new corpus configuration + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the assessment engine. + /// The engine or corpus does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.UpdateAssessmentEngines)] + [HttpPut("{id}/reference-corpus")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> ReplaceReferenceCorpusAsync( + [NotNull] string id, + [FromBody] AssessmentCorpusConfigDto corpusConfig, + [FromServices] IRequestClient getDataFileClient, + CancellationToken cancellationToken + ) + { + await AuthorizeAsync(id, cancellationToken); + Corpus newCorpus = await MapAsync(getDataFileClient, corpusConfig, cancellationToken); + Corpus updatedCorpus = await _engineService.ReplaceReferenceCorpusAsync(id, newCorpus, cancellationToken); + return Ok(Map(id, Endpoints.GetAssessmentReferenceCorpus, updatedCorpus)); + } + + /// + /// Get all assessment jobs. + /// + /// The engine id + /// + /// The jobs + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the engine. + /// The engine does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadAssessmentEngines)] + [HttpGet("{id}/jobs")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task>> GetAllJobsAsync( + [NotNull] string id, + CancellationToken cancellationToken + ) + { + await AuthorizeAsync(id, cancellationToken); + return Ok((await _jobService.GetAllAsync(id, cancellationToken)).Select(Map)); + } + + /// + /// Get an assessment job. + /// + /// + /// If the `minRevision` is not defined, the current job, at whatever state it is, + /// will be immediately returned. If `minRevision` is defined, Serval will wait for + /// up to 40 seconds for the engine to job to the `minRevision` specified, else + /// will timeout. + /// A use case is to actively query the state of the current job, where the subsequent + /// request sets the `minRevision` to the returned `revision` + 1 and timeouts are handled gracefully. + /// This method should use request throttling. + /// Note: Within the returned job, percentCompleted is a value between 0 and 1. + /// + /// The engine id + /// The job id + /// The minimum revision + /// + /// The job + /// The client is not authenticated. + /// The authenticated client does not own the engine. + /// The engine or job does not exist. + /// The long polling request timed out. This is expected behavior if you're using long-polling with the minRevision strategy specified in the docs. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadAssessmentEngines)] + [HttpGet("{id}/jobs/{jobId}", Name = Endpoints.GetAssessmentJob)] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status408RequestTimeout)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> GetJobAsync( + [NotNull] string id, + [NotNull] string jobId, + [FromQuery] long? minRevision, + CancellationToken cancellationToken + ) + { + await AuthorizeAsync(id, cancellationToken); + if (minRevision != null) + { + EntityChange change = await TaskEx.Timeout( + ct => _jobService.GetNewerRevisionAsync(jobId, minRevision.Value, ct), + _apiOptions.CurrentValue.LongPollTimeout, + cancellationToken + ); + return change.Type switch + { + EntityChangeType.None => StatusCode(StatusCodes.Status408RequestTimeout), + EntityChangeType.Delete => NotFound(), + _ => Ok(Map(change.Entity!)), + }; + } + else + { + Job job = await _jobService.GetAsync(jobId, cancellationToken); + return Ok(Map(job)); + } + } + + /// + /// Start an assessment job. + /// + /// The engine id + /// The job config (see remarks) + /// + /// The new job + /// The job configuration was invalid. + /// The client is not authenticated. + /// The authenticated client does not own the engine. + /// The engine does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.UpdateAssessmentEngines)] + [HttpPost("{id}/jobs")] + [ProducesResponseType(StatusCodes.Status201Created)] + [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> StartJobAsync( + [NotNull] string id, + [FromBody] AssessmentJobConfigDto jobConfig, + CancellationToken cancellationToken + ) + { + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); + Job job = Map(engine, jobConfig); + await _engineService.StartJobAsync(job, cancellationToken); + + AssessmentJobDto dto = Map(job); + return Created(dto.Url, dto); + } + + /// + /// Delete an assessment job. + /// + /// The engine id + /// + /// The job was successfully deleted. + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the engine. + /// The engine does not exist and therefore cannot be deleted. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.DeleteAssessmentEngines)] + [HttpDelete("{id}/jobs/{jobId}")] + [ProducesResponseType(typeof(void), StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task DeleteJobAsync( + [NotNull] string id, + [NotNull] string jobId, + CancellationToken cancellationToken + ) + { + await AuthorizeAsync(id, cancellationToken); + await _jobService.DeleteAsync(jobId, cancellationToken); + return Ok(); + } + + /// + /// Cancel an assessment job. + /// + /// + /// + /// The engine id + /// The job id + /// + /// The job was cancelled successfully. + /// The job is not active. + /// The client is not authenticated. + /// The authenticated client does not own the engine. + /// The engine does not exist. + /// The engine does not support canceling jobs. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.UpdateAssessmentEngines)] + [HttpPost("{id}/jobs/{jobId}/cancel")] + [ProducesResponseType(typeof(void), StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status204NoContent)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status405MethodNotAllowed)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task CancelJobAsync( + [NotNull] string id, + [NotNull] string jobId, + CancellationToken cancellationToken + ) + { + await AuthorizeAsync(id, cancellationToken); + if (!await _engineService.CancelJobAsync(id, jobId, cancellationToken)) + return NoContent(); + return Ok(); + } + + /// + /// Get all results of an assessment job. + /// + /// The engine id + /// The job id + /// The text id (optional) + /// + /// The results + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the engine. + /// The engine or corpus does not exist. + /// The engine needs to be built first. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadAssessmentEngines)] + [HttpGet("{id}/jobs/{jobId}/results")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status409Conflict)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task>> GetAllResultsAsync( + [NotNull] string id, + [NotNull] string jobId, + [FromQuery] string? textId, + CancellationToken cancellationToken + ) + { + await AuthorizeAsync(id, cancellationToken); + + IEnumerable results = await _resultService.GetAllAsync(id, jobId, textId, cancellationToken); + return Ok(results.Select(Map)); + } + + /// + /// Get all results for the specified text of an assessment job. + /// + /// The engine id + /// The job id + /// The text id + /// + /// The results + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the engine. + /// The engine or corpus does not exist. + /// The engine needs to be built first. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadAssessmentEngines)] + [HttpGet("{id}/jobs/{jobId}/results/{textId}")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status409Conflict)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task>> GetResultsByTextIdAsync( + [NotNull] string id, + [NotNull] string jobId, + [NotNull] string textId, + CancellationToken cancellationToken + ) + { + await AuthorizeAsync(id, cancellationToken); + + IEnumerable results = await _resultService.GetAllAsync(id, jobId, textId, cancellationToken); + return Ok(results.Select(Map)); + } + + private async Task AuthorizeAsync(string id, CancellationToken cancellationToken) + { + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); + } + + private AssessmentEngineDto Map(Engine source) + { + return new AssessmentEngineDto + { + Id = source.Id, + Url = _urlService.GetUrl(Endpoints.GetAssessmentEngine, new { id = source.Id }), + Name = source.Name, + Type = source.Type.ToKebabCase(), + Corpus = Map(source.Id, Endpoints.GetAssessmentCorpus, source.Corpus), + ReferenceCorpus = source.ReferenceCorpus is null + ? null + : Map(source.Id, Endpoints.GetAssessmentReferenceCorpus, source.ReferenceCorpus) + }; + } + + private async Task MapAsync( + IRequestClient getDataFileClient, + AssessmentEngineConfigDto source, + CancellationToken cancellationToken + ) + { + return new Engine + { + Name = source.Name, + Type = source.Type.ToPascalCase(), + Owner = Owner, + Corpus = await MapAsync(getDataFileClient, source.Corpus, cancellationToken), + ReferenceCorpus = source.ReferenceCorpus is null + ? null + : await MapAsync(getDataFileClient, source.ReferenceCorpus, cancellationToken) + }; + } + + private static AssessmentResultDto Map(Result source) + { + return new AssessmentResultDto + { + TextId = source.TextId, + Ref = source.Ref, + Score = source.Score, + Description = source.Description + }; + } + + private AssessmentJobDto Map(Job source) + { + return new AssessmentJobDto + { + Id = source.Id, + Url = _urlService.GetUrl(Endpoints.GetAssessmentJob, new { id = source.EngineRef, jobId = source.Id }), + Revision = source.Revision, + Name = source.Name, + Engine = new ResourceLinkDto + { + Id = source.EngineRef, + Url = _urlService.GetUrl(Endpoints.GetAssessmentEngine, new { id = source.EngineRef }) + }, + TextIds = source.TextIds, + ScriptureRange = source.ScriptureRange, + PercentCompleted = source.PercentCompleted, + Message = source.Message, + State = source.State, + DateFinished = source.DateFinished, + Options = source.Options + }; + } + + private static Job Map(Engine engine, AssessmentJobConfigDto source) + { + if (source.TextIds is not null && source.ScriptureRange is not null) + throw new InvalidOperationException("Set at most one of TextIds and ScriptureRange."); + + return new Job + { + EngineRef = engine.Id, + Name = source.Name, + TextIds = source.TextIds?.ToList(), + ScriptureRange = source.ScriptureRange, + Options = Map(source.Options) + }; + } + + private static Dictionary? Map(object? source) + { + try + { + return JsonSerializer.Deserialize>( + source?.ToString() ?? "{}", + ObjectJsonSerializerOptions + ); + } + catch (Exception e) + { + throw new InvalidOperationException($"Unable to parse field 'options' : {e.Message}", e); + } + } + + private AssessmentCorpusDto Map(string engineId, string getCorpusEndpointName, Corpus source) + { + return new AssessmentCorpusDto + { + Url = _urlService.GetUrl(getCorpusEndpointName, new { id = engineId }), + Name = source.Name, + Language = source.Language, + Files = source.Files.Select(Map).ToList() + }; + } + + private AssessmentCorpusFileDto Map(CorpusFile source) + { + return new AssessmentCorpusFileDto + { + File = new ResourceLinkDto + { + Id = source.Id, + Url = _urlService.GetUrl(Endpoints.GetDataFile, new { id = source.Id }) + }, + TextId = source.TextId + }; + } + + private async Task MapAsync( + IRequestClient getDataFileClient, + AssessmentCorpusConfigDto source, + CancellationToken cancellationToken + ) + { + return new Corpus + { + Name = source.Name, + Language = source.Language, + Files = await MapAsync(getDataFileClient, source.Files, cancellationToken) + }; + } + + private async Task> MapAsync( + IRequestClient getDataFileClient, + IEnumerable fileConfigs, + CancellationToken cancellationToken + ) + { + var files = new List(); + foreach (AssessmentCorpusFileConfigDto fileConfig in fileConfigs) + { + Response response = await getDataFileClient.GetResponse< + DataFileResult, + DataFileNotFound + >(new GetDataFile { DataFileId = fileConfig.FileId, Owner = Owner }, cancellationToken); + if (response.Is(out Response? result)) + { + files.Add( + new CorpusFile + { + Id = fileConfig.FileId, + Filename = result.Message.Filename, + TextId = fileConfig.TextId ?? result.Message.Name, + Format = result.Message.Format + } + ); + } + else if (response.Is(out Response? _)) + { + throw new InvalidOperationException($"The data file {fileConfig.FileId} cannot be found."); + } + } + return files; + } +} diff --git a/src/Serval/src/Serval.Assessment/Models/Corpus.cs b/src/Serval/src/Serval.Assessment/Models/Corpus.cs new file mode 100644 index 00000000..33ef0981 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Models/Corpus.cs @@ -0,0 +1,8 @@ +namespace Serval.Assessment.Models; + +public record Corpus +{ + public string? Name { get; init; } + public required string Language { get; init; } + public required IReadOnlyList Files { get; init; } +} diff --git a/src/Serval/src/Serval.Assessment/Models/CorpusFile.cs b/src/Serval/src/Serval.Assessment/Models/CorpusFile.cs new file mode 100644 index 00000000..fa491558 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Models/CorpusFile.cs @@ -0,0 +1,9 @@ +namespace Serval.Assessment.Models; + +public record CorpusFile +{ + public required string Id { get; init; } + public required string Filename { get; init; } + public required FileFormat Format { get; init; } + public required string TextId { get; init; } +} diff --git a/src/Serval/src/Serval.Assessment/Models/Engine.cs b/src/Serval/src/Serval.Assessment/Models/Engine.cs new file mode 100644 index 00000000..337b8875 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Models/Engine.cs @@ -0,0 +1,12 @@ +namespace Serval.Assessment.Models; + +public record Engine : IOwnedEntity +{ + public string Id { get; set; } = ""; + public int Revision { get; set; } = 1; + public required string Owner { get; init; } + public string? Name { get; init; } + public required string Type { get; init; } + public required Corpus Corpus { get; init; } + public Corpus? ReferenceCorpus { get; init; } +} diff --git a/src/Serval/src/Serval.Assessment/Models/Job.cs b/src/Serval/src/Serval.Assessment/Models/Job.cs new file mode 100644 index 00000000..c1863e46 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Models/Job.cs @@ -0,0 +1,16 @@ +namespace Serval.Assessment.Models; + +public record Job : IEntity +{ + public string Id { get; set; } = ""; + public int Revision { get; set; } = 1; + public string? Name { get; init; } + public required string EngineRef { get; init; } + public IReadOnlyList? TextIds { get; set; } + public string? ScriptureRange { get; set; } + public double? PercentCompleted { get; init; } + public string? Message { get; init; } + public JobState State { get; init; } = JobState.Pending; + public DateTime? DateFinished { get; init; } + public IReadOnlyDictionary? Options { get; init; } +} diff --git a/src/Serval/src/Serval.Assessment/Models/Result.cs b/src/Serval/src/Serval.Assessment/Models/Result.cs new file mode 100644 index 00000000..b346f222 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Models/Result.cs @@ -0,0 +1,13 @@ +namespace Serval.Assessment.Models; + +public record Result : IEntity +{ + public string Id { get; set; } = ""; + public int Revision { get; set; } = 1; + public required string EngineRef { get; init; } + public required string JobRef { get; init; } + public required string TextId { get; init; } + public required string Ref { get; init; } + public double? Score { get; init; } + public string? Description { get; init; } +} diff --git a/src/Serval/src/Serval.Assessment/Serval.Assessment.csproj b/src/Serval/src/Serval.Assessment/Serval.Assessment.csproj new file mode 100644 index 00000000..d43943fa --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Serval.Assessment.csproj @@ -0,0 +1,26 @@ + + + + net8.0 + enable + enable + true + true + true + $(NoWarn);CS1591;CS1573 + + + + + + + + + + + + + + + + diff --git a/src/Serval/src/Serval.Assessment/Services/AssessmentPlatformServiceV1.cs b/src/Serval/src/Serval.Assessment/Services/AssessmentPlatformServiceV1.cs new file mode 100644 index 00000000..61bd88ce --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Services/AssessmentPlatformServiceV1.cs @@ -0,0 +1,257 @@ +using Google.Protobuf.WellKnownTypes; +using Serval.Assessment.V1; + +namespace Serval.Assessment.Services; + +public class AssessmentPlatformServiceV1( + IRepository jobs, + IRepository engines, + IRepository results, + IDataAccessContext dataAccessContext, + IPublishEndpoint publishEndpoint +) : AssessmentPlatformApi.AssessmentPlatformApiBase +{ + private const int ResultInsertBatchSize = 128; + private static readonly Empty Empty = new(); + + private readonly IRepository _jobs = jobs; + private readonly IRepository _engines = engines; + private readonly IRepository _results = results; + private readonly IDataAccessContext _dataAccessContext = dataAccessContext; + private readonly IPublishEndpoint _publishEndpoint = publishEndpoint; + + public override async Task JobStarted(JobStartedRequest request, ServerCallContext context) + { + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + Job? job = await _jobs.UpdateAsync( + request.JobId, + u => u.Set(b => b.State, JobState.Active), + cancellationToken: ct + ); + if (job is null) + throw new RpcException(new Status(StatusCode.NotFound, "The job does not exist.")); + + Engine? engine = await _engines.GetAsync(job.EngineRef, cancellationToken: ct); + if (engine is null) + throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); + + await _publishEndpoint.Publish( + new AssessmentJobStarted + { + JobId = job.Id, + EngineId = engine.Id, + Owner = engine.Owner + }, + ct + ); + }, + cancellationToken: context.CancellationToken + ); + return Empty; + } + + public override async Task JobCompleted(JobCompletedRequest request, ServerCallContext context) + { + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + Job? job = await _jobs.UpdateAsync( + request.JobId, + u => + u.Set(b => b.State, JobState.Completed) + .Set(b => b.Message, "Completed") + .Set(b => b.DateFinished, DateTime.UtcNow), + cancellationToken: ct + ); + if (job is null) + throw new RpcException(new Status(StatusCode.NotFound, "The job does not exist.")); + + Engine? engine = await _engines.GetAsync(job.EngineRef, cancellationToken: ct); + if (engine is null) + throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); + + await _publishEndpoint.Publish( + new AssessmentJobFinished + { + JobId = job.Id, + EngineId = engine.Id, + Owner = engine.Owner, + JobState = job.State, + Message = job.Message!, + DateFinished = job.DateFinished!.Value + }, + ct + ); + }, + cancellationToken: context.CancellationToken + ); + + return Empty; + } + + public override async Task JobCanceled(JobCanceledRequest request, ServerCallContext context) + { + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + Job? job = await _jobs.UpdateAsync( + request.JobId, + u => + { + u.Set(j => j.Message, "Canceled"); + u.Set(j => j.DateFinished, DateTime.UtcNow); + u.Set(j => j.State, JobState.Canceled); + }, + cancellationToken: ct + ); + if (job is null) + throw new RpcException(new Status(StatusCode.NotFound, "The job does not exist.")); + + Engine? engine = await _engines.GetAsync(job.EngineRef, cancellationToken: ct); + if (engine is null) + throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); + + await _publishEndpoint.Publish( + new AssessmentJobFinished + { + JobId = job.Id, + EngineId = engine.Id, + Owner = engine.Owner, + JobState = job.State, + Message = job.Message!, + DateFinished = job.DateFinished!.Value + }, + ct + ); + }, + cancellationToken: context.CancellationToken + ); + + return Empty; + } + + public override async Task JobFaulted(JobFaultedRequest request, ServerCallContext context) + { + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + Job? job = await _jobs.UpdateAsync( + request.JobId, + u => + { + u.Set(b => b.State, JobState.Faulted); + u.Set(b => b.Message, request.Message); + u.Set(b => b.DateFinished, DateTime.UtcNow); + }, + cancellationToken: ct + ); + if (job is null) + throw new RpcException(new Status(StatusCode.NotFound, "The job does not exist.")); + + Engine? engine = await _engines.GetAsync(job.EngineRef, cancellationToken: ct); + if (engine is null) + throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); + + await _publishEndpoint.Publish( + new AssessmentJobFinished + { + JobId = job.Id, + EngineId = engine.Id, + Owner = engine.Owner, + JobState = job.State, + Message = job.Message!, + DateFinished = job.DateFinished!.Value + }, + ct + ); + }, + cancellationToken: context.CancellationToken + ); + + return Empty; + } + + public override async Task JobRestarting(JobRestartingRequest request, ServerCallContext context) + { + Job? job = await _jobs.UpdateAsync( + request.JobId, + u => + { + u.Set(j => j.Message, "Restarting"); + u.Unset(j => j.PercentCompleted); + u.Set(j => j.State, JobState.Pending); + }, + cancellationToken: context.CancellationToken + ); + if (job is null) + throw new RpcException(new Status(StatusCode.NotFound, "The job does not exist.")); + + return Empty; + } + + public override async Task UpdateJobStatus(UpdateJobStatusRequest request, ServerCallContext context) + { + await _jobs.UpdateAsync( + j => j.Id == request.JobId && (j.State == JobState.Active || j.State == JobState.Pending), + u => + { + if (request.HasPercentCompleted) + { + u.Set( + j => j.PercentCompleted, + Math.Round(request.PercentCompleted, 4, MidpointRounding.AwayFromZero) + ); + } + if (request.HasMessage) + u.Set(j => j.Message, request.Message); + }, + cancellationToken: context.CancellationToken + ); + + return Empty; + } + + public override async Task InsertResults( + IAsyncStreamReader requestStream, + ServerCallContext context + ) + { + string jobId = ""; + string engineId = ""; + List batch = []; + await foreach (InsertResultsRequest request in requestStream.ReadAllAsync(context.CancellationToken)) + { + if (jobId != request.JobId) + { + Job? job = await _jobs.GetAsync(request.JobId, context.CancellationToken); + if (job is null) + throw new RpcException(new Status(StatusCode.NotFound, "The job does not exist.")); + engineId = job.EngineRef; + jobId = request.JobId; + } + + batch.Add( + new Result + { + EngineRef = engineId, + JobRef = request.JobId, + TextId = request.TextId, + Ref = request.Ref, + Score = request.HasScore ? request.Score : null, + Description = request.HasDescription ? request.Description : null + } + ); + if (batch.Count == ResultInsertBatchSize) + { + await _results.InsertAllAsync(batch, context.CancellationToken); + batch.Clear(); + } + } + if (batch.Count > 0) + await _results.InsertAllAsync(batch, CancellationToken.None); + + return Empty; + } +} diff --git a/src/Serval/src/Serval.Assessment/Services/EngineService.cs b/src/Serval/src/Serval.Assessment/Services/EngineService.cs new file mode 100644 index 00000000..01ced93a --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Services/EngineService.cs @@ -0,0 +1,239 @@ +using Serval.Assessment.V1; + +namespace Serval.Assessment.Services; + +public class EngineService( + IRepository engines, + IRepository jobs, + IRepository results, + GrpcClientFactory grpcClientFactory, + IOptionsMonitor dataFileOptions, + IDataAccessContext dataAccessContext, + ILoggerFactory loggerFactory, + IScriptureDataFileService scriptureDataFileService +) : OwnedEntityServiceBase(engines), IEngineService +{ + private readonly IRepository _jobs = jobs; + private readonly IRepository _results = results; + private readonly GrpcClientFactory _grpcClientFactory = grpcClientFactory; + private readonly IOptionsMonitor _dataFileOptions = dataFileOptions; + private readonly IDataAccessContext _dataAccessContext = dataAccessContext; + private readonly ILogger _logger = loggerFactory.CreateLogger(); + private readonly IScriptureDataFileService _scriptureDataFileService = scriptureDataFileService; + + public override async Task CreateAsync(Engine engine, CancellationToken cancellationToken = default) + { + try + { + await Entities.InsertAsync(engine, cancellationToken); + var client = _grpcClientFactory.CreateClient(engine.Type); + if (client is null) + throw new InvalidOperationException($"'{engine.Type}' is an invalid engine type."); + var request = new CreateRequest { EngineType = engine.Type, EngineId = engine.Id, }; + if (engine.Name is not null) + request.EngineName = engine.Name; + await client.CreateAsync(request, cancellationToken: cancellationToken); + } + catch + { + await Entities.DeleteAsync(engine, CancellationToken.None); + throw; + } + return engine; + } + + public override async Task DeleteAsync(string id, CancellationToken cancellationToken = default) + { + Engine? engine = await Entities.GetAsync(id, cancellationToken); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{id}'."); + + var client = _grpcClientFactory.CreateClient(engine.Type); + await client.DeleteAsync( + new DeleteRequest { EngineType = engine.Type, EngineId = engine.Id }, + cancellationToken: cancellationToken + ); + + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + await Entities.DeleteAsync(id, ct); + await _jobs.DeleteAllAsync(b => b.EngineRef == id, ct); + await _results.DeleteAllAsync(r => r.EngineRef == id, ct); + }, + CancellationToken.None + ); + } + + public async Task ReplaceCorpusAsync( + string id, + Models.Corpus corpus, + CancellationToken cancellationToken = default + ) + { + Engine? engine = await Entities.UpdateAsync( + id, + u => u.Set(e => e.Corpus, corpus), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{id}'."); + return engine.Corpus; + } + + public async Task ReplaceReferenceCorpusAsync( + string id, + Models.Corpus referenceCorpus, + CancellationToken cancellationToken = default + ) + { + Engine? engine = await Entities.UpdateAsync( + id, + u => u.Set(e => e.ReferenceCorpus, referenceCorpus), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{id}'."); + return engine.ReferenceCorpus!; + } + + public async Task StartJobAsync(Job job, CancellationToken cancellationToken = default) + { + Engine engine = await GetAsync(job.EngineRef, cancellationToken); + await _jobs.InsertAsync(job, cancellationToken); + + try + { + AssessmentEngineApi.AssessmentEngineApiClient client = + _grpcClientFactory.CreateClient(engine.Type); + var request = new StartJobRequest + { + EngineType = engine.Type, + EngineId = engine.Id, + JobId = job.Id, + Options = JsonSerializer.Serialize(job.Options), + Corpus = Map(engine.Corpus), + IncludeAll = job.TextIds is null || job.TextIds.Count == 0 + }; + if (engine.ReferenceCorpus is not null) + request.ReferenceCorpus = Map(engine.ReferenceCorpus); + if (job.TextIds is not null) + request.IncludeTextIds.Add(job.TextIds); + if (job.ScriptureRange is not null) + { + if ( + engine.Corpus.Files.Count > 1 + || engine.Corpus.Files[0].Format != Shared.Contracts.FileFormat.Paratext + ) + { + throw new InvalidOperationException($"The engine is not compatible with using a scripture range."); + } + + try + { + ScrVers versification = _scriptureDataFileService + .GetParatextProjectSettings(request.Corpus.Files[0].Location) + .Versification; + Dictionary chapters = ScriptureRangeParser + .GetChapters(job.ScriptureRange, versification) + .ToDictionary(kvp => kvp.Key, kvp => new ScriptureChapters { Chapters = { kvp.Value } }); + request.IncludeChapters.Add(chapters); + } + catch (ArgumentException ae) + { + throw new InvalidOperationException( + $"The scripture range {job.ScriptureRange} is not valid: {ae.Message}" + ); + } + } + + // Log the job request summary + try + { + var jobRequestSummary = (JsonObject)JsonNode.Parse(JsonSerializer.Serialize(request))!; + // correct job options parsing + jobRequestSummary.Remove("Options"); + try + { + jobRequestSummary.Add("Options", JsonNode.Parse(request.Options)); + } + catch (JsonException) + { + jobRequestSummary.Add("Options", "Job \"Options\" failed parsing: " + (request.Options ?? "null")); + } + jobRequestSummary.Add("Event", "JobRequest"); + jobRequestSummary.Add("ClientId", engine.Owner); + _logger.LogInformation("{request}", jobRequestSummary.ToJsonString()); + } + catch (JsonException) + { + _logger.LogInformation("Error parsing job request summary."); + _logger.LogInformation("{request}", JsonSerializer.Serialize(request)); + } + + await client.StartJobAsync(request, cancellationToken: cancellationToken); + } + catch + { + await _jobs.DeleteAsync(job, CancellationToken.None); + throw; + } + } + + public async Task CancelJobAsync(string id, string jobId, CancellationToken cancellationToken = default) + { + Engine? engine = await GetAsync(id, cancellationToken); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{id}'."); + + AssessmentEngineApi.AssessmentEngineApiClient client = + _grpcClientFactory.CreateClient(engine.Type); + try + { + await client.CancelJobAsync( + new CancelJobRequest + { + EngineType = engine.Type, + EngineId = engine.Id, + JobId = jobId + }, + cancellationToken: cancellationToken + ); + } + catch (RpcException re) + { + if (re.StatusCode is StatusCode.Aborted) + return false; + throw; + } + return true; + } + + public Task DeleteAllCorpusFilesAsync(string dataFileId, CancellationToken cancellationToken = default) + { + return Entities.UpdateAllAsync( + e => e.Corpus.Files.Any(f => f.Id == dataFileId) || e.ReferenceCorpus!.Files.Any(f => f.Id == dataFileId), + u => + { + u.RemoveAll(e => e.Corpus.Files, f => f.Id == dataFileId); + u.RemoveAll(e => e.ReferenceCorpus!.Files, f => f.Id == dataFileId); + }, + cancellationToken + ); + } + + private V1.Corpus Map(Models.Corpus source) + { + return new V1.Corpus { Language = source.Language, Files = { source.Files.Select(Map) } }; + } + + private V1.CorpusFile Map(Models.CorpusFile source) + { + return new V1.CorpusFile + { + TextId = source.TextId, + Format = (V1.FileFormat)source.Format, + Location = Path.Combine(_dataFileOptions.CurrentValue.FilesDirectory, source.Filename) + }; + } +} diff --git a/src/Serval/src/Serval.Assessment/Services/IEngineService.cs b/src/Serval/src/Serval.Assessment/Services/IEngineService.cs new file mode 100644 index 00000000..4025d6f3 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Services/IEngineService.cs @@ -0,0 +1,22 @@ +namespace Serval.Assessment.Services; + +public interface IEngineService +{ + Task> GetAllAsync(string owner, CancellationToken cancellationToken = default); + Task GetAsync(string id, CancellationToken cancellationToken = default); + + Task CreateAsync(Engine engine, CancellationToken cancellationToken = default); + Task DeleteAsync(string id, CancellationToken cancellationToken = default); + + Task ReplaceCorpusAsync(string id, Corpus corpus, CancellationToken cancellationToken = default); + Task ReplaceReferenceCorpusAsync( + string id, + Corpus referenceCorpus, + CancellationToken cancellationToken = default + ); + + Task StartJobAsync(Job job, CancellationToken cancellationToken = default); + Task CancelJobAsync(string id, string jobId, CancellationToken cancellationToken = default); + + Task DeleteAllCorpusFilesAsync(string dataFileId, CancellationToken cancellationToken = default); +} diff --git a/src/Serval/src/Serval.Assessment/Services/IJobService.cs b/src/Serval/src/Serval.Assessment/Services/IJobService.cs new file mode 100644 index 00000000..5ceb6f83 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Services/IJobService.cs @@ -0,0 +1,13 @@ +namespace Serval.Assessment.Services; + +public interface IJobService +{ + Task> GetAllAsync(string engineId, CancellationToken cancellationToken = default); + Task DeleteAsync(string id, CancellationToken cancellationToken = default); + Task GetAsync(string id, CancellationToken cancellationToken = default); + Task> GetNewerRevisionAsync( + string id, + long minRevision, + CancellationToken cancellationToken = default + ); +} diff --git a/src/Serval/src/Serval.Assessment/Services/IResultService.cs b/src/Serval/src/Serval.Assessment/Services/IResultService.cs new file mode 100644 index 00000000..6c5c4a26 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Services/IResultService.cs @@ -0,0 +1,11 @@ +namespace Serval.Assessment.Services; + +public interface IResultService +{ + Task> GetAllAsync( + string engineId, + string jobId, + string? textId = null, + CancellationToken cancellationToken = default + ); +} diff --git a/src/Serval/src/Serval.Assessment/Services/JobService.cs b/src/Serval/src/Serval.Assessment/Services/JobService.cs new file mode 100644 index 00000000..9bb0bb8f --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Services/JobService.cs @@ -0,0 +1,62 @@ +namespace Serval.Assessment.Services; + +public class JobService(IDataAccessContext dataAccessContext, IRepository jobs, IRepository results) + : EntityServiceBase(jobs), + IJobService +{ + private readonly IDataAccessContext _dataAccessContext = dataAccessContext; + private readonly IRepository _results = results; + + public async Task> GetAllAsync(string engineId, CancellationToken cancellationToken = default) + { + return await Entities.GetAllAsync(e => e.EngineRef == engineId, cancellationToken); + } + + public override Task DeleteAsync(string id, CancellationToken cancellationToken = default) + { + return _dataAccessContext.WithTransactionAsync( + async ct => + { + Job? job = await Entities.DeleteAsync(id, ct); + if (job is null) + throw new EntityNotFoundException($"Could not find the Job '{id}'."); + + await _results.DeleteAllAsync(r => r.JobRef == id, ct); + }, + cancellationToken + ); + } + + public Task> GetNewerRevisionAsync( + string id, + long minRevision, + CancellationToken cancellationToken = default + ) + { + return GetNewerRevisionAsync(e => e.Id == id, minRevision, cancellationToken); + } + + private async Task> GetNewerRevisionAsync( + Expression> filter, + long minRevision, + CancellationToken cancellationToken = default + ) + { + using ISubscription subscription = await Entities.SubscribeAsync(filter, cancellationToken); + EntityChange curChange = subscription.Change; + if (curChange.Type == EntityChangeType.Delete && minRevision > 1) + return curChange; + while (true) + { + if (curChange.Entity is not null) + { + if (curChange.Type != EntityChangeType.Delete && minRevision <= curChange.Entity.Revision) + return curChange; + } + await subscription.WaitForChangeAsync(cancellationToken: cancellationToken); + curChange = subscription.Change; + if (curChange.Type == EntityChangeType.Delete) + return curChange; + } + } +} diff --git a/src/Serval/src/Serval.Assessment/Services/ResultService.cs b/src/Serval/src/Serval.Assessment/Services/ResultService.cs new file mode 100644 index 00000000..d25c33b2 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Services/ResultService.cs @@ -0,0 +1,17 @@ +namespace Serval.Assessment.Services; + +public class ResultService(IRepository results) : EntityServiceBase(results), IResultService +{ + public async Task> GetAllAsync( + string engineId, + string jobId, + string? textId = null, + CancellationToken cancellationToken = default + ) + { + return await Entities.GetAllAsync( + r => r.EngineRef == engineId && r.JobRef == jobId && (textId == null || r.TextId == textId), + cancellationToken + ); + } +} diff --git a/src/Serval/src/Serval.Assessment/Usings.cs b/src/Serval/src/Serval.Assessment/Usings.cs new file mode 100644 index 00000000..17020327 --- /dev/null +++ b/src/Serval/src/Serval.Assessment/Usings.cs @@ -0,0 +1,31 @@ +global using System.Diagnostics.CodeAnalysis; +global using System.Linq.Expressions; +global using System.Text.Json; +global using System.Text.Json.Nodes; +global using Asp.Versioning; +global using CaseExtensions; +global using Grpc.Core; +global using Grpc.Net.ClientFactory; +global using MassTransit; +global using Microsoft.AspNetCore.Authorization; +global using Microsoft.AspNetCore.Http; +global using Microsoft.AspNetCore.Mvc; +global using Microsoft.AspNetCore.Routing; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; +global using Microsoft.FeatureManagement.Mvc; +global using NSwag.Annotations; +global using Serval.Assessment.Configuration; +global using Serval.Assessment.Consumers; +global using Serval.Assessment.Contracts; +global using Serval.Assessment.Models; +global using Serval.Assessment.Services; +global using Serval.Shared.Configuration; +global using Serval.Shared.Contracts; +global using Serval.Shared.Controllers; +global using Serval.Shared.Models; +global using Serval.Shared.Services; +global using Serval.Shared.Utils; +global using SIL.DataAccess; +global using SIL.Scripture; diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index da8f5130..5da89446 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -469,6 +469,1971 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c } } + [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] + public partial interface IAssessmentEnginesClient + { + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get all assessment engines. + /// + /// The engines + /// A server side error occurred. + System.Threading.Tasks.Task> GetAllAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Create a new assessment engine. + /// + /// The engine configuration (see above) + /// The new engine + /// A server side error occurred. + System.Threading.Tasks.Task CreateAsync(AssessmentEngineConfig engineConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get an assessment engine. + /// + /// The engine id + /// The engine + /// A server side error occurred. + System.Threading.Tasks.Task GetAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Delete an assessment engine. + /// + /// The engine id + /// The engine was successfully deleted. + /// A server side error occurred. + System.Threading.Tasks.Task DeleteAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get the configuration of the corpus for an assessment engine. + /// + /// The assessment engine id + /// The corpus configuration + /// A server side error occurred. + System.Threading.Tasks.Task GetCorpusAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Replace the corpus configuration for an assessment engine. + /// + /// The assessment engine id + /// The new corpus configuration + /// The corpus configuration + /// A server side error occurred. + System.Threading.Tasks.Task ReplaceCorpusAsync(string id, AssessmentCorpusConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get the configuration of the reference corpus for an assessment engine. + /// + /// The assessment engine id + /// The corpus configuration + /// A server side error occurred. + System.Threading.Tasks.Task GetReferenceCorpusAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Replace the reference corpus configuration for an assessment engine. + /// + /// The assessment engine id + /// The corpus configuration + /// The new corpus configuration + /// A server side error occurred. + System.Threading.Tasks.Task ReplaceReferenceCorpusAsync(string id, AssessmentCorpusConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get all assessment jobs. + /// + /// The engine id + /// The jobs + /// A server side error occurred. + System.Threading.Tasks.Task> GetAllJobsAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Start an assessment job. + /// + /// The engine id + /// The job config (see remarks) + /// The new job + /// A server side error occurred. + System.Threading.Tasks.Task StartJobAsync(string id, AssessmentJobConfig jobConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get an assessment job. + /// + /// + /// If the `minRevision` is not defined, the current job, at whatever state it is, + ///
will be immediately returned. If `minRevision` is defined, Serval will wait for + ///
up to 40 seconds for the engine to job to the `minRevision` specified, else + ///
will timeout. + ///
A use case is to actively query the state of the current job, where the subsequent + ///
request sets the `minRevision` to the returned `revision` + 1 and timeouts are handled gracefully. + ///
This method should use request throttling. + ///
Note: Within the returned job, percentCompleted is a value between 0 and 1. + /// + /// The engine id + /// The job id + /// The minimum revision + /// The job + /// A server side error occurred. + System.Threading.Tasks.Task GetJobAsync(string id, string jobId, long? minRevision = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Delete an assessment job. + /// + /// The engine id + /// The job was successfully deleted. + /// A server side error occurred. + System.Threading.Tasks.Task DeleteJobAsync(string id, string jobId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Cancel an assessment job. + /// + /// The engine id + /// The job id + /// The job was cancelled successfully. + /// A server side error occurred. + System.Threading.Tasks.Task CancelJobAsync(string id, string jobId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get all results of an assessment job. + /// + /// The engine id + /// The job id + /// The text id (optional) + /// The results + /// A server side error occurred. + System.Threading.Tasks.Task> GetAllResultsAsync(string id, string jobId, string? textId = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get all results for the specified text of an assessment job. + /// + /// The engine id + /// The job id + /// The text id + /// The results + /// A server side error occurred. + System.Threading.Tasks.Task> GetResultsByTextIdAsync(string id, string jobId, string textId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + } + + [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AssessmentEnginesClient : IAssessmentEnginesClient + { + #pragma warning disable 8618 // Set by constructor via BaseUrl property + private string _baseUrl; + #pragma warning restore 8618 // Set by constructor via BaseUrl property + private System.Net.Http.HttpClient _httpClient; + private static System.Lazy _settings = new System.Lazy(CreateSerializerSettings, true); + + public AssessmentEnginesClient(System.Net.Http.HttpClient httpClient) + { + BaseUrl = "/api/v1"; + _httpClient = httpClient; + } + + private static Newtonsoft.Json.JsonSerializerSettings CreateSerializerSettings() + { + var settings = new Newtonsoft.Json.JsonSerializerSettings(); + UpdateJsonSerializerSettings(settings); + return settings; + } + + public string BaseUrl + { + get { return _baseUrl; } + set + { + _baseUrl = value; + if (!string.IsNullOrEmpty(_baseUrl) && !_baseUrl.EndsWith("/")) + _baseUrl += '/'; + } + } + + protected Newtonsoft.Json.JsonSerializerSettings JsonSerializerSettings { get { return _settings.Value; } } + + static partial void UpdateJsonSerializerSettings(Newtonsoft.Json.JsonSerializerSettings settings); + + partial void PrepareRequest(System.Net.Http.HttpClient client, System.Net.Http.HttpRequestMessage request, string url); + partial void PrepareRequest(System.Net.Http.HttpClient client, System.Net.Http.HttpRequestMessage request, System.Text.StringBuilder urlBuilder); + partial void ProcessResponse(System.Net.Http.HttpClient client, System.Net.Http.HttpResponseMessage response); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get all assessment engines. + /// + /// The engines + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task> GetAllAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(BaseUrl)) urlBuilder_.Append(BaseUrl); + // Operation Path: "assessment/engines" + urlBuilder_.Append("assessment/engines"); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Create a new assessment engine. + /// + /// The engine configuration (see above) + /// The new engine + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task CreateAsync(AssessmentEngineConfig engineConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (engineConfig == null) + throw new System.ArgumentNullException("engineConfig"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(engineConfig, _settings.Value); + var content_ = new System.Net.Http.StringContent(json_); + content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); + request_.Content = content_; + request_.Method = new System.Net.Http.HttpMethod("POST"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(BaseUrl)) urlBuilder_.Append(BaseUrl); + // Operation Path: "assessment/engines" + urlBuilder_.Append("assessment/engines"); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 201) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 400) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("Bad request. Is the engine type correct?", status_, responseText_, headers_, null); + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get an assessment engine. + /// + /// The engine id + /// The engine + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task GetAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(BaseUrl)) urlBuilder_.Append(BaseUrl); + // Operation Path: "assessment/engines/{id}" + urlBuilder_.Append("assessment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Delete an assessment engine. + /// + /// The engine id + /// The engine was successfully deleted. + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task DeleteAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("DELETE"); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(BaseUrl)) urlBuilder_.Append(BaseUrl); + // Operation Path: "assessment/engines/{id}" + urlBuilder_.Append("assessment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + return; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine does not exist and therefore cannot be deleted.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get the configuration of the corpus for an assessment engine. + /// + /// The assessment engine id + /// The corpus configuration + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task GetCorpusAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(BaseUrl)) urlBuilder_.Append(BaseUrl); + // Operation Path: "assessment/engines/{id}/corpus" + urlBuilder_.Append("assessment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/corpus"); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the assessment engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Replace the corpus configuration for an assessment engine. + /// + /// The assessment engine id + /// The new corpus configuration + /// The corpus configuration + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task ReplaceCorpusAsync(string id, AssessmentCorpusConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + if (corpusConfig == null) + throw new System.ArgumentNullException("corpusConfig"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(corpusConfig, _settings.Value); + var content_ = new System.Net.Http.StringContent(json_); + content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); + request_.Content = content_; + request_.Method = new System.Net.Http.HttpMethod("PUT"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(BaseUrl)) urlBuilder_.Append(BaseUrl); + // Operation Path: "assessment/engines/{id}/corpus" + urlBuilder_.Append("assessment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/corpus"); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the assessment engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get the configuration of the reference corpus for an assessment engine. + /// + /// The assessment engine id + /// The corpus configuration + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task GetReferenceCorpusAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(BaseUrl)) urlBuilder_.Append(BaseUrl); + // Operation Path: "assessment/engines/{id}/reference-corpus" + urlBuilder_.Append("assessment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/reference-corpus"); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 204) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine does not have a reference corpus.", status_, responseText_, headers_, null); + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the assessment engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Replace the reference corpus configuration for an assessment engine. + /// + /// The assessment engine id + /// The corpus configuration + /// The new corpus configuration + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task ReplaceReferenceCorpusAsync(string id, AssessmentCorpusConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + if (corpusConfig == null) + throw new System.ArgumentNullException("corpusConfig"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(corpusConfig, _settings.Value); + var content_ = new System.Net.Http.StringContent(json_); + content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); + request_.Content = content_; + request_.Method = new System.Net.Http.HttpMethod("PUT"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(BaseUrl)) urlBuilder_.Append(BaseUrl); + // Operation Path: "assessment/engines/{id}/reference-corpus" + urlBuilder_.Append("assessment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/reference-corpus"); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the assessment engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get all assessment jobs. + /// + /// The engine id + /// The jobs + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task> GetAllJobsAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(BaseUrl)) urlBuilder_.Append(BaseUrl); + // Operation Path: "assessment/engines/{id}/jobs" + urlBuilder_.Append("assessment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/jobs"); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Start an assessment job. + /// + /// The engine id + /// The job config (see remarks) + /// The new job + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task StartJobAsync(string id, AssessmentJobConfig jobConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + if (jobConfig == null) + throw new System.ArgumentNullException("jobConfig"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(jobConfig, _settings.Value); + var content_ = new System.Net.Http.StringContent(json_); + content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); + request_.Content = content_; + request_.Method = new System.Net.Http.HttpMethod("POST"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(BaseUrl)) urlBuilder_.Append(BaseUrl); + // Operation Path: "assessment/engines/{id}/jobs" + urlBuilder_.Append("assessment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/jobs"); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 201) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 400) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The job configuration was invalid.", status_, responseText_, headers_, null); + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get an assessment job. + /// + /// + /// If the `minRevision` is not defined, the current job, at whatever state it is, + ///
will be immediately returned. If `minRevision` is defined, Serval will wait for + ///
up to 40 seconds for the engine to job to the `minRevision` specified, else + ///
will timeout. + ///
A use case is to actively query the state of the current job, where the subsequent + ///
request sets the `minRevision` to the returned `revision` + 1 and timeouts are handled gracefully. + ///
This method should use request throttling. + ///
Note: Within the returned job, percentCompleted is a value between 0 and 1. + /// + /// The engine id + /// The job id + /// The minimum revision + /// The job + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task GetJobAsync(string id, string jobId, long? minRevision = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + if (jobId == null) + throw new System.ArgumentNullException("jobId"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(BaseUrl)) urlBuilder_.Append(BaseUrl); + // Operation Path: "assessment/engines/{id}/jobs/{jobId}" + urlBuilder_.Append("assessment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/jobs/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(jobId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append('?'); + if (minRevision != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("minRevision")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(minRevision, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + urlBuilder_.Length--; + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine or job does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 408) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The long polling request timed out. This is expected behavior if you\'re using long-polling with the minRevision strategy specified in the docs.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Delete an assessment job. + /// + /// The engine id + /// The job was successfully deleted. + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task DeleteJobAsync(string id, string jobId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + if (jobId == null) + throw new System.ArgumentNullException("jobId"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("DELETE"); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(BaseUrl)) urlBuilder_.Append(BaseUrl); + // Operation Path: "assessment/engines/{id}/jobs/{jobId}" + urlBuilder_.Append("assessment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/jobs/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(jobId, System.Globalization.CultureInfo.InvariantCulture))); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + return; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine does not exist and therefore cannot be deleted.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Cancel an assessment job. + /// + /// The engine id + /// The job id + /// The job was cancelled successfully. + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task CancelJobAsync(string id, string jobId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + if (jobId == null) + throw new System.ArgumentNullException("jobId"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Content = new System.Net.Http.StringContent(string.Empty, System.Text.Encoding.UTF8, "application/json"); + request_.Method = new System.Net.Http.HttpMethod("POST"); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(BaseUrl)) urlBuilder_.Append(BaseUrl); + // Operation Path: "assessment/engines/{id}/jobs/{jobId}/cancel" + urlBuilder_.Append("assessment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/jobs/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(jobId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/cancel"); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + return; + } + else + if (status_ == 204) + { + return; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 405) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine does not support canceling jobs.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get all results of an assessment job. + /// + /// The engine id + /// The job id + /// The text id (optional) + /// The results + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task> GetAllResultsAsync(string id, string jobId, string? textId = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + if (jobId == null) + throw new System.ArgumentNullException("jobId"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(BaseUrl)) urlBuilder_.Append(BaseUrl); + // Operation Path: "assessment/engines/{id}/jobs/{jobId}/results" + urlBuilder_.Append("assessment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/jobs/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(jobId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/results"); + urlBuilder_.Append('?'); + if (textId != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("textId")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(textId, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + urlBuilder_.Length--; + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 409) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get all results for the specified text of an assessment job. + /// + /// The engine id + /// The job id + /// The text id + /// The results + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task> GetResultsByTextIdAsync(string id, string jobId, string textId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + if (jobId == null) + throw new System.ArgumentNullException("jobId"); + + if (textId == null) + throw new System.ArgumentNullException("textId"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(BaseUrl)) urlBuilder_.Append(BaseUrl); + // Operation Path: "assessment/engines/{id}/jobs/{jobId}/results/{textId}" + urlBuilder_.Append("assessment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/jobs/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(jobId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/results/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(textId, System.Globalization.CultureInfo.InvariantCulture))); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 409) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + protected struct ObjectResponseResult + { + public ObjectResponseResult(T responseObject, string responseText) + { + this.Object = responseObject; + this.Text = responseText; + } + + public T Object { get; } + + public string Text { get; } + } + + public bool ReadResponseAsString { get; set; } + + protected virtual async System.Threading.Tasks.Task> ReadObjectResponseAsync(System.Net.Http.HttpResponseMessage response, System.Collections.Generic.IReadOnlyDictionary> headers, System.Threading.CancellationToken cancellationToken) + { + if (response == null || response.Content == null) + { + return new ObjectResponseResult(default(T)!, string.Empty); + } + + if (ReadResponseAsString) + { + var responseText = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + try + { + var typedBody = Newtonsoft.Json.JsonConvert.DeserializeObject(responseText, JsonSerializerSettings); + return new ObjectResponseResult(typedBody!, responseText); + } + catch (Newtonsoft.Json.JsonException exception) + { + var message = "Could not deserialize the response body string as " + typeof(T).FullName + "."; + throw new ServalApiException(message, (int)response.StatusCode, responseText, headers, exception); + } + } + else + { + try + { + using (var responseStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) + using (var streamReader = new System.IO.StreamReader(responseStream)) + using (var jsonTextReader = new Newtonsoft.Json.JsonTextReader(streamReader)) + { + var serializer = Newtonsoft.Json.JsonSerializer.Create(JsonSerializerSettings); + var typedBody = serializer.Deserialize(jsonTextReader); + return new ObjectResponseResult(typedBody!, string.Empty); + } + } + catch (Newtonsoft.Json.JsonException exception) + { + var message = "Could not deserialize the response body stream as " + typeof(T).FullName + "."; + throw new ServalApiException(message, (int)response.StatusCode, string.Empty, headers, exception); + } + } + } + + private string ConvertToString(object? value, System.Globalization.CultureInfo cultureInfo) + { + if (value == null) + { + return ""; + } + + if (value is System.Enum) + { + var name = System.Enum.GetName(value.GetType(), value); + if (name != null) + { + var field = System.Reflection.IntrospectionExtensions.GetTypeInfo(value.GetType()).GetDeclaredField(name); + if (field != null) + { + var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field, typeof(System.Runtime.Serialization.EnumMemberAttribute)) + as System.Runtime.Serialization.EnumMemberAttribute; + if (attribute != null) + { + return attribute.Value != null ? attribute.Value : name; + } + } + + var converted = System.Convert.ToString(System.Convert.ChangeType(value, System.Enum.GetUnderlyingType(value.GetType()), cultureInfo)); + return converted == null ? string.Empty : converted; + } + } + else if (value is bool) + { + return System.Convert.ToString((bool)value, cultureInfo).ToLowerInvariant(); + } + else if (value is byte[]) + { + return System.Convert.ToBase64String((byte[]) value); + } + else if (value is string[]) + { + return string.Join(",", (string[])value); + } + else if (value.GetType().IsArray) + { + var valueArray = (System.Array)value; + var valueTextArray = new string[valueArray.Length]; + for (var i = 0; i < valueArray.Length; i++) + { + valueTextArray[i] = ConvertToString(valueArray.GetValue(i), cultureInfo); + } + return string.Join(",", valueTextArray); + } + + var result = System.Convert.ToString(value, cultureInfo); + return result == null ? "" : result; + } + } + [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] public partial interface IDataFilesClient { @@ -5617,6 +7582,252 @@ public partial class DeploymentInfo } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AssessmentEngine + { + [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Id { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Url { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Name { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("type", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Type { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("corpus", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public AssessmentCorpus Corpus { get; set; } = new AssessmentCorpus(); + + [Newtonsoft.Json.JsonProperty("referenceCorpus", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public AssessmentCorpus? ReferenceCorpus { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AssessmentCorpus + { + [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Url { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Name { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("language", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Language { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("files", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Files { get; set; } = new System.Collections.ObjectModel.Collection(); + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AssessmentCorpusFile + { + [Newtonsoft.Json.JsonProperty("file", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public ResourceLink File { get; set; } = new ResourceLink(); + + [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? TextId { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class ResourceLink + { + [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Id { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Url { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AssessmentEngineConfig + { + /// + /// The assessment engine name. + /// + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Name { get; set; } = default!; + + /// + /// The assessment engine type. + /// + [Newtonsoft.Json.JsonProperty("type", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Type { get; set; } = default!; + + /// + /// The corpus. + /// + [Newtonsoft.Json.JsonProperty("corpus", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public AssessmentCorpusConfig Corpus { get; set; } = new AssessmentCorpusConfig(); + + /// + /// The reference corpus. + /// + [Newtonsoft.Json.JsonProperty("referenceCorpus", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public AssessmentCorpusConfig? ReferenceCorpus { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AssessmentCorpusConfig + { + /// + /// The corpus name. + /// + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Name { get; set; } = default!; + + /// + /// The language tag. + /// + [Newtonsoft.Json.JsonProperty("language", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Language { get; set; } = default!; + + /// + /// The corpus files. + /// + [Newtonsoft.Json.JsonProperty("files", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Files { get; set; } = new System.Collections.ObjectModel.Collection(); + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AssessmentCorpusFileConfig + { + [Newtonsoft.Json.JsonProperty("fileId", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string FileId { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? TextId { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AssessmentJob + { + [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Id { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Url { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("revision", Required = Newtonsoft.Json.Required.Always)] + public int Revision { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Name { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("engine", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public ResourceLink Engine { get; set; } = new ResourceLink(); + + [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TextIds { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? ScriptureRange { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("percentCompleted", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public double? PercentCompleted { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("message", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Message { get; set; } = default!; + + /// + /// The current job state. + /// + [Newtonsoft.Json.JsonProperty("state", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + [Newtonsoft.Json.JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] + public JobState State { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("dateFinished", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.DateTimeOffset? DateFinished { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("options", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public object? Options { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] + public enum JobState + { + + [System.Runtime.Serialization.EnumMember(Value = @"Pending")] + Pending = 0, + + [System.Runtime.Serialization.EnumMember(Value = @"Active")] + Active = 1, + + [System.Runtime.Serialization.EnumMember(Value = @"Completed")] + Completed = 2, + + [System.Runtime.Serialization.EnumMember(Value = @"Faulted")] + Faulted = 3, + + [System.Runtime.Serialization.EnumMember(Value = @"Canceled")] + Canceled = 4, + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AssessmentJobConfig + { + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Name { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TextIds { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? ScriptureRange { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("options", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public object? Options { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AssessmentResult + { + [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string TextId { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("ref", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Ref { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("score", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public double? Score { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("description", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Description { get; set; } = default!; + + } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] public partial class DataFile { @@ -5915,19 +8126,6 @@ public partial class TranslationCorpus } - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class ResourceLink - { - [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Id { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Url { get; set; } = default!; - - } - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] public partial class TranslationCorpusFile { @@ -6125,27 +8323,6 @@ public partial class PretranslateCorpus } - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] - public enum JobState - { - - [System.Runtime.Serialization.EnumMember(Value = @"Pending")] - Pending = 0, - - [System.Runtime.Serialization.EnumMember(Value = @"Active")] - Active = 1, - - [System.Runtime.Serialization.EnumMember(Value = @"Completed")] - Completed = 2, - - [System.Runtime.Serialization.EnumMember(Value = @"Faulted")] - Faulted = 3, - - [System.Runtime.Serialization.EnumMember(Value = @"Canceled")] - Canceled = 4, - - } - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] public partial class TranslationBuildConfig { @@ -6267,6 +8444,12 @@ public enum WebhookEvent [System.Runtime.Serialization.EnumMember(Value = @"TranslationBuildFinished")] TranslationBuildFinished = 1, + [System.Runtime.Serialization.EnumMember(Value = @"AssessmentJobStarted")] + AssessmentJobStarted = 2, + + [System.Runtime.Serialization.EnumMember(Value = @"AssessmentJobFinished")] + AssessmentJobFinished = 3, + } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] diff --git a/src/Serval/src/Serval.DataFiles/Controllers/DataFilesController.cs b/src/Serval/src/Serval.DataFiles/Controllers/DataFilesController.cs index 364e1c0c..32218a68 100644 --- a/src/Serval/src/Serval.DataFiles/Controllers/DataFilesController.cs +++ b/src/Serval/src/Serval.DataFiles/Controllers/DataFilesController.cs @@ -41,7 +41,7 @@ public async Task> GetAllAsync(CancellationToken cancel /// The file does not exist /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.ReadFiles)] - [HttpGet("{id}", Name = "GetDataFile")] + [HttpGet("{id}", Name = Endpoints.GetDataFile)] [ProducesResponseType(StatusCodes.Status200OK)] [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] @@ -230,7 +230,7 @@ private DataFileDto Map(DataFile source) return new DataFileDto { Id = source.Id, - Url = _urlService.GetUrl("GetDataFile", new { id = source.Id }), + Url = _urlService.GetUrl(Endpoints.GetDataFile, new { id = source.Id }), Name = source.Name, Format = source.Format, Revision = source.Revision diff --git a/src/Serval/src/Serval.DataFiles/Services/DataFileService.cs b/src/Serval/src/Serval.DataFiles/Services/DataFileService.cs index 1c09b052..6df3a01f 100644 --- a/src/Serval/src/Serval.DataFiles/Services/DataFileService.cs +++ b/src/Serval/src/Serval.DataFiles/Services/DataFileService.cs @@ -1,6 +1,6 @@ namespace Serval.DataFiles.Services; -public class DataFileService : EntityServiceBase, IDataFileService +public class DataFileService : OwnedEntityServiceBase, IDataFileService { private readonly IOptionsMonitor _options; private readonly IDataAccessContext _dataAccessContext; @@ -34,11 +34,6 @@ public async Task GetAsync(string id, string owner, CancellationToken return dataFile; } - public async Task> GetAllAsync(string owner, CancellationToken cancellationToken = default) - { - return await Entities.GetAllAsync(c => c.Owner == owner, cancellationToken); - } - public async Task CreateAsync(DataFile dataFile, Stream stream, CancellationToken cancellationToken = default) { string filename = Path.GetRandomFileName(); diff --git a/src/Serval/src/Serval.DataFiles/Services/DeletedFileCleaner.cs b/src/Serval/src/Serval.DataFiles/Services/DeletedFileCleaner.cs index 3336ba2b..33a23d89 100644 --- a/src/Serval/src/Serval.DataFiles/Services/DeletedFileCleaner.cs +++ b/src/Serval/src/Serval.DataFiles/Services/DeletedFileCleaner.cs @@ -38,9 +38,10 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { while (!stoppingToken.IsCancellationRequested) { - DateTimeOffset? next = _cronExpression.GetNextOccurrence(DateTimeOffset.Now, TimeZoneInfo.Local); + DateTimeOffset now = DateTimeOffset.Now; + DateTimeOffset? next = _cronExpression.GetNextOccurrence(now, TimeZoneInfo.Local); Debug.Assert(next.HasValue); - await Task.Delay(next.Value - DateTimeOffset.Now, stoppingToken); + await Task.Delay(next.Value - now, stoppingToken); await CleanAsync(stoppingToken); } } diff --git a/src/Serval/src/Serval.Grpc/Protos/serval/assessment/v1/engine.proto b/src/Serval/src/Serval.Grpc/Protos/serval/assessment/v1/engine.proto new file mode 100644 index 00000000..2fb422d6 --- /dev/null +++ b/src/Serval/src/Serval.Grpc/Protos/serval/assessment/v1/engine.proto @@ -0,0 +1,62 @@ +syntax = "proto3"; + +package serval.assessment.v1; + +import "google/protobuf/empty.proto"; + +service AssessmentEngineApi { + rpc Create(CreateRequest) returns (google.protobuf.Empty); + rpc Delete(DeleteRequest) returns (google.protobuf.Empty); + rpc StartJob(StartJobRequest) returns (google.protobuf.Empty); + rpc CancelJob(CancelJobRequest) returns (google.protobuf.Empty); +} + +message CreateRequest { + string engine_type = 1; + string engine_id = 2; + optional string engine_name = 3; +} + +message DeleteRequest { + string engine_type = 1; + string engine_id = 2; +} + +message StartJobRequest { + string engine_type = 1; + string engine_id = 2; + string job_id = 3; + Corpus corpus = 4; + optional Corpus reference_corpus = 5; + bool include_all = 6; + map include_chapters = 7; + repeated string include_text_ids = 8; + optional string options = 9; +} + +message CancelJobRequest { + string engine_type = 1; + string engine_id = 2; + string job_id = 3; +} + +message ScriptureChapters { + repeated int32 chapters = 1; +} + +message Corpus { + string id = 1; + string language = 2; + repeated CorpusFile files = 3; +} + +message CorpusFile { + string location = 1; + FileFormat format = 2; + string text_id = 3; +} + +enum FileFormat { + FILE_FORMAT_TEXT = 0; + FILE_FORMAT_PARATEXT = 1; +} diff --git a/src/Serval/src/Serval.Grpc/Protos/serval/assessment/v1/platform.proto b/src/Serval/src/Serval.Grpc/Protos/serval/assessment/v1/platform.proto new file mode 100644 index 00000000..f49bddd9 --- /dev/null +++ b/src/Serval/src/Serval.Grpc/Protos/serval/assessment/v1/platform.proto @@ -0,0 +1,51 @@ +syntax = "proto3"; + +package serval.assessment.v1; + +import "google/protobuf/empty.proto"; + +service AssessmentPlatformApi { + rpc UpdateJobStatus(UpdateJobStatusRequest) returns (google.protobuf.Empty); + rpc JobStarted(JobStartedRequest) returns (google.protobuf.Empty); + rpc JobCompleted(JobCompletedRequest) returns (google.protobuf.Empty); + rpc JobCanceled(JobCanceledRequest) returns (google.protobuf.Empty); + rpc JobFaulted(JobFaultedRequest) returns (google.protobuf.Empty); + rpc JobRestarting(JobRestartingRequest) returns (google.protobuf.Empty); + + rpc InsertResults(stream InsertResultsRequest) returns (google.protobuf.Empty); +} + +message UpdateJobStatusRequest { + string job_id = 1; + optional double percent_completed = 2; + optional string message = 3; +} + +message JobStartedRequest { + string job_id = 1; +} + +message JobCompletedRequest { + string job_id = 1; +} + +message JobCanceledRequest { + string job_id = 1; +} + +message JobFaultedRequest { + string job_id = 1; + string message = 2; +} + +message JobRestartingRequest { + string job_id = 1; +} + +message InsertResultsRequest { + string job_id = 1; + string text_id = 2; + string ref = 3; + optional double score = 4; + optional string description = 5; +} diff --git a/src/Serval/src/Serval.Grpc/Protos/serval/health/v1/health.proto b/src/Serval/src/Serval.Grpc/Protos/serval/health/v1/health.proto new file mode 100644 index 00000000..68c6974f --- /dev/null +++ b/src/Serval/src/Serval.Grpc/Protos/serval/health/v1/health.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package serval.health.v1; + +import "google/protobuf/empty.proto"; + +service HealthApi { + rpc HealthCheck(google.protobuf.Empty) returns (HealthCheckResponse); +} + +message HealthCheckResponse { + HealthCheckStatus status = 1; + map data = 2; + optional string error = 3; +} + +enum HealthCheckStatus { + UNHEALTHY = 0; + DEGRADED = 1; + HEALTHY = 2; +} diff --git a/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/engine.proto b/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/engine.proto index cf879a24..6ae643c5 100644 --- a/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/engine.proto +++ b/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/engine.proto @@ -16,7 +16,6 @@ service TranslationEngineApi { rpc GetModelDownloadUrl(GetModelDownloadUrlRequest) returns (GetModelDownloadUrlResponse); rpc GetQueueSize(GetQueueSizeRequest) returns (GetQueueSizeResponse); rpc GetLanguageInfo(GetLanguageInfoRequest) returns (GetLanguageInfoResponse); - rpc HealthCheck(google.protobuf.Empty) returns (HealthCheckResponse); } message CreateRequest { @@ -134,12 +133,6 @@ message TranslationResult { repeated Phrase phrases = 7; } -message HealthCheckResponse { - HealthCheckStatus status = 1; - map data = 2; - optional string error = 3; -} - message WordGraphArc { int32 prev_state = 1; int32 next_state = 2; @@ -193,9 +186,3 @@ enum TranslationSource { TRANSLATION_SOURCE_SECONDARY = 1; TRANSLATION_SOURCE_HUMAN = 2; } - -enum HealthCheckStatus { - UNHEALTHY = 0; - DEGRADED = 1; - HEALTHY = 2; -} \ No newline at end of file diff --git a/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto b/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto index 0c5773e3..84b24ab1 100644 --- a/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto +++ b/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto @@ -13,7 +13,7 @@ service TranslationPlatformApi { rpc BuildRestarting(BuildRestartingRequest) returns (google.protobuf.Empty); rpc IncrementTranslationEngineCorpusSize(IncrementTranslationEngineCorpusSizeRequest) returns (google.protobuf.Empty); - rpc InsertPretranslations(stream InsertPretranslationRequest) returns (google.protobuf.Empty); + rpc InsertPretranslations(stream InsertPretranslationsRequest) returns (google.protobuf.Empty); } message UpdateBuildStatusRequest { @@ -52,11 +52,7 @@ message IncrementTranslationEngineCorpusSizeRequest { int32 count = 2; } -message DeleteAllPretranslationsRequest { - string engine_id = 1; -} - -message InsertPretranslationRequest { +message InsertPretranslationsRequest { string engine_id = 1; string corpus_id = 2; string text_id = 3; diff --git a/src/Serval/src/Serval.Grpc/Utils/WriteGrpcHealthCheckResponse.cs b/src/Serval/src/Serval.Grpc/Utils/WriteGrpcHealthCheckResponse.cs index a1b829bd..19a666af 100644 --- a/src/Serval/src/Serval.Grpc/Utils/WriteGrpcHealthCheckResponse.cs +++ b/src/Serval/src/Serval.Grpc/Utils/WriteGrpcHealthCheckResponse.cs @@ -1,6 +1,6 @@ using Microsoft.Extensions.Diagnostics.HealthChecks; -namespace Serval.Translation.V1; +namespace Serval.Health.V1; public class WriteGrpcHealthCheckResponse { diff --git a/src/Serval/src/Serval.Shared/Contracts/AssessmentJobFinished.cs b/src/Serval/src/Serval.Shared/Contracts/AssessmentJobFinished.cs new file mode 100644 index 00000000..0751a3d1 --- /dev/null +++ b/src/Serval/src/Serval.Shared/Contracts/AssessmentJobFinished.cs @@ -0,0 +1,11 @@ +namespace Serval.Shared.Contracts; + +public record AssessmentJobFinished +{ + public required string JobId { get; init; } + public required string EngineId { get; init; } + public required string Owner { get; init; } + public required JobState JobState { get; init; } + public required string Message { get; init; } + public required DateTime DateFinished { get; init; } +} diff --git a/src/Serval/src/Serval.Shared/Contracts/AssessmentJobFinishedDto.cs b/src/Serval/src/Serval.Shared/Contracts/AssessmentJobFinishedDto.cs new file mode 100644 index 00000000..9382e35f --- /dev/null +++ b/src/Serval/src/Serval.Shared/Contracts/AssessmentJobFinishedDto.cs @@ -0,0 +1,10 @@ +namespace Serval.Shared.Contracts; + +public record AssessmentJobFinishedDto +{ + public required ResourceLinkDto Job { get; init; } + public required ResourceLinkDto Engine { get; init; } + public required JobState JobState { get; init; } + public required string Message { get; init; } + public required DateTime DateFinished { get; init; } +} diff --git a/src/Serval/src/Serval.Shared/Contracts/AssessmentJobStarted.cs b/src/Serval/src/Serval.Shared/Contracts/AssessmentJobStarted.cs new file mode 100644 index 00000000..4a03d86c --- /dev/null +++ b/src/Serval/src/Serval.Shared/Contracts/AssessmentJobStarted.cs @@ -0,0 +1,8 @@ +namespace Serval.Shared.Contracts; + +public record AssessmentJobStarted +{ + public required string JobId { get; init; } + public required string EngineId { get; init; } + public required string Owner { get; init; } +} diff --git a/src/Serval/src/Serval.Shared/Contracts/AssessmentJobStartedDto.cs b/src/Serval/src/Serval.Shared/Contracts/AssessmentJobStartedDto.cs new file mode 100644 index 00000000..16333e02 --- /dev/null +++ b/src/Serval/src/Serval.Shared/Contracts/AssessmentJobStartedDto.cs @@ -0,0 +1,7 @@ +namespace Serval.Shared.Contracts; + +public record AssessmentJobStartedDto +{ + public required ResourceLinkDto Job { get; init; } + public required ResourceLinkDto Engine { get; init; } +} diff --git a/src/Serval/src/Serval.Shared/Contracts/DataFileUpdated.cs b/src/Serval/src/Serval.Shared/Contracts/DataFileUpdated.cs new file mode 100644 index 00000000..1d32e196 --- /dev/null +++ b/src/Serval/src/Serval.Shared/Contracts/DataFileUpdated.cs @@ -0,0 +1,6 @@ +namespace Serval.Shared.Contracts; + +public record DataFileUpdated +{ + public required string DataFileId { get; init; } +} diff --git a/src/Serval/src/Serval.Shared/Contracts/TranslationBuildFinishedDto.cs b/src/Serval/src/Serval.Shared/Contracts/TranslationBuildFinishedDto.cs index 2d5b5b81..2d7091d9 100644 --- a/src/Serval/src/Serval.Shared/Contracts/TranslationBuildFinishedDto.cs +++ b/src/Serval/src/Serval.Shared/Contracts/TranslationBuildFinishedDto.cs @@ -5,5 +5,6 @@ public record TranslationBuildFinishedDto public required ResourceLinkDto Build { get; init; } public required ResourceLinkDto Engine { get; init; } public required JobState BuildState { get; init; } + public required string Message { get; init; } public required DateTime DateFinished { get; init; } } diff --git a/src/Serval/src/Serval.Shared/Controllers/Endpoints.cs b/src/Serval/src/Serval.Shared/Controllers/Endpoints.cs new file mode 100644 index 00000000..2779062b --- /dev/null +++ b/src/Serval/src/Serval.Shared/Controllers/Endpoints.cs @@ -0,0 +1,17 @@ +namespace Serval.Shared.Controllers; + +public static class Endpoints +{ + public const string GetDataFile = "GetDataFile"; + + public const string GetTranslationEngine = "GetTranslationEngine"; + public const string GetTranslationCorpus = "GetTranslationCorpus"; + public const string GetTranslationBuild = "GetTranslationBuild"; + + public const string GetAssessmentEngine = "GetAssessmentEngine"; + public const string GetAssessmentCorpus = "GetAssessmentCorpus"; + public const string GetAssessmentReferenceCorpus = "GetAssessmentReferenceCorpus"; + public const string GetAssessmentJob = "GetAssessmentJob"; + + public const string GetWebhook = "GetWebhook"; +} diff --git a/src/Serval/src/Serval.Shared/Controllers/Scopes.cs b/src/Serval/src/Serval.Shared/Controllers/Scopes.cs index 7d941680..b94ccf87 100644 --- a/src/Serval/src/Serval.Shared/Controllers/Scopes.cs +++ b/src/Serval/src/Serval.Shared/Controllers/Scopes.cs @@ -7,6 +7,11 @@ public static class Scopes public const string UpdateTranslationEngines = "update:translation_engines"; public const string DeleteTranslationEngines = "delete:translation_engines"; + public const string CreateAssessmentEngines = "create:assessment_engines"; + public const string ReadAssessmentEngines = "read:assessment_engines"; + public const string UpdateAssessmentEngines = "update:assessment_engines"; + public const string DeleteAssessmentEngines = "delete:assessment_engines"; + public const string CreateHooks = "create:hooks"; public const string ReadHooks = "read:hooks"; public const string DeleteHooks = "delete:hooks"; @@ -19,12 +24,15 @@ public static class Scopes public const string ReadStatus = "read:status"; public static IEnumerable All => - new[] - { + [ CreateTranslationEngines, ReadTranslationEngines, UpdateTranslationEngines, DeleteTranslationEngines, + CreateAssessmentEngines, + ReadAssessmentEngines, + UpdateAssessmentEngines, + DeleteAssessmentEngines, CreateHooks, ReadHooks, DeleteHooks, @@ -33,5 +41,5 @@ public static class Scopes UpdateFiles, DeleteFiles, ReadStatus - }; + ]; } diff --git a/src/Serval/src/Serval.Shared/Serval.Shared.csproj b/src/Serval/src/Serval.Shared/Serval.Shared.csproj index 7e46a9ee..c464aa9b 100644 --- a/src/Serval/src/Serval.Shared/Serval.Shared.csproj +++ b/src/Serval/src/Serval.Shared/Serval.Shared.csproj @@ -25,6 +25,7 @@ + diff --git a/src/Serval/src/Serval.Shared/Services/EntityServiceBase.cs b/src/Serval/src/Serval.Shared/Services/EntityServiceBase.cs index 6970a80c..e506b402 100644 --- a/src/Serval/src/Serval.Shared/Services/EntityServiceBase.cs +++ b/src/Serval/src/Serval.Shared/Services/EntityServiceBase.cs @@ -1,14 +1,9 @@ namespace Serval.Shared.Services; -public class EntityServiceBase +public abstract class EntityServiceBase(IRepository entities) where T : IEntity { - protected EntityServiceBase(IRepository entities) - { - Entities = entities; - } - - protected IRepository Entities { get; } + protected IRepository Entities { get; } = entities; public async Task GetAsync(string id, CancellationToken cancellationToken = default) { @@ -18,9 +13,10 @@ public async Task GetAsync(string id, CancellationToken cancellationToken = d return entity; } - public virtual Task CreateAsync(T entity, CancellationToken cancellationToken = default) + public virtual async Task CreateAsync(T entity, CancellationToken cancellationToken = default) { - return Entities.InsertAsync(entity, cancellationToken); + await Entities.InsertAsync(entity, cancellationToken); + return entity; } public virtual async Task DeleteAsync(string id, CancellationToken cancellationToken = default) diff --git a/src/Serval/src/Serval.Translation/Services/GrpcServiceHealthCheck.cs b/src/Serval/src/Serval.Shared/Services/GrpcServiceHealthCheck.cs similarity index 84% rename from src/Serval/src/Serval.Translation/Services/GrpcServiceHealthCheck.cs rename to src/Serval/src/Serval.Shared/Services/GrpcServiceHealthCheck.cs index 69bded91..04f75f62 100644 --- a/src/Serval/src/Serval.Translation/Services/GrpcServiceHealthCheck.cs +++ b/src/Serval/src/Serval.Shared/Services/GrpcServiceHealthCheck.cs @@ -1,5 +1,4 @@ -using Microsoft.Extensions.Diagnostics.HealthChecks; -using Serval.Translation.V1; +using Serval.Health.V1; namespace Serval.Shared.Services; @@ -12,8 +11,9 @@ public async Task CheckHealthAsync( CancellationToken cancellationToken = default ) { - TranslationEngineApi.TranslationEngineApiClient client = - _grpcClientFactory.CreateClient(context.Registration.Name); + HealthApi.HealthApiClient client = _grpcClientFactory.CreateClient( + $"{context.Registration.Name}-Health" + ); HealthCheckResponse? healthCheckResponse = await client.HealthCheckAsync( new Google.Protobuf.WellKnownTypes.Empty(), cancellationToken: cancellationToken diff --git a/src/Serval/src/Serval.Shared/Services/OwnedEntityServiceBase.cs b/src/Serval/src/Serval.Shared/Services/OwnedEntityServiceBase.cs new file mode 100644 index 00000000..2ddcd521 --- /dev/null +++ b/src/Serval/src/Serval.Shared/Services/OwnedEntityServiceBase.cs @@ -0,0 +1,10 @@ +namespace Serval.Shared.Services; + +public abstract class OwnedEntityServiceBase(IRepository entities) : EntityServiceBase(entities) + where T : IOwnedEntity +{ + public virtual async Task> GetAllAsync(string owner, CancellationToken cancellationToken = default) + { + return await Entities.GetAllAsync(e => e.Owner == owner, cancellationToken); + } +} diff --git a/src/Serval/src/Serval.Shared/Usings.cs b/src/Serval/src/Serval.Shared/Usings.cs index e494cd2d..3e84144f 100644 --- a/src/Serval/src/Serval.Shared/Usings.cs +++ b/src/Serval/src/Serval.Shared/Usings.cs @@ -2,11 +2,13 @@ global using System.Text.Json; global using System.Text.Json.Serialization; global using Grpc.Core; +global using Grpc.Net.ClientFactory; global using Microsoft.AspNetCore.Authorization; global using Microsoft.AspNetCore.Http; global using Microsoft.AspNetCore.Mvc; global using Microsoft.AspNetCore.Mvc.Filters; global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.Diagnostics.HealthChecks; global using Microsoft.Extensions.Logging; global using Microsoft.Extensions.Options; global using Serval.Shared.Configuration; diff --git a/src/Serval/src/Serval.Translation/Configuration/IServalBuilderExtensions.cs b/src/Serval/src/Serval.Translation/Configuration/IServalBuilderExtensions.cs index a66dbb6e..190d627f 100644 --- a/src/Serval/src/Serval.Translation/Configuration/IServalBuilderExtensions.cs +++ b/src/Serval/src/Serval.Translation/Configuration/IServalBuilderExtensions.cs @@ -1,4 +1,5 @@ -using Serval.Translation.V1; +using Serval.Health.V1; +using Serval.Translation.V1; namespace Microsoft.Extensions.DependencyInjection; @@ -35,6 +36,10 @@ public static IServalBuilder AddTranslation( engine.Type, o => o.Address = new Uri(engine.Address) ); + builder.Services.AddGrpcClient( + $"{engine.Type}-Health", + o => o.Address = new Uri(engine.Address) + ); builder.Services.AddHealthChecks().AddCheck(engine.Type); } diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusUpdateConfigDto.cs b/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusUpdateConfigDto.cs index 7b73c2fa..8dd96311 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusUpdateConfigDto.cs +++ b/src/Serval/src/Serval.Translation/Contracts/TranslationCorpusUpdateConfigDto.cs @@ -16,7 +16,7 @@ ValidationContext validationContext { yield return new System.ComponentModel.DataAnnotations.ValidationResult( "At least one field must be specified.", - new[] { nameof(SourceFiles), nameof(TargetFiles) } + [nameof(SourceFiles), nameof(TargetFiles)] ); } } diff --git a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs b/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs index 81119cc8..d11d8679 100644 --- a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs +++ b/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs @@ -52,7 +52,7 @@ public async Task> GetAllAsync(CancellationTok /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.ReadTranslationEngines)] - [HttpGet("{id}", Name = "GetTranslationEngine")] + [HttpGet("{id}", Name = Endpoints.GetTranslationEngine)] [ProducesResponseType(StatusCodes.Status200OK)] [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] @@ -450,7 +450,7 @@ CancellationToken cancellationToken /// The engine or corpus does not exist. /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.ReadTranslationEngines)] - [HttpGet("{id}/corpora/{corpusId}", Name = "GetTranslationCorpus")] + [HttpGet("{id}/corpora/{corpusId}", Name = Endpoints.GetTranslationCorpus)] [ProducesResponseType(StatusCodes.Status200OK)] [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] @@ -737,7 +737,7 @@ CancellationToken cancellationToken /// The long polling request timed out. This is expected behavior if you're using long-polling with the minRevision strategy specified in the docs. /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.ReadTranslationEngines)] - [HttpGet("{id}/builds/{buildId}", Name = "GetTranslationBuild")] + [HttpGet("{id}/builds/{buildId}", Name = Endpoints.GetTranslationBuild)] [ProducesResponseType(StatusCodes.Status200OK)] [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] @@ -1126,7 +1126,7 @@ private TranslationEngineDto Map(Engine source) return new TranslationEngineDto { Id = source.Id, - Url = _urlService.GetUrl("GetTranslationEngine", new { id = source.Id }), + Url = _urlService.GetUrl(Endpoints.GetTranslationEngine, new { id = source.Id }), Name = source.Name, SourceLanguage = source.SourceLanguage, TargetLanguage = source.TargetLanguage, @@ -1144,13 +1144,13 @@ private TranslationBuildDto Map(Build source) return new TranslationBuildDto { Id = source.Id, - Url = _urlService.GetUrl("GetTranslationBuild", new { id = source.EngineRef, buildId = source.Id }), + Url = _urlService.GetUrl(Endpoints.GetTranslationBuild, new { id = source.EngineRef, buildId = source.Id }), Revision = source.Revision, Name = source.Name, Engine = new ResourceLinkDto { Id = source.EngineRef, - Url = _urlService.GetUrl("GetTranslationEngine", new { id = source.EngineRef }) + Url = _urlService.GetUrl(Endpoints.GetTranslationEngine, new { id = source.EngineRef }) }, TrainOn = source.TrainOn?.Select(s => Map(source.EngineRef, s)).ToList(), Pretranslate = source.Pretranslate?.Select(s => Map(source.EngineRef, s)).ToList(), @@ -1171,7 +1171,10 @@ private PretranslateCorpusDto Map(string engineId, PretranslateCorpus source) Corpus = new ResourceLinkDto { Id = source.CorpusRef, - Url = _urlService.GetUrl("GetTranslationCorpus", new { id = engineId, corpusId = source.CorpusRef }) + Url = _urlService.GetUrl( + Endpoints.GetTranslationCorpus, + new { id = engineId, corpusId = source.CorpusRef } + ) }, TextIds = source.TextIds, ScriptureRange = source.ScriptureRange @@ -1185,7 +1188,10 @@ private TrainingCorpusDto Map(string engineId, TrainingCorpus source) Corpus = new ResourceLinkDto { Id = source.CorpusRef, - Url = _urlService.GetUrl("GetTranslationCorpus", new { id = engineId, corpusId = source.CorpusRef }) + Url = _urlService.GetUrl( + Endpoints.GetTranslationCorpus, + new { id = engineId, corpusId = source.CorpusRef } + ) }, TextIds = source.TextIds, ScriptureRange = source.ScriptureRange @@ -1263,11 +1269,11 @@ private TranslationCorpusDto Map(string engineId, Corpus source) return new TranslationCorpusDto { Id = source.Id, - Url = _urlService.GetUrl("GetTranslationCorpus", new { id = engineId, corpusId = source.Id }), + Url = _urlService.GetUrl(Endpoints.GetTranslationCorpus, new { id = engineId, corpusId = source.Id }), Engine = new ResourceLinkDto { Id = engineId, - Url = _urlService.GetUrl("GetTranslationEngine", new { id = engineId }) + Url = _urlService.GetUrl(Endpoints.GetTranslationEngine, new { id = engineId }) }, Name = source.Name, SourceLanguage = source.SourceLanguage, @@ -1284,7 +1290,7 @@ private TranslationCorpusFileDto Map(CorpusFile source) File = new ResourceLinkDto { Id = source.Id, - Url = _urlService.GetUrl("GetDataFile", new { id = source.Id }) + Url = _urlService.GetUrl(Endpoints.GetDataFile, new { id = source.Id }) }, TextId = source.TextId }; diff --git a/src/Serval/src/Serval.Translation/Services/EngineService.cs b/src/Serval/src/Serval.Translation/Services/EngineService.cs index 00ec01d0..bfee7000 100644 --- a/src/Serval/src/Serval.Translation/Services/EngineService.cs +++ b/src/Serval/src/Serval.Translation/Services/EngineService.cs @@ -13,7 +13,7 @@ public class EngineService( IDataAccessContext dataAccessContext, ILoggerFactory loggerFactory, IScriptureDataFileService scriptureDataFileService -) : EntityServiceBase(engines), IEngineService +) : OwnedEntityServiceBase(engines), IEngineService { private readonly IRepository _builds = builds; private readonly IRepository _pretranslations = pretranslations; @@ -118,11 +118,6 @@ await client.TrainSegmentPairAsync( ); } - public async Task> GetAllAsync(string owner, CancellationToken cancellationToken = default) - { - return await Entities.GetAllAsync(e => e.Owner == owner, cancellationToken); - } - public override async Task CreateAsync(Engine engine, CancellationToken cancellationToken = default) { bool updateIsModelPersisted = engine.IsModelPersisted is null; diff --git a/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs b/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs index 8643b9fd..615e8c89 100644 --- a/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs +++ b/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs @@ -279,7 +279,7 @@ await _engines.UpdateAsync( } public override async Task InsertPretranslations( - IAsyncStreamReader requestStream, + IAsyncStreamReader requestStream, ServerCallContext context ) { @@ -287,7 +287,7 @@ ServerCallContext context int nextModelRevision = 0; var batch = new List(); - await foreach (InsertPretranslationRequest request in requestStream.ReadAllAsync(context.CancellationToken)) + await foreach (InsertPretranslationsRequest request in requestStream.ReadAllAsync(context.CancellationToken)) { if (request.EngineId != engineId) { diff --git a/src/Serval/src/Serval.Webhooks/Configuration/IMediatorRegistrationConfiguratorExtensions.cs b/src/Serval/src/Serval.Webhooks/Configuration/IMediatorRegistrationConfiguratorExtensions.cs index 7239bf06..37e15a45 100644 --- a/src/Serval/src/Serval.Webhooks/Configuration/IMediatorRegistrationConfiguratorExtensions.cs +++ b/src/Serval/src/Serval.Webhooks/Configuration/IMediatorRegistrationConfiguratorExtensions.cs @@ -8,6 +8,8 @@ this IMediatorRegistrationConfigurator configurator { configurator.AddConsumer(); configurator.AddConsumer(); + configurator.AddConsumer(); + configurator.AddConsumer(); return configurator; } } diff --git a/src/Serval/src/Serval.Webhooks/Consumers/AssessmentJobFinishedConsumer.cs b/src/Serval/src/Serval.Webhooks/Consumers/AssessmentJobFinishedConsumer.cs new file mode 100644 index 00000000..117536f9 --- /dev/null +++ b/src/Serval/src/Serval.Webhooks/Consumers/AssessmentJobFinishedConsumer.cs @@ -0,0 +1,36 @@ +namespace Serval.Webhooks.Consumers; + +public class AssessmentJobFinishedConsumer(IWebhookService webhookService, IUrlService urlService) + : IConsumer +{ + private readonly IWebhookService _webhookService = webhookService; + private readonly IUrlService _urlService = urlService; + + public async Task Consume(ConsumeContext context) + { + await _webhookService.SendEventAsync( + WebhookEvent.AssessmentJobFinished, + context.Message.Owner, + new AssessmentJobFinishedDto + { + Job = new ResourceLinkDto + { + Id = context.Message.JobId, + Url = _urlService.GetUrl( + Endpoints.GetAssessmentJob, + new { id = context.Message.EngineId, jobId = context.Message.JobId } + ) + }, + Engine = new ResourceLinkDto + { + Id = context.Message.EngineId, + Url = _urlService.GetUrl(Endpoints.GetAssessmentEngine, new { id = context.Message.EngineId })! + }, + JobState = context.Message.JobState, + Message = context.Message.Message, + DateFinished = context.Message.DateFinished + }, + context.CancellationToken + ); + } +} diff --git a/src/Serval/src/Serval.Webhooks/Consumers/AssessmentJobStartedConsumer.cs b/src/Serval/src/Serval.Webhooks/Consumers/AssessmentJobStartedConsumer.cs new file mode 100644 index 00000000..06e21d40 --- /dev/null +++ b/src/Serval/src/Serval.Webhooks/Consumers/AssessmentJobStartedConsumer.cs @@ -0,0 +1,33 @@ +namespace Serval.Webhooks.Consumers; + +public class AssessmentJobStartedConsumer(IWebhookService webhookService, IUrlService urlService) + : IConsumer +{ + private readonly IWebhookService _webhookService = webhookService; + private readonly IUrlService _urlService = urlService; + + public async Task Consume(ConsumeContext context) + { + await _webhookService.SendEventAsync( + WebhookEvent.AssessmentJobStarted, + context.Message.Owner, + new AssessmentJobStartedDto + { + Job = new ResourceLinkDto + { + Id = context.Message.JobId, + Url = _urlService.GetUrl( + Endpoints.GetAssessmentJob, + new { id = context.Message.EngineId, jobId = context.Message.JobId } + ) + }, + Engine = new ResourceLinkDto + { + Id = context.Message.EngineId, + Url = _urlService.GetUrl(Endpoints.GetAssessmentEngine, new { id = context.Message.EngineId }) + } + }, + context.CancellationToken + ); + } +} diff --git a/src/Serval/src/Serval.Webhooks/Consumers/TranslationBuildFinishedConsumer.cs b/src/Serval/src/Serval.Webhooks/Consumers/TranslationBuildFinishedConsumer.cs index d208bbc9..648a7a0a 100644 --- a/src/Serval/src/Serval.Webhooks/Consumers/TranslationBuildFinishedConsumer.cs +++ b/src/Serval/src/Serval.Webhooks/Consumers/TranslationBuildFinishedConsumer.cs @@ -17,16 +17,17 @@ await _webhookService.SendEventAsync( { Id = context.Message.BuildId, Url = _urlService.GetUrl( - "GetTranslationBuild", + Endpoints.GetTranslationBuild, new { id = context.Message.EngineId, buildId = context.Message.BuildId } ) }, Engine = new ResourceLinkDto { Id = context.Message.EngineId, - Url = _urlService.GetUrl("GetTranslationEngine", new { id = context.Message.EngineId })! + Url = _urlService.GetUrl(Endpoints.GetTranslationEngine, new { id = context.Message.EngineId })! }, BuildState = context.Message.BuildState, + Message = context.Message.Message, DateFinished = context.Message.DateFinished }, context.CancellationToken diff --git a/src/Serval/src/Serval.Webhooks/Consumers/TranslationBuildStartedConsumer.cs b/src/Serval/src/Serval.Webhooks/Consumers/TranslationBuildStartedConsumer.cs index 36030fcf..182f70c8 100644 --- a/src/Serval/src/Serval.Webhooks/Consumers/TranslationBuildStartedConsumer.cs +++ b/src/Serval/src/Serval.Webhooks/Consumers/TranslationBuildStartedConsumer.cs @@ -17,14 +17,14 @@ await _webhookService.SendEventAsync( { Id = context.Message.BuildId, Url = _urlService.GetUrl( - "GetTranslationBuild", + Endpoints.GetTranslationBuild, new { id = context.Message.EngineId, buildId = context.Message.BuildId } ) }, Engine = new ResourceLinkDto { Id = context.Message.EngineId, - Url = _urlService.GetUrl("GetTranslationEngine", new { id = context.Message.EngineId }) + Url = _urlService.GetUrl(Endpoints.GetTranslationEngine, new { id = context.Message.EngineId }) } }, context.CancellationToken diff --git a/src/Serval/src/Serval.Webhooks/Contracts/WebhookEvent.cs b/src/Serval/src/Serval.Webhooks/Contracts/WebhookEvent.cs index 4864b60b..1dc5d0a4 100644 --- a/src/Serval/src/Serval.Webhooks/Contracts/WebhookEvent.cs +++ b/src/Serval/src/Serval.Webhooks/Contracts/WebhookEvent.cs @@ -3,5 +3,8 @@ public enum WebhookEvent { TranslationBuildStarted, - TranslationBuildFinished + TranslationBuildFinished, + + AssessmentJobStarted, + AssessmentJobFinished } diff --git a/src/Serval/src/Serval.Webhooks/Controllers/WebhooksController.cs b/src/Serval/src/Serval.Webhooks/Controllers/WebhooksController.cs index 39d5cb3f..8758faca 100644 --- a/src/Serval/src/Serval.Webhooks/Controllers/WebhooksController.cs +++ b/src/Serval/src/Serval.Webhooks/Controllers/WebhooksController.cs @@ -32,7 +32,7 @@ public async Task> GetAllAsync(CancellationToken cancell /// The webhook does not exist /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.ReadHooks)] - [HttpGet("{id}", Name = "GetWebhook")] + [HttpGet("{id}", Name = Endpoints.GetWebhook)] [ProducesResponseType(StatusCodes.Status200OK)] [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] @@ -99,7 +99,7 @@ private WebhookDto Map(Webhook source) return new WebhookDto { Id = source.Id, - Url = _urlService.GetUrl("GetWebhook", new { id = source.Id }), + Url = _urlService.GetUrl(Endpoints.GetWebhook, new { id = source.Id }), PayloadUrl = source.Url, Events = source.Events.ToList() }; diff --git a/src/Serval/src/Serval.Webhooks/Services/IWebhookService.cs b/src/Serval/src/Serval.Webhooks/Services/IWebhookService.cs index 30a8e91c..e861d140 100644 --- a/src/Serval/src/Serval.Webhooks/Services/IWebhookService.cs +++ b/src/Serval/src/Serval.Webhooks/Services/IWebhookService.cs @@ -5,7 +5,7 @@ public interface IWebhookService Task> GetAllAsync(string owner, CancellationToken cancellationToken = default); Task GetAsync(string id, CancellationToken cancellationToken = default); - Task CreateAsync(Webhook hook, CancellationToken cancellationToken = default); + Task CreateAsync(Webhook hook, CancellationToken cancellationToken = default); Task DeleteAsync(string id, CancellationToken cancellationToken = default); Task SendEventAsync( diff --git a/src/Serval/src/Serval.Webhooks/Services/WebhookService.cs b/src/Serval/src/Serval.Webhooks/Services/WebhookService.cs index 0bf90c9f..f5450c99 100644 --- a/src/Serval/src/Serval.Webhooks/Services/WebhookService.cs +++ b/src/Serval/src/Serval.Webhooks/Services/WebhookService.cs @@ -1,16 +1,11 @@ namespace Serval.Webhooks.Services; public class WebhookService(IRepository hooks, IBackgroundJobClient jobClient) - : EntityServiceBase(hooks), + : OwnedEntityServiceBase(hooks), IWebhookService { private readonly IBackgroundJobClient _jobClient = jobClient; - public async Task> GetAllAsync(string owner, CancellationToken cancellationToken = default) - { - return await Entities.GetAllAsync(c => c.Owner == owner, cancellationToken); - } - public async Task SendEventAsync( WebhookEvent webhookEvent, string owner, diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/AssessmentEngineTests.cs b/src/Serval/test/Serval.ApiServer.IntegrationTests/AssessmentEngineTests.cs new file mode 100644 index 00000000..328964e2 --- /dev/null +++ b/src/Serval/test/Serval.ApiServer.IntegrationTests/AssessmentEngineTests.cs @@ -0,0 +1,223 @@ +using Google.Protobuf.WellKnownTypes; +using Serval.Assessment.Models; +using Serval.Assessment.V1; +using static Serval.ApiServer.Utils; + +namespace Serval.ApiServer; + +[TestFixture] +public class AssessmentEngineTests +{ + private const string EngineType = "Test"; + private const string ClientId1 = "client1"; + + [Test] + public async Task CreateAsync() + { + using TestEnvironment env = new(); + DataFiles.Models.DataFile dataFile = await env.AddDataFileAsync(); + + AssessmentEnginesClient client = env.CreateClient(); + AssessmentEngine result = await client.CreateAsync( + new() + { + Name = "test", + Type = EngineType, + Corpus = new() { Language = "en", Files = { new() { FileId = dataFile.Id } } } + } + ); + Assert.That(result.Name, Is.EqualTo("test")); + AssessmentEngine? engine = await client.GetAsync(result.Id); + Assert.That(engine, Is.Not.Null); + Assert.That(engine.Name, Is.EqualTo("test")); + } + + [Test] + public async Task StartJobAsync() + { + using TestEnvironment env = new(); + Engine engine = await env.AddEngineAsync(); + + AssessmentEnginesClient client = env.CreateClient(); + AssessmentJob result = await client.StartJobAsync(engine.Id, new() { Name = "test" }); + Assert.That(result.Name, Is.EqualTo("test")); + AssessmentJob? job = await client.GetJobAsync(engine.Id, result.Id); + Assert.That(job, Is.Not.Null); + Assert.That(job.Name, Is.EqualTo("test")); + } + + [Test] + public async Task GetAllResultsAsync() + { + using TestEnvironment env = new(); + Job job = await env.AddJobAsync(); + await env.Results.InsertAllAsync( + [ + new() + { + EngineRef = job.EngineRef, + JobRef = job.Id, + TextId = "text1", + Ref = "1" + }, + new() + { + EngineRef = job.EngineRef, + JobRef = job.Id, + TextId = "text2", + Ref = "2" + } + ] + ); + + AssessmentEnginesClient client = env.CreateClient(); + + IList results = await client.GetAllResultsAsync(job.EngineRef, job.Id); + Assert.That(results, Has.Count.EqualTo(2)); + + results = await client.GetAllResultsAsync(job.EngineRef, job.Id, textId: "text1"); + Assert.That(results, Has.Count.EqualTo(1)); + Assert.That(results[0].Ref, Is.EqualTo("1")); + } + + private class TestEnvironment : DisposableBase + { + private readonly IServiceScope _scope; + private readonly MongoClient _mongoClient; + + public TestEnvironment() + { + MongoClientSettings clientSettings = new() { LinqProvider = LinqProvider.V2 }; + _mongoClient = new MongoClient(clientSettings); + ResetDatabases(); + + Factory = new ServalWebApplicationFactory(); + _scope = Factory.Services.CreateScope(); + Engines = _scope.ServiceProvider.GetRequiredService>(); + DataFiles = _scope.ServiceProvider.GetRequiredService>(); + Results = _scope.ServiceProvider.GetRequiredService>(); + Jobs = _scope.ServiceProvider.GetRequiredService>(); + + Client = Substitute.For(); + Client + .CreateAsync(Arg.Any(), null, null, Arg.Any()) + .Returns(CreateAsyncUnaryCall(new Empty())); + Client + .DeleteAsync(Arg.Any(), null, null, Arg.Any()) + .Returns(CreateAsyncUnaryCall(new Empty())); + Client + .StartJobAsync(Arg.Any(), null, null, Arg.Any()) + .Returns(CreateAsyncUnaryCall(new Empty())); + Client + .CancelJobAsync(Arg.Any(), null, null, Arg.Any()) + .Returns(CreateAsyncUnaryCall(new Empty())); + Client + .DeleteAsync(Arg.Any(), null, null, Arg.Any()) + .Returns(CreateAsyncUnaryCall(new Empty())); + } + + public ServalWebApplicationFactory Factory { get; } + public IRepository Engines { get; } + public IRepository DataFiles { get; } + public IRepository Results { get; } + public IRepository Jobs { get; } + public AssessmentEngineApi.AssessmentEngineApiClient Client { get; } + + public AssessmentEnginesClient CreateClient(IEnumerable? scope = null) + { + scope ??= + [ + Scopes.CreateAssessmentEngines, + Scopes.ReadAssessmentEngines, + Scopes.UpdateAssessmentEngines, + Scopes.DeleteAssessmentEngines + ]; + HttpClient httpClient = Factory + .WithWebHostBuilder(builder => + { + builder.ConfigureTestServices(services => + { + GrpcClientFactory grpcClientFactory = Substitute.For(); + grpcClientFactory + .CreateClient(EngineType) + .Returns(Client); + services.AddSingleton(grpcClientFactory); + }); + }) + .CreateClient(); + httpClient.DefaultRequestHeaders.Add("Scope", string.Join(" ", scope)); + return new AssessmentEnginesClient(httpClient); + } + + public async Task AddDataFileAsync() + { + DataFiles.Models.DataFile dataFile = + new() + { + Owner = ClientId1, + Format = Shared.Contracts.FileFormat.Paratext, + Id = "f00000000000000000000001", + Name = "file1.zip", + Filename = "file1.zip" + }; + await DataFiles.InsertAsync(dataFile); + return dataFile; + } + + public async Task AddEngineAsync() + { + DataFiles.Models.DataFile dataFile = await AddDataFileAsync(); + Engine engine = + new() + { + Owner = ClientId1, + Type = EngineType, + Corpus = new() + { + Language = "en", + Files = + [ + new() + { + Id = dataFile.Id, + Format = Shared.Contracts.FileFormat.Paratext, + Filename = dataFile.Filename, + TextId = "all" + } + ] + }, + }; + await Engines.InsertAsync(engine); + return engine; + } + + public async Task AddJobAsync() + { + Engine engine = await AddEngineAsync(); + Job job = + new() + { + Name = "test", + EngineRef = engine.Id, + State = Shared.Contracts.JobState.Completed, + Message = "Completed", + DateFinished = DateTime.UtcNow + }; + await Jobs.InsertAsync(job); + return job; + } + + public void ResetDatabases() + { + _mongoClient.DropDatabase("serval_test"); + _mongoClient.DropDatabase("serval_test_jobs"); + } + + protected override void DisposeManagedResources() + { + _scope.Dispose(); + Factory.Dispose(); + ResetDatabases(); + } + } +} diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs b/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs index c6c93dbc..f27c6880 100644 --- a/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs +++ b/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs @@ -1,5 +1,7 @@ using Google.Protobuf.WellKnownTypes; +using Serval.Translation.Models; using Serval.Translation.V1; +using static Serval.ApiServer.Utils; namespace Serval.ApiServer; @@ -1357,29 +1359,6 @@ public void TearDown() _env.Dispose(); } - private static AsyncUnaryCall CreateAsyncUnaryCall(StatusCode statusCode) - { - var status = new Status(statusCode, string.Empty); - return new AsyncUnaryCall( - Task.FromException(new RpcException(status)), - Task.FromResult(new Metadata()), - () => status, - () => [], - () => { } - ); - } - - private static AsyncUnaryCall CreateAsyncUnaryCall(TResponse response) - { - return new AsyncUnaryCall( - Task.FromResult(response), - Task.FromResult(new Metadata()), - () => Status.DefaultSuccess, - () => [], - () => { } - ); - } - private class TestEnvironment : DisposableBase { private readonly IServiceScope _scope; diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/Usings.cs b/src/Serval/test/Serval.ApiServer.IntegrationTests/Usings.cs index fa43f490..09fc9f85 100644 --- a/src/Serval/test/Serval.ApiServer.IntegrationTests/Usings.cs +++ b/src/Serval/test/Serval.ApiServer.IntegrationTests/Usings.cs @@ -20,6 +20,5 @@ global using Serval.Shared.Configuration; global using Serval.Shared.Controllers; global using Serval.Shared.Services; -global using Serval.Translation.Models; global using SIL.DataAccess; global using SIL.ObjectModel; diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/Utils.cs b/src/Serval/test/Serval.ApiServer.IntegrationTests/Utils.cs new file mode 100644 index 00000000..fd571f51 --- /dev/null +++ b/src/Serval/test/Serval.ApiServer.IntegrationTests/Utils.cs @@ -0,0 +1,27 @@ +namespace Serval.ApiServer; + +public static class Utils +{ + public static AsyncUnaryCall CreateAsyncUnaryCall(StatusCode statusCode) + { + var status = new Status(statusCode, string.Empty); + return new AsyncUnaryCall( + Task.FromException(new RpcException(status)), + Task.FromResult(new Metadata()), + () => status, + () => [], + () => { } + ); + } + + public static AsyncUnaryCall CreateAsyncUnaryCall(TResponse response) + { + return new AsyncUnaryCall( + Task.FromResult(response), + Task.FromResult(new Metadata()), + () => Status.DefaultSuccess, + () => [], + () => { } + ); + } +} diff --git a/src/Serval/test/Serval.Shared.Tests/Services/ScriptureDataFileServiceTests.cs b/src/Serval/test/Serval.Shared.Tests/Services/ScriptureDataFileServiceTests.cs index be7481ec..b4dc6841 100644 --- a/src/Serval/test/Serval.Shared.Tests/Services/ScriptureDataFileServiceTests.cs +++ b/src/Serval/test/Serval.Shared.Tests/Services/ScriptureDataFileServiceTests.cs @@ -17,9 +17,9 @@ public void GetZipParatextProjectTextUpdater() TestEnvironment env = new(); using ZipParatextProjectTextUpdater updater = env.Service.GetZipParatextProjectTextUpdater("file1.zip"); Assert.That( - updater.UpdateUsfm("MAT", [], preferExistingText: true), + updater.UpdateUsfm("MAT", [], preferExistingText: true).ReplaceLineEndings("\n"), Is.EqualTo( - $@"\id MAT - PROJ + $@"\id MAT - PROJ \h {Canon.BookIdToEnglishName("MAT")} \c 1 \p @@ -29,8 +29,9 @@ public void GetZipParatextProjectTextUpdater() \p \v 1 Chapter two, verse one. \v 2 Chapter two, verse two. -".Replace("\n", "\r\n") - ) +" + ) + .IgnoreLineEndings() ); } diff --git a/src/Serval/test/Serval.Shared.Tests/Usings.cs b/src/Serval/test/Serval.Shared.Tests/Usings.cs index b68ab424..57001f09 100644 --- a/src/Serval/test/Serval.Shared.Tests/Usings.cs +++ b/src/Serval/test/Serval.Shared.Tests/Usings.cs @@ -4,6 +4,8 @@ global using Microsoft.Extensions.Options; global using NSubstitute; global using NUnit.Framework; +global using NUnit.Framework.Constraints; global using Serval.Shared.Configuration; +global using Serval.Shared.Utils; global using SIL.Machine.Corpora; global using SIL.Scripture; diff --git a/src/Serval/test/Serval.Translation.Tests/Services/NUnitExtensions.cs b/src/Serval/test/Serval.Shared.Tests/Utils/NUnitExtensions.cs similarity index 87% rename from src/Serval/test/Serval.Translation.Tests/Services/NUnitExtensions.cs rename to src/Serval/test/Serval.Shared.Tests/Utils/NUnitExtensions.cs index 1b138714..853c7416 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/NUnitExtensions.cs +++ b/src/Serval/test/Serval.Shared.Tests/Utils/NUnitExtensions.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Services; +namespace Serval.Shared.Utils; public static class NUnitExtensions { diff --git a/src/Serval/test/Serval.Translation.Tests/Serval.Translation.Tests.csproj b/src/Serval/test/Serval.Translation.Tests/Serval.Translation.Tests.csproj index 25544c8d..3f6406b2 100644 --- a/src/Serval/test/Serval.Translation.Tests/Serval.Translation.Tests.csproj +++ b/src/Serval/test/Serval.Translation.Tests/Serval.Translation.Tests.csproj @@ -33,6 +33,7 @@ + diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs index cfa294a4..10f3ca14 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs @@ -155,12 +155,12 @@ public TestEnvironment() public TranslationPlatformServiceV1 PlatformService { get; } } - private class MockAsyncStreamReader(string engineId) : IAsyncStreamReader + private class MockAsyncStreamReader(string engineId) : IAsyncStreamReader { private bool _endOfStream = false; public string EngineId { get; } = engineId; - public InsertPretranslationRequest Current => new() { EngineId = EngineId }; + public InsertPretranslationsRequest Current => new() { EngineId = EngineId }; public Task MoveNext(CancellationToken cancellationToken) { diff --git a/src/Serval/test/Serval.Translation.Tests/Usings.cs b/src/Serval/test/Serval.Translation.Tests/Usings.cs index 74d21283..ef8a3ff7 100644 --- a/src/Serval/test/Serval.Translation.Tests/Usings.cs +++ b/src/Serval/test/Serval.Translation.Tests/Usings.cs @@ -6,7 +6,6 @@ global using Microsoft.Extensions.Options; global using NSubstitute; global using NUnit.Framework; -global using NUnit.Framework.Constraints; global using Serval.Shared.Configuration; global using Serval.Shared.Services; global using Serval.Shared.Utils; diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/IHealthChecksBuilderExtensions.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/IHealthChecksBuilderExtensions.cs new file mode 100644 index 00000000..83fd6a21 --- /dev/null +++ b/src/ServiceToolkit/src/SIL.ServiceToolkit/Configuration/IHealthChecksBuilderExtensions.cs @@ -0,0 +1,12 @@ +using SIL.ServiceToolkit.Services; + +namespace Microsoft.Extensions.DependencyInjection; + +public static class IHealthChecksBuilderExtensions +{ + public static IHealthChecksBuilder AddHangfire(this IHealthChecksBuilder builder, string name = "Hangfire") + { + builder.AddCheck(name); + return builder; + } +} diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/SIL.ServiceToolkit.csproj b/src/ServiceToolkit/src/SIL.ServiceToolkit/SIL.ServiceToolkit.csproj new file mode 100644 index 00000000..b5cb78cd --- /dev/null +++ b/src/ServiceToolkit/src/SIL.ServiceToolkit/SIL.ServiceToolkit.csproj @@ -0,0 +1,20 @@ + + + + net8.0 + enable + enable + + + + + + + + + + + + + + diff --git a/src/Machine/src/Serval.Machine.Shared/Services/CancellationInterceptor.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/CancellationInterceptor.cs similarity index 95% rename from src/Machine/src/Serval.Machine.Shared/Services/CancellationInterceptor.cs rename to src/ServiceToolkit/src/SIL.ServiceToolkit/Services/CancellationInterceptor.cs index 73b06c2b..c33f509b 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/CancellationInterceptor.cs +++ b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/CancellationInterceptor.cs @@ -1,4 +1,4 @@ -namespace Serval.Machine.Shared.Services; +namespace SIL.ServiceToolkit.Services; public class CancellationInterceptor(ILogger logger) : Interceptor { diff --git a/src/Machine/src/Serval.Machine.Shared/Services/HangfireHealthCheck.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/HangfireHealthCheck.cs similarity index 94% rename from src/Machine/src/Serval.Machine.Shared/Services/HangfireHealthCheck.cs rename to src/ServiceToolkit/src/SIL.ServiceToolkit/Services/HangfireHealthCheck.cs index c3c14751..0beeff33 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/HangfireHealthCheck.cs +++ b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/HangfireHealthCheck.cs @@ -1,4 +1,4 @@ -namespace Serval.Machine.Shared.Services; +namespace SIL.ServiceToolkit.Services; public class HangfireHealthCheck(JobStorage jobStorage, IOptions options) : IHealthCheck { diff --git a/src/Machine/src/Serval.Machine.Shared/Utils/RecurrentTask.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/RecurrentTask.cs similarity index 96% rename from src/Machine/src/Serval.Machine.Shared/Utils/RecurrentTask.cs rename to src/ServiceToolkit/src/SIL.ServiceToolkit/Services/RecurrentTask.cs index 2e2f91ec..b220e9a4 100644 --- a/src/Machine/src/Serval.Machine.Shared/Utils/RecurrentTask.cs +++ b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/RecurrentTask.cs @@ -1,4 +1,4 @@ -namespace Serval.Machine.Shared.Utils; +namespace SIL.ServiceToolkit.Services; public abstract class RecurrentTask( string serviceName, diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Usings.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Usings.cs new file mode 100644 index 00000000..0d9630d6 --- /dev/null +++ b/src/ServiceToolkit/src/SIL.ServiceToolkit/Usings.cs @@ -0,0 +1,12 @@ +global using System.Diagnostics.CodeAnalysis; +global using System.Text.Json.Nodes; +global using System.Text.RegularExpressions; +global using Grpc.Core; +global using Grpc.Core.Interceptors; +global using Hangfire; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Diagnostics.HealthChecks; +global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; +global using SIL.WritingSystems; diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Utils/LanguageTagParser.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Utils/LanguageTagParser.cs new file mode 100644 index 00000000..62b3594f --- /dev/null +++ b/src/ServiceToolkit/src/SIL.ServiceToolkit/Utils/LanguageTagParser.cs @@ -0,0 +1,132 @@ +namespace SIL.ServiceToolkit.Utils; + +public partial class LanguageTagParser +{ + private static readonly Dictionary StandardLanguages = + new() + { + { "ar", "arb" }, + { "ms", "zsm" }, + { "lv", "lvs" }, + { "ne", "npi" }, + { "sw", "swh" }, + { "cmn", "zh" } + }; + + private static readonly Dictionary StandardScripts = new() { { "Kore", "Hang" } }; + + private readonly Dictionary _defaultScripts; + + [GeneratedRegex("(?'language'[a-zA-Z]{2,8})([_-](?'script'[a-zA-Z]{4}))?", RegexOptions.ExplicitCapture)] + private static partial Regex LangTagPattern(); + + public LanguageTagParser() + { + // initialize SLDR language tags to retrieve latest langtags.json file + _defaultScripts = InitializeDefaultScripts(); + } + + private static Dictionary InitializeDefaultScripts() + { + Sldr.InitializeLanguageTags(); + string cachedAllTagsPath = Path.Combine(Sldr.SldrCachePath, "langtags.json"); + + if (!File.Exists(cachedAllTagsPath)) + { + using HttpClient client = new(); + using HttpResponseMessage response = client.Send( + new HttpRequestMessage( + HttpMethod.Get, + "https://raw.githubusercontent.com/silnrsi/langtags/master/pub/langtags.json" + ) + ); + response.EnsureSuccessStatusCode(); + using Stream responseStream = response.Content.ReadAsStream(); + using FileStream fileStream = new(cachedAllTagsPath, FileMode.Create); + responseStream.CopyTo(fileStream); + } + using FileStream stream = new(cachedAllTagsPath, FileMode.Open); + var json = JsonNode.Parse(stream); + + Dictionary defaultScripts = []; + foreach (JsonNode? entry in json!.AsArray()) + { + if (entry is null) + continue; + + var script = (string?)entry["script"]; + if (script is null) + continue; + + JsonNode? tags = entry["tags"]; + if (tags is not null) + { + foreach (var t in tags.AsArray().Select(v => (string?)v)) + { + if ( + t is not null + && IetfLanguageTag.TryGetParts(t, out _, out string? s, out _, out _) + && s is null + ) + { + defaultScripts[t] = script; + } + } + } + + var tag = (string?)entry["tag"]; + if (tag is not null) + defaultScripts[tag] = script; + } + return defaultScripts; + } + + public bool TryParse( + string languageTag, + [MaybeNullWhen(false)] out string languageCode, + [MaybeNullWhen(false)] out string scriptCode + ) + { + languageCode = null; + scriptCode = null; + + // Try to find a pattern of {language code}_{script} + Match langTagMatch = LangTagPattern().Match(languageTag); + if (!langTagMatch.Success) + return false; + + string parsedLanguage = langTagMatch.Groups["language"].Value; + string languageSubtag = parsedLanguage; + languageCode = parsedLanguage; + + // Best attempt to convert language to a registered ISO 639-3 code + // Uses https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry for mapping + + // If they gave us the ISO code, revert it to the 2 character code + if (StandardSubtags.TryGetLanguageFromIso3Code(languageSubtag, out LanguageSubtag tempSubtag)) + languageSubtag = tempSubtag.Code; + + // There are a few extra conversions not in SIL Writing Systems that we need to handle + if (StandardLanguages.TryGetValue(languageSubtag, out string? tempName)) + languageSubtag = tempName; + + if (StandardSubtags.RegisteredLanguages.TryGet(languageSubtag, out LanguageSubtag? languageSubtagObj)) + languageCode = languageSubtagObj.Iso3Code; + + // Use default script unless there is one parsed out of the language tag + Group scriptGroup = langTagMatch.Groups["script"]; + + if (scriptGroup.Success) + scriptCode = scriptGroup.Value; + else if (_defaultScripts.TryGetValue(languageTag, out string? tempScript2)) + scriptCode = tempScript2; + else if (_defaultScripts.TryGetValue(languageSubtag, out string? tempScript)) + scriptCode = tempScript; + + // There are a few extra conversions not in SIL Writing Systems that we need to handle + if (scriptCode is not null && StandardScripts.TryGetValue(scriptCode, out string? tempScript3)) + scriptCode = tempScript3; + + return scriptCode is not null; + } +} From 19ab4da21198110bad9a5b8241b3dfa66b61541a Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Thu, 22 Aug 2024 12:49:46 -0400 Subject: [PATCH 13/15] Fix project dependency mismerge --- .../src/Serval.Machine.Shared/Serval.Machine.Shared.csproj | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj b/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj index 8397d98d..4ea74e68 100644 --- a/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj +++ b/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj @@ -36,11 +36,10 @@ - - + @@ -58,4 +57,4 @@ - + \ No newline at end of file From c9d3784bf97a352aae238d4cf1ae74007d03df9a Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Thu, 22 Aug 2024 12:55:21 -0400 Subject: [PATCH 14/15] Fix dependency mismerge --- src/Serval/src/Serval.Shared/Serval.Shared.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Serval/src/Serval.Shared/Serval.Shared.csproj b/src/Serval/src/Serval.Shared/Serval.Shared.csproj index c464aa9b..87217c39 100644 --- a/src/Serval/src/Serval.Shared/Serval.Shared.csproj +++ b/src/Serval/src/Serval.Shared/Serval.Shared.csproj @@ -20,6 +20,7 @@ + From e808ca03723f7fcdfbbd3a7d1b82eba8d553d396 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Thu, 22 Aug 2024 13:55:26 -0400 Subject: [PATCH 15/15] Adjust test to new terms inclusion --- .../Services/PreprocessBuildJobTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs index 08b6d414..65ed5fef 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs @@ -104,7 +104,7 @@ public async Task RunAsync_EnableKeyTerms() Assert.That(src1Count, Is.EqualTo(0)); Assert.That(src2Count, Is.EqualTo(0)); Assert.That(trgCount, Is.EqualTo(0)); - Assert.That(termCount, Is.EqualTo(1)); + Assert.That(termCount, Is.EqualTo(5726)); }); }