diff --git a/Serval.sln b/Serval.sln index c850094d..b0db17fb 100644 --- a/Serval.sln +++ b/Serval.sln @@ -52,7 +52,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.E2ETests", "tests\Se EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.DataFiles.Tests", "tests\Serval.DataFiles.Tests\Serval.DataFiles.Tests.csproj", "{63E4D71B-11BE-4D68-A876-5B1B5F0A4C88}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SIL.DataAccess.Tests", "tests\SIL.DataAccess.Tests\SIL.DataAccess.Tests.csproj", "{71151518-8774-44D0-8E69-D77FA447BEFA}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.DataAccess.Tests", "tests\SIL.DataAccess.Tests\SIL.DataAccess.Tests.csproj", "{71151518-8774-44D0-8E69-D77FA447BEFA}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serval.Shared.Tests", "tests\Serval.Shared.Tests\Serval.Shared.Tests.csproj", "{0E220C65-AA88-450E-AFB2-844E49060B3F}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -120,6 +122,10 @@ Global {71151518-8774-44D0-8E69-D77FA447BEFA}.Debug|Any CPU.Build.0 = Debug|Any CPU {71151518-8774-44D0-8E69-D77FA447BEFA}.Release|Any CPU.ActiveCfg = Release|Any CPU {71151518-8774-44D0-8E69-D77FA447BEFA}.Release|Any CPU.Build.0 = Release|Any CPU + {0E220C65-AA88-450E-AFB2-844E49060B3F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0E220C65-AA88-450E-AFB2-844E49060B3F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0E220C65-AA88-450E-AFB2-844E49060B3F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0E220C65-AA88-450E-AFB2-844E49060B3F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -140,6 +146,7 @@ Global {1F020042-D7B8-4541-9691-26ECFD1FFC73} = {66246A1C-8D45-40FB-A660-C58577122CA7} {63E4D71B-11BE-4D68-A876-5B1B5F0A4C88} = {66246A1C-8D45-40FB-A660-C58577122CA7} {71151518-8774-44D0-8E69-D77FA447BEFA} = {66246A1C-8D45-40FB-A660-C58577122CA7} + {0E220C65-AA88-450E-AFB2-844E49060B3F} = {66246A1C-8D45-40FB-A660-C58577122CA7} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9F18C25E-E140-43C3-B177-D562E1628370} diff --git a/src/Serval.ApiServer/Templates/Client.Class.ProcessResponse.liquid b/src/Serval.ApiServer/Templates/Client.Class.ProcessResponse.liquid new file mode 100644 index 00000000..603ac78a --- /dev/null +++ b/src/Serval.ApiServer/Templates/Client.Class.ProcessResponse.liquid @@ -0,0 +1,72 @@ +{% if response.HasType -%} +{% if response.IsFile -%} +{% if response.IsSuccess -%} +var responseStream_ = response_.Content == null ? System.IO.Stream.Null : await response_.Content.ReadAsStreamAsync().ConfigureAwait(false); +var fileResponse_ = new FileResponse(status_, headers_, responseStream_, {% if InjectHttpClient or DisposeHttpClient == false %}null{% else %}client_{% endif %}, response_); +disposeClient_ = false; disposeResponse_ = false; // response and client are disposed by FileResponse +return fileResponse_; +{% else -%} +var objectResponse_ = await ReadObjectResponseAsync<{{ response.Type }}>(response_, headers_, cancellationToken).ConfigureAwait(false); +throw new {{ ExceptionClass }}<{{ response.Type }}>("{{ response.ExceptionDescription }}", status_, objectResponse_.Text, headers_, objectResponse_.Object, null); +{% endif -%} +{% elsif response.IsPlainText or operation.Produces == "text/plain" -%} +var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); +var result_ = ({{ response.Type }})System.Convert.ChangeType(responseData_, typeof({{ response.Type }})); +{% if response.IsSuccess -%} +{% if operation.WrapResponse -%} +return new {{ ResponseClass }}<{{ operation.UnwrappedResultType }}>(status_, headers_, result_); +{% else -%} +return result_; +{% endif -%} +{% else -%} +throw new {{ ExceptionClass }}<{{ response.Type }}>("{{ response.ExceptionDescription }}", status_, responseData_, headers_, result_, null); +{% endif -%} +{% else -%} +var objectResponse_ = await ReadObjectResponseAsync<{{ response.Type }}>(response_, headers_, cancellationToken).ConfigureAwait(false); +{% if response.IsNullable == false -%} +if (objectResponse_.Object == null) +{ + throw new {{ ExceptionClass }}("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); +} +{% endif -%} +{% if response.IsSuccess -%} +{% if operation.WrapResponse -%} +return new {{ ResponseClass }}<{{ operation.UnwrappedResultType }}>(status_, headers_, objectResponse_.Object); +{% else -%} +return objectResponse_.Object; +{% endif -%} +{% endif -%} +{% if response.IsSuccess == false -%} +{% if response.InheritsExceptionSchema -%} +var responseObject_ = objectResponse_.Object != null ? objectResponse_.Object : new {{ response.Type }}(); +responseObject_.Data.Add("HttpStatus", status_.ToString()); +responseObject_.Data.Add("HttpHeaders", headers_); +responseObject_.Data.Add("HttpResponse", objectResponse_.Text); +{% if WrapDtoExceptions -%} +throw new {{ ExceptionClass }}("{{ response.ExceptionDescription }}", status_, objectResponse_.Text, headers_, responseObject_); +{% else -%} +throw responseObject_; +{% endif -%} +{% else -%} +throw new {{ ExceptionClass }}<{{ response.Type }}>("{{ response.ExceptionDescription }}", status_, objectResponse_.Text, headers_, objectResponse_.Object, null); +{% endif -%} +{% endif -%} +{% endif -%} +{% elsif response.IsSuccess -%} +{% if operation.HasResultType -%} +{% if operation.WrapResponse -%} +return new {{ ResponseClass }}<{{ operation.UnwrappedResultType }}>(status_, headers_, {{ operation.UnwrappedResultDefaultValue }}); +{% else -%} +return {{ operation.UnwrappedResultDefaultValue }}; +{% endif -%} +{% else -%} +{% if operation.WrapResponse -%} +return new {{ ResponseClass }}(status_, headers_); +{% else -%} +return; +{% endif -%} +{% endif -%} +{% else -%}{% comment %} implied: `if !response.HasType` so just read it as text {% endcomment %} +string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); +throw new {{ ExceptionClass }}("{{ response.ExceptionDescription }}", status_, responseText_, headers_, null); +{% endif %} \ No newline at end of file diff --git a/src/Serval.ApiServer/nswag.json b/src/Serval.ApiServer/nswag.json index a81c442e..615e17bf 100644 --- a/src/Serval.ApiServer/nswag.json +++ b/src/Serval.ApiServer/nswag.json @@ -4,6 +4,7 @@ "documentGenerator": { "aspNetCoreToOpenApi": { "project": "Serval.ApiServer.csproj", + "documentName": "v1", "msBuildProjectExtensionsPath": null, "configuration": null, "runtime": null, @@ -12,52 +13,9 @@ "msBuildOutputPath": null, "verbose": true, "workingDirectory": null, - "requireParametersWithoutDefault": false, - "apiGroupNames": null, - "defaultPropertyNameHandling": "Default", - "defaultReferenceTypeNullHandling": "Null", - "defaultDictionaryValueReferenceTypeNullHandling": "NotNull", - "defaultResponseReferenceTypeNullHandling": "NotNull", - "generateOriginalParameterNames": true, - "defaultEnumHandling": "Integer", - "flattenInheritanceHierarchy": false, - "generateKnownTypes": true, - "generateEnumMappingDescription": false, - "generateXmlObjects": false, - "generateAbstractProperties": false, - "generateAbstractSchemas": true, - "ignoreObsoleteProperties": false, - "allowReferencesWithProperties": false, - "useXmlDocumentation": true, - "resolveExternalXmlDocumentation": true, - "excludedTypeNames": [], - "serviceHost": null, - "serviceBasePath": null, - "serviceSchemes": [], - "infoTitle": "My Title", - "infoDescription": null, - "infoVersion": "1.0.0", - "documentTemplate": null, - "documentProcessorTypes": [], - "operationProcessorTypes": [], - "typeNameGeneratorType": null, - "schemaNameGeneratorType": null, - "contractResolverType": null, - "serializerSettingsType": null, - "useDocumentProvider": true, - "documentName": "v1", "aspNetCoreEnvironment": "Development", - "createWebHostBuilderMethod": null, - "startupType": null, - "allowNullableBodyParameters": true, - "useHttpAttributeNameAsOperationId": false, "output": null, - "outputType": "Swagger2", - "newLineBehavior": "Auto", - "assemblyPaths": [], - "assemblyConfig": null, - "referencePaths": [], - "useNuGetCache": false + "newLineBehavior": "Auto" } }, "codeGenerators": { @@ -65,7 +23,9 @@ "clientBaseClass": null, "configurationClass": null, "generateClientClasses": true, + "suppressClientClassesOutput": false, "generateClientInterfaces": true, + "suppressClientInterfacesOutput": false, "clientBaseInterface": null, "injectHttpClient": true, "disposeHttpClient": true, @@ -83,6 +43,8 @@ "exposeJsonSerializerSettings": false, "clientClassAccessModifier": "public", "typeAccessModifier": "public", + "propertySetterAccessModifier": "", + "generateNativeRecords": false, "generateContractsOutput": false, "contractsNamespace": null, "contractsOutputFilePath": null, @@ -138,10 +100,7 @@ "generateDtoTypes": true, "generateOptionalPropertiesAsNullable": false, "generateNullableReferenceTypes": true, - "templateDirectory": null, - "typeNameGeneratorType": null, - "propertyNameGeneratorType": null, - "enumNameGeneratorType": null, + "templateDirectory": "Templates", "serviceHost": null, "serviceSchemes": null, "output": "../Serval.Client/Client.g.cs", diff --git a/src/Serval.Client/Client.g.cs b/src/Serval.Client/Client.g.cs index 8d5808c2..65b76d18 100644 --- a/src/Serval.Client/Client.g.cs +++ b/src/Serval.Client/Client.g.cs @@ -1382,7 +1382,7 @@ public partial interface ITranslationEnginesClient ///
} /// /// The translation engine configuration (see above) - /// The translation engine was created successfully + /// The new translation engine /// A server side error occurred. System.Threading.Tasks.Task CreateAsync(TranslationEngineConfig engineConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); @@ -1400,7 +1400,7 @@ public partial interface ITranslationEnginesClient /// Delete a translation engine /// /// The translation engine id - /// The engine was successfully deleted + /// The engine was successfully deleted. /// A server side error occurred. System.Threading.Tasks.Task DeleteAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); @@ -1446,7 +1446,7 @@ public partial interface ITranslationEnginesClient /// /// The translation engine id /// The segment pair - /// The engine was trained successfully + /// The engine was trained successfully. /// A server side error occurred. System.Threading.Tasks.Task TrainSegmentAsync(string id, SegmentPair segmentPair, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); @@ -1473,7 +1473,7 @@ public partial interface ITranslationEnginesClient /// /// The translation engine id /// The corpus configuration (see remarks) - /// The corpus was added successfully + /// The added corpus /// A server side error occurred. System.Threading.Tasks.Task AddCorpusAsync(string id, TranslationCorpusConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); @@ -1491,8 +1491,8 @@ public partial interface ITranslationEnginesClient /// Update a corpus with a new set of files /// /// - /// See posting a new corpus for details of use. Will completely replace corpus' file associations. - ///
Will not affect jobs already queued or running. Will not affect existing pretranslations until new build is complete. + /// See posting a new corpus for details of use. Will completely replace corpus' file associations. + ///
Will not affect jobs already queued or running. Will not affect existing pretranslations until new build is complete. /// /// The translation engine id /// The corpus id @@ -1520,7 +1520,7 @@ public partial interface ITranslationEnginesClient /// /// The translation engine id /// The corpus id - /// The data file was deleted successfully + /// The data file was deleted successfully. /// A server side error occurred. System.Threading.Tasks.Task DeleteCorpusAsync(string id, string corpusId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); @@ -1545,6 +1545,41 @@ public partial interface ITranslationEnginesClient /// A server side error occurred. System.Threading.Tasks.Task> GetAllPretranslationsAsync(string id, string corpusId, string? textId = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get all pretranslations for the specified text in a corpus of a translation engine + /// + /// + /// Pretranslations are arranged in a list of dictionaries with the following fields per pretranslation: + ///
* **TextId**: The TextId of the SourceFile defined when the corpus was created. + ///
* **Refs** (a list of strings): A list of references including: + ///
* The references defined in the SourceFile per line, if any. + ///
* An auto-generated reference of `[TextId]:[lineNumber]`, 1 indexed. + ///
* **Translation**: the text of the pretranslation + /// + /// The translation engine id + /// The corpus id + /// The text id + /// The pretranslations + /// A server side error occurred. + System.Threading.Tasks.Task> GetPretranslationsByTextIdAsync(string id, string corpusId, string textId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get a pretranslated Scripture book in USFM format. + /// + /// + /// If the USFM book exists in the target corpus, then the pretranslated text will be inserted into any empty + ///
segments in the the target book and returned. If the USFM book does not exist in the target corpus, then the + ///
pretranslated text will be inserted into an empty template created from the source USFM book and returned. + /// + /// The translation engine id + /// The corpus id + /// The text id + /// The book in USFM format + /// A server side error occurred. + System.Threading.Tasks.Task GetPretranslatedUsfmAsync(string id, string corpusId, string textId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// /// Get all build jobs for a translation engine @@ -1576,7 +1611,7 @@ public partial interface ITranslationEnginesClient /// /// The translation engine id /// The build config (see remarks) - /// The build job was started successfully + /// The new build job /// A server side error occurred. System.Threading.Tasks.Task StartBuildAsync(string id, TranslationBuildConfig buildConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); @@ -1619,7 +1654,7 @@ public partial interface ITranslationEnginesClient /// Cancel the current build job (whether pending or active) for a translation engine /// /// The translation engine id - /// The build job was cancelled successfully + /// The build job was cancelled successfully. /// A server side error occurred. System.Threading.Tasks.Task CancelBuildAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); @@ -1724,19 +1759,19 @@ public string BaseUrl if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation.", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -1790,7 +1825,7 @@ public string BaseUrl ///
} /// /// The translation engine configuration (see above) - /// The translation engine was created successfully + /// The new translation engine /// A server side error occurred. public virtual async System.Threading.Tasks.Task CreateAsync(TranslationEngineConfig engineConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { @@ -1851,31 +1886,25 @@ public string BaseUrl if (status_ == 400) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("Bad request. Is the engine type correct?", status_, responseText_, headers_, null); + throw new ServalApiException("Bad request. Is the engine type correct?", status_, responseText_, headers_, null); } else if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine", status_, responseText_, headers_, null); - } - else - if (status_ == 422) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine was specified incorrectly. Did you use the same language for the source and target?", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -1960,25 +1989,25 @@ public string BaseUrl if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -2005,7 +2034,7 @@ public string BaseUrl /// Delete a translation engine /// /// The translation engine id - /// The engine was successfully deleted + /// The engine was successfully deleted. /// A server side error occurred. public virtual async System.Threading.Tasks.Task DeleteAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { @@ -2057,25 +2086,25 @@ public string BaseUrl if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist and therefore cannot be deleted", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist and therefore cannot be deleted.", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -2175,37 +2204,37 @@ public string BaseUrl if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); } else if (status_ == 405) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The method is not supported", status_, responseText_, headers_, null); + throw new ServalApiException("The method is not supported.", status_, responseText_, headers_, null); } else if (status_ == 409) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine needs to be built before it can translate segments", status_, responseText_, headers_, null); + throw new ServalApiException("The engine needs to be built before it can translate segments.", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -2310,37 +2339,37 @@ public string BaseUrl if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); } else if (status_ == 405) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The method is not supported", status_, responseText_, headers_, null); + throw new ServalApiException("The method is not supported.", status_, responseText_, headers_, null); } else if (status_ == 409) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine needs to be built before it can translate segments", status_, responseText_, headers_, null); + throw new ServalApiException("The engine needs to be built before it can translate segments.", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -2440,37 +2469,37 @@ public string BaseUrl if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); } else if (status_ == 405) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The method is not supported", status_, responseText_, headers_, null); + throw new ServalApiException("The method is not supported.", status_, responseText_, headers_, null); } else if (status_ == 409) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine needs to be built first", status_, responseText_, headers_, null); + throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -2503,7 +2532,7 @@ public string BaseUrl /// /// The translation engine id /// The segment pair - /// The engine was trained successfully + /// The engine was trained successfully. /// A server side error occurred. public virtual async System.Threading.Tasks.Task TrainSegmentAsync(string id, SegmentPair segmentPair, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { @@ -2569,37 +2598,37 @@ public string BaseUrl if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); } else if (status_ == 405) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The method is not supported", status_, responseText_, headers_, null); + throw new ServalApiException("The method is not supported.", status_, responseText_, headers_, null); } else if (status_ == 409) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine needs to be built first", status_, responseText_, headers_, null); + throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -2644,7 +2673,7 @@ public string BaseUrl /// /// The translation engine id /// The corpus configuration (see remarks) - /// The corpus was added successfully + /// The added corpus /// A server side error occurred. public virtual async System.Threading.Tasks.Task AddCorpusAsync(string id, TranslationCorpusConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { @@ -2716,31 +2745,25 @@ public string BaseUrl if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist", status_, responseText_, headers_, null); - } - else - if (status_ == 422) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine was specified incorrectly. Did you use the same language for the source and target?", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -2871,8 +2894,8 @@ public string BaseUrl /// Update a corpus with a new set of files /// /// - /// See posting a new corpus for details of use. Will completely replace corpus' file associations. - ///
Will not affect jobs already queued or running. Will not affect existing pretranslations until new build is complete. + /// See posting a new corpus for details of use. Will completely replace corpus' file associations. + ///
Will not affect jobs already queued or running. Will not affect existing pretranslations until new build is complete. /// /// The translation engine id /// The corpus id @@ -2953,25 +2976,25 @@ public string BaseUrl if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine or corpus does not exist", status_, responseText_, headers_, null); + throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -3062,25 +3085,25 @@ public string BaseUrl if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine or corpus does not exist", status_, responseText_, headers_, null); + throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -3111,7 +3134,7 @@ public string BaseUrl /// /// The translation engine id /// The corpus id - /// The data file was deleted successfully + /// The data file was deleted successfully. /// A server side error occurred. public virtual async System.Threading.Tasks.Task DeleteCorpusAsync(string id, string corpusId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { @@ -3168,25 +3191,25 @@ public string BaseUrl if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine or corpus does not exist", status_, responseText_, headers_, null); + throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -3293,6 +3316,271 @@ public string BaseUrl } else if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 409) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get all pretranslations for the specified text in a corpus of a translation engine + /// + /// + /// Pretranslations are arranged in a list of dictionaries with the following fields per pretranslation: + ///
* **TextId**: The TextId of the SourceFile defined when the corpus was created. + ///
* **Refs** (a list of strings): A list of references including: + ///
* The references defined in the SourceFile per line, if any. + ///
* An auto-generated reference of `[TextId]:[lineNumber]`, 1 indexed. + ///
* **Translation**: the text of the pretranslation + /// + /// The translation engine id + /// The corpus id + /// The text id + /// The pretranslations + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task> GetPretranslationsByTextIdAsync(string id, string corpusId, string textId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + if (corpusId == null) + throw new System.ArgumentNullException("corpusId"); + + if (textId == null) + throw new System.ArgumentNullException("textId"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(BaseUrl)) urlBuilder_.Append(BaseUrl); + // Operation Path: "translation/engines/{id}/corpora/{corpusId}/pretranslations/{textId}" + urlBuilder_.Append("translation/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/corpora/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(corpusId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/pretranslations/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(textId, System.Globalization.CultureInfo.InvariantCulture))); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 409) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get a pretranslated Scripture book in USFM format. + /// + /// + /// If the USFM book exists in the target corpus, then the pretranslated text will be inserted into any empty + ///
segments in the the target book and returned. If the USFM book does not exist in the target corpus, then the + ///
pretranslated text will be inserted into an empty template created from the source USFM book and returned. + /// + /// The translation engine id + /// The corpus id + /// The text id + /// The book in USFM format + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task GetPretranslatedUsfmAsync(string id, string corpusId, string textId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + if (corpusId == null) + throw new System.ArgumentNullException("corpusId"); + + if (textId == null) + throw new System.ArgumentNullException("textId"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("text/plain")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(BaseUrl)) urlBuilder_.Append(BaseUrl); + // Operation Path: "translation/engines/{id}/corpora/{corpusId}/pretranslations/{textId}/usfm" + urlBuilder_.Append("translation/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/corpora/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(corpusId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/pretranslations/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(textId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/usfm"); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + var result_ = (string)System.Convert.ChangeType(responseData_, typeof(string)); + return result_; + } + else + if (status_ == 204) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The specified book does not exist in the source or target corpus.", status_, responseText_, headers_, null); + } + else + if (status_ == 400) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The corpus is not a valid Scripture corpus.", status_, responseText_, headers_, null); + } + else + if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); @@ -3301,25 +3589,25 @@ public string BaseUrl if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine or corpus does not exist", status_, responseText_, headers_, null); + throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); } else if (status_ == 409) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine needs to be built first", status_, responseText_, headers_, null); + throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -3405,25 +3693,25 @@ public string BaseUrl if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -3467,7 +3755,7 @@ public string BaseUrl /// /// The translation engine id /// The build config (see remarks) - /// The build job was started successfully + /// The new build job /// A server side error occurred. public virtual async System.Threading.Tasks.Task StartBuildAsync(string id, TranslationBuildConfig buildConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { @@ -3533,31 +3821,31 @@ public string BaseUrl if (status_ == 400) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A corpus id is invalid", status_, responseText_, headers_, null); + throw new ServalApiException("A corpus id is invalid.", status_, responseText_, headers_, null); } else if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client does not own the translation engine", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); } else if (status_ == 409) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("There is already an active or pending build or a build in the process of being canceled", status_, responseText_, headers_, null); + throw new ServalApiException("There is already an active or pending build or a build in the process of being canceled.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -3668,40 +3956,34 @@ public string BaseUrl return objectResponse_.Object; } else - if (status_ == 400) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("Bad request", status_, responseText_, headers_, null); - } - else if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client does not own the translation engine", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine or build does not exist", status_, responseText_, headers_, null); + throw new ServalApiException("The engine or build does not exist.", status_, responseText_, headers_, null); } else if (status_ == 408) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The long polling request timed out. This is expected behavior if you\'re using long-polling with the minRevision strategy specified in the docs", status_, responseText_, headers_, null); + throw new ServalApiException("The long polling request timed out. This is expected behavior if you\'re using long-polling with the minRevision strategy specified in the docs.", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -3797,7 +4079,7 @@ public string BaseUrl if (status_ == 204) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("There is no build currently running", status_, responseText_, headers_, null); + throw new ServalApiException("There is no build currently running.", status_, responseText_, headers_, null); } else if (status_ == 400) @@ -3809,31 +4091,31 @@ public string BaseUrl if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client does not own the translation engine", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); } else if (status_ == 408) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The long polling request timed out. This is expected behavior if you\'re using long-polling with the minRevision strategy specified in the docs", status_, responseText_, headers_, null); + throw new ServalApiException("The long polling request timed out. This is expected behavior if you\'re using long-polling with the minRevision strategy specified in the docs.", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -3860,7 +4142,7 @@ public string BaseUrl /// Cancel the current build job (whether pending or active) for a translation engine /// /// The translation engine id - /// The build job was cancelled successfully + /// The build job was cancelled successfully. /// A server side error occurred. public virtual async System.Threading.Tasks.Task CancelBuildAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { @@ -3911,34 +4193,39 @@ public string BaseUrl return; } else + if (status_ == 204) + { + return; + } + else if (status_ == 401) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); } else if (status_ == 403) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client does not own the translation engine", status_, responseText_, headers_, null); + throw new ServalApiException("The authenticated client does not own the translation engine.", status_, responseText_, headers_, null); } else if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine does not exist or there is no active build ", status_, responseText_, headers_, null); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); } else if (status_ == 405) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The translation engine does not support cancelling builds", status_, responseText_, headers_, null); + throw new ServalApiException("The translation engine does not support cancelling builds.", status_, responseText_, headers_, null); } else if (status_ == 503) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); } else { @@ -4073,7 +4360,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c } [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] - public partial interface ITranslationClient + public partial interface ITranslationEngineTypesClient { /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. @@ -4111,7 +4398,7 @@ public partial interface ITranslationClient } [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TranslationClient : ITranslationClient + public partial class TranslationEngineTypesClient : ITranslationEngineTypesClient { #pragma warning disable 8618 // Set by constructor via BaseUrl property private string _baseUrl; @@ -4119,7 +4406,7 @@ public partial class TranslationClient : ITranslationClient private System.Net.Http.HttpClient _httpClient; private static System.Lazy _settings = new System.Lazy(CreateSerializerSettings, true); - public TranslationClient(System.Net.Http.HttpClient httpClient) + public TranslationEngineTypesClient(System.Net.Http.HttpClient httpClient) { BaseUrl = "/api/v1"; _httpClient = httpClient; diff --git a/src/Serval.Client/Serval.Client.csproj b/src/Serval.Client/Serval.Client.csproj index 9a932015..76996f38 100644 --- a/src/Serval.Client/Serval.Client.csproj +++ b/src/Serval.Client/Serval.Client.csproj @@ -5,6 +5,7 @@ 1.2.0 Client classes for Serval. Serval.Client + 8618 diff --git a/src/Serval.DataFiles/Consumers/GetDataFileConsumer.cs b/src/Serval.DataFiles/Consumers/GetDataFileConsumer.cs index a83f46d0..a1ffe589 100644 --- a/src/Serval.DataFiles/Consumers/GetDataFileConsumer.cs +++ b/src/Serval.DataFiles/Consumers/GetDataFileConsumer.cs @@ -11,19 +11,13 @@ public GetDataFileConsumer(IDataFileService dataFileService) public async Task Consume(ConsumeContext context) { - DataFile? dataFile = await _dataFileService.GetAsync( - context.Message.DataFileId, - context.Message.Owner, - context.CancellationToken - ); - if (dataFile is null) + try { - await context.RespondAsync( - new DataFileNotFound { DataFileId = context.Message.DataFileId, Owner = context.Message.Owner } + DataFile dataFile = await _dataFileService.GetAsync( + context.Message.DataFileId, + context.Message.Owner, + context.CancellationToken ); - } - else - { await context.RespondAsync( new DataFileResult { @@ -34,5 +28,11 @@ await context.RespondAsync( } ); } + catch (EntityNotFoundException) + { + await context.RespondAsync( + new DataFileNotFound { DataFileId = context.Message.DataFileId, Owner = context.Message.Owner } + ); + } } } diff --git a/src/Serval.DataFiles/Controllers/DataFilesController.cs b/src/Serval.DataFiles/Controllers/DataFilesController.cs index d9691c8f..ce60456e 100644 --- a/src/Serval.DataFiles/Controllers/DataFilesController.cs +++ b/src/Serval.DataFiles/Controllers/DataFilesController.cs @@ -56,12 +56,8 @@ public async Task> GetAllAsync(CancellationToken cancel [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] public async Task> GetAsync([NotNull] string id, CancellationToken cancellationToken) { - DataFile? dataFile = await _dataFileService.GetAsync(id, cancellationToken); - if (dataFile == null) - return NotFound(); - if (!await AuthorizeIsOwnerAsync(dataFile)) - return Forbid(); - + DataFile dataFile = await _dataFileService.GetAsync(id, cancellationToken); + await AuthorizeAsync(dataFile); return Ok(Map(dataFile)); } @@ -155,16 +151,9 @@ CancellationToken cancellationToken [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] public async Task DownloadAsync([NotNull] string id, CancellationToken cancellationToken) { - DataFile? dataFile = await _dataFileService.GetAsync(id, cancellationToken); - if (dataFile == null) - return NotFound(); - if (!await AuthorizeIsOwnerAsync(dataFile)) - return Forbid(); - - Stream? stream = await _dataFileService.ReadAsync(id, cancellationToken); - if (stream == null) - return NotFound(); - + DataFile dataFile = await _dataFileService.GetAsync(id, cancellationToken); + await AuthorizeAsync(dataFile); + Stream stream = await _dataFileService.ReadAsync(id, cancellationToken); return File(stream, "application/octet-stream", dataFile.Name); } @@ -195,16 +184,11 @@ public async Task> UpdateAsync( CancellationToken cancellationToken ) { - DataFile? dataFile = await _dataFileService.GetAsync(id, cancellationToken); - if (dataFile == null) - return NotFound(); - if (!await AuthorizeIsOwnerAsync(dataFile)) - return Forbid(); + await AuthorizeAsync(id, cancellationToken); + DataFile dataFile; using (Stream stream = file.OpenReadStream()) dataFile = await _dataFileService.UpdateAsync(id, stream, cancellationToken); - if (dataFile is null) - return NotFound(); var dto = Map(dataFile); return Ok(dto); @@ -234,18 +218,17 @@ CancellationToken cancellationToken [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] public async Task DeleteAsync([NotNull] string id, CancellationToken cancellationToken) { - DataFile? dataFile = await _dataFileService.GetAsync(id, cancellationToken); - if (dataFile == null) - return NotFound(); - if (!await AuthorizeIsOwnerAsync(dataFile)) - return Forbid(); - - if (!await _dataFileService.DeleteAsync(id, cancellationToken)) - return NotFound(); - + await AuthorizeAsync(id, cancellationToken); + await _dataFileService.DeleteAsync(id, cancellationToken); return Ok(); } + private async Task AuthorizeAsync(string id, CancellationToken cancellationToken) + { + DataFile dataFile = await _dataFileService.GetAsync(id, cancellationToken); + await AuthorizeAsync(dataFile); + } + private DataFileDto Map(DataFile source) { return new DataFileDto diff --git a/src/Serval.DataFiles/Services/DataFileService.cs b/src/Serval.DataFiles/Services/DataFileService.cs index 4cebcd97..d46d7ab1 100644 --- a/src/Serval.DataFiles/Services/DataFileService.cs +++ b/src/Serval.DataFiles/Services/DataFileService.cs @@ -26,9 +26,12 @@ IFileSystem fileSystem _fileSystem.CreateDirectory(_options.CurrentValue.FilesDirectory); } - public Task GetAsync(string id, string owner, CancellationToken cancellationToken = default) + public async Task GetAsync(string id, string owner, CancellationToken cancellationToken = default) { - return Entities.GetAsync(f => f.Id == id && f.Owner == owner, cancellationToken); + DataFile? dataFile = await Entities.GetAsync(f => f.Id == id && f.Owner == owner, cancellationToken); + if (dataFile is null) + throw new EntityNotFoundException($"Could not find the DataFile '{id}' with owner '{owner}'."); + return dataFile; } public async Task> GetAllAsync(string owner, CancellationToken cancellationToken = default) @@ -53,16 +56,16 @@ public async Task CreateAsync(DataFile dataFile, Stream stream, CancellationToke } } - public async Task ReadAsync(string id, CancellationToken cancellationToken = default) + public async Task ReadAsync(string id, CancellationToken cancellationToken = default) { DataFile? dataFile = await GetAsync(id, cancellationToken); if (dataFile is null) - return null; + throw new EntityNotFoundException($"Could not find the DataFile '{id}'."); string path = GetDataFilePath(dataFile.Filename); return _fileSystem.OpenRead(path); } - public async Task UpdateAsync(string id, Stream stream, CancellationToken cancellationToken = default) + public async Task UpdateAsync(string id, Stream stream, CancellationToken cancellationToken = default) { string filename = Path.GetRandomFileName(); string path = GetDataFilePath(filename); @@ -81,7 +84,7 @@ public async Task CreateAsync(DataFile dataFile, Stream stream, CancellationToke ); if (originalDataFile is null) { - deleteFile = true; + throw new EntityNotFoundException($"Could not find the DataFile '{id}'."); } else { @@ -91,7 +94,7 @@ await _deletedFiles.InsertAsync( ); } await _dataAccessContext.CommitTransactionAsync(cancellationToken); - return originalDataFile is null ? null : await GetAsync(id, cancellationToken); + return await GetAsync(id, cancellationToken); } catch { @@ -105,21 +108,18 @@ await _deletedFiles.InsertAsync( } } - public override async Task DeleteAsync(string id, CancellationToken cancellationToken = default) + public override async Task DeleteAsync(string id, CancellationToken cancellationToken = default) { - // return true if the deletion was successful, false if the file did not exist or was already deleted. await _dataAccessContext.BeginTransactionAsync(cancellationToken); DataFile? dataFile = await Entities.DeleteAsync(id, cancellationToken); - if (dataFile is not null) - { - await _deletedFiles.InsertAsync( - new DeletedFile { Filename = dataFile.Filename, DeletedAt = DateTime.UtcNow }, - cancellationToken - ); - } + if (dataFile is null) + throw new EntityNotFoundException($"Could not find the DataFile '{id}'."); + await _deletedFiles.InsertAsync( + new DeletedFile { Filename = dataFile.Filename, DeletedAt = DateTime.UtcNow }, + cancellationToken + ); await _mediator.Publish(new DataFileDeleted { DataFileId = id }, cancellationToken); await _dataAccessContext.CommitTransactionAsync(CancellationToken.None); - return dataFile is not null; } private string GetDataFilePath(string filename) diff --git a/src/Serval.DataFiles/Services/IDataFileService.cs b/src/Serval.DataFiles/Services/IDataFileService.cs index 83ff01b9..b10d8f25 100644 --- a/src/Serval.DataFiles/Services/IDataFileService.cs +++ b/src/Serval.DataFiles/Services/IDataFileService.cs @@ -3,10 +3,10 @@ public interface IDataFileService { Task> GetAllAsync(string owner, CancellationToken cancellationToken = default); - Task GetAsync(string id, CancellationToken cancellationToken = default); - Task GetAsync(string id, string owner, CancellationToken cancellationToken = default); + Task GetAsync(string id, CancellationToken cancellationToken = default); + Task GetAsync(string id, string owner, CancellationToken cancellationToken = default); Task CreateAsync(DataFile dataFile, Stream stream, CancellationToken cancellationToken = default); - Task ReadAsync(string id, CancellationToken cancellationToken = default); - Task UpdateAsync(string id, Stream stream, CancellationToken cancellationToken = default); - Task DeleteAsync(string id, CancellationToken cancellationToken = default); + Task ReadAsync(string id, CancellationToken cancellationToken = default); + Task UpdateAsync(string id, Stream stream, CancellationToken cancellationToken = default); + Task DeleteAsync(string id, CancellationToken cancellationToken = default); } diff --git a/src/Serval.DataFiles/Usings.cs b/src/Serval.DataFiles/Usings.cs index a10d6a20..0d81ccb1 100644 --- a/src/Serval.DataFiles/Usings.cs +++ b/src/Serval.DataFiles/Usings.cs @@ -23,4 +23,5 @@ global using Serval.Shared.Controllers; global using Serval.Shared.Models; global using Serval.Shared.Services; +global using Serval.Shared.Utils; global using SIL.DataAccess; diff --git a/src/Serval.Shared/Configuration/IServiceCollectionExtensions.cs b/src/Serval.Shared/Configuration/IServiceCollectionExtensions.cs index 6902f510..2671ac40 100644 --- a/src/Serval.Shared/Configuration/IServiceCollectionExtensions.cs +++ b/src/Serval.Shared/Configuration/IServiceCollectionExtensions.cs @@ -5,6 +5,7 @@ public static class IServiceCollectionExtensions public static IServalBuilder AddServal(this IServiceCollection services, IConfiguration? configuration = null) { services.AddTransient(); + services.AddTransient(); return new ServalBuilder(services, configuration); } } diff --git a/src/Serval.Shared/Controllers/BadRequestExceptionFilter.cs b/src/Serval.Shared/Controllers/BadRequestExceptionFilter.cs new file mode 100644 index 00000000..076e2065 --- /dev/null +++ b/src/Serval.Shared/Controllers/BadRequestExceptionFilter.cs @@ -0,0 +1,13 @@ +namespace Serval.Shared.Controllers; + +public class BadRequestExceptionFilter : ExceptionFilterAttribute +{ + public override void OnException(ExceptionContext context) + { + if (context.Exception is InvalidOperationException) + { + context.Result = new BadRequestObjectResult(context.Exception.Message); + context.ExceptionHandled = true; + } + } +} diff --git a/src/Serval.Shared/Controllers/ErrorResultFilter.cs b/src/Serval.Shared/Controllers/ErrorResultFilter.cs index 3c6bfa03..31e2a271 100644 --- a/src/Serval.Shared/Controllers/ErrorResultFilter.cs +++ b/src/Serval.Shared/Controllers/ErrorResultFilter.cs @@ -1,6 +1,3 @@ -using System.Diagnostics; -using System.Text.Json; - namespace Serval.Shared.Controllers { public class ErrorResultFilter : IAlwaysRunResultFilter diff --git a/src/Serval.Shared/Controllers/ForbiddenExceptionFilter.cs b/src/Serval.Shared/Controllers/ForbiddenExceptionFilter.cs new file mode 100644 index 00000000..b80e621e --- /dev/null +++ b/src/Serval.Shared/Controllers/ForbiddenExceptionFilter.cs @@ -0,0 +1,13 @@ +namespace Serval.Shared.Controllers; + +public class ForbiddenExceptionFilter : ExceptionFilterAttribute +{ + public override void OnException(ExceptionContext context) + { + if (context.Exception is ForbiddenException) + { + context.Result = new ForbidResult(); + context.ExceptionHandled = true; + } + } +} diff --git a/src/Serval.Shared/Controllers/NotFoundExceptionFilter.cs b/src/Serval.Shared/Controllers/NotFoundExceptionFilter.cs new file mode 100644 index 00000000..53f5b332 --- /dev/null +++ b/src/Serval.Shared/Controllers/NotFoundExceptionFilter.cs @@ -0,0 +1,15 @@ +using Serval.Shared.Utils; + +namespace Serval.Shared.Controllers; + +public class NotFoundExceptionFilter : ExceptionFilterAttribute +{ + public override void OnException(ExceptionContext context) + { + if (context.Exception is EntityNotFoundException) + { + context.Result = new NotFoundResult(); + context.ExceptionHandled = true; + } + } +} diff --git a/src/Serval.Shared/Controllers/ServalControllerBase.cs b/src/Serval.Shared/Controllers/ServalControllerBase.cs index bebcdf29..34e75bb6 100644 --- a/src/Serval.Shared/Controllers/ServalControllerBase.cs +++ b/src/Serval.Shared/Controllers/ServalControllerBase.cs @@ -7,6 +7,9 @@ [TypeFilter(typeof(ServiceUnavailableExceptionFilter))] [TypeFilter(typeof(ErrorResultFilter))] [TypeFilter(typeof(AbortedRpcExceptionFilter))] +[TypeFilter(typeof(NotFoundExceptionFilter))] +[TypeFilter(typeof(ForbiddenExceptionFilter))] +[TypeFilter(typeof(BadRequestExceptionFilter))] public abstract class ServalControllerBase : Controller { private readonly IAuthorizationService _authService; @@ -18,9 +21,10 @@ protected ServalControllerBase(IAuthorizationService authService) protected string Owner => User.Identity!.Name!; - protected async Task AuthorizeIsOwnerAsync(IOwnedEntity ownedEntity) + protected async Task AuthorizeAsync(IOwnedEntity ownedEntity) { AuthorizationResult result = await _authService.AuthorizeAsync(User, ownedEntity, "IsOwner"); - return result.Succeeded; + if (!result.Succeeded) + throw new ForbiddenException(); } } diff --git a/src/Serval.Shared/Serval.Shared.csproj b/src/Serval.Shared/Serval.Shared.csproj index 59b805ed..8c6ecf97 100644 --- a/src/Serval.Shared/Serval.Shared.csproj +++ b/src/Serval.Shared/Serval.Shared.csproj @@ -15,10 +15,12 @@ + + diff --git a/src/Serval.Shared/Services/EntityServiceBase.cs b/src/Serval.Shared/Services/EntityServiceBase.cs index abcff427..6970a80c 100644 --- a/src/Serval.Shared/Services/EntityServiceBase.cs +++ b/src/Serval.Shared/Services/EntityServiceBase.cs @@ -10,9 +10,12 @@ protected EntityServiceBase(IRepository entities) protected IRepository Entities { get; } - public Task GetAsync(string id, CancellationToken cancellationToken = default) + public async Task GetAsync(string id, CancellationToken cancellationToken = default) { - return Entities.GetAsync(id, cancellationToken); + T? entity = await Entities.GetAsync(id, cancellationToken); + if (entity is null) + throw new EntityNotFoundException($"Could not find the {typeof(T).Name} '{id}'."); + return entity; } public virtual Task CreateAsync(T entity, CancellationToken cancellationToken = default) @@ -20,8 +23,10 @@ public virtual Task CreateAsync(T entity, CancellationToken cancellationToken = return Entities.InsertAsync(entity, cancellationToken); } - public virtual async Task DeleteAsync(string id, CancellationToken cancellationToken = default) + public virtual async Task DeleteAsync(string id, CancellationToken cancellationToken = default) { - return await Entities.DeleteAsync(id, cancellationToken) is not null; + T? entity = await Entities.DeleteAsync(id, cancellationToken); + if (entity is null) + throw new EntityNotFoundException($"Could not find the {typeof(T).Name} '{id}'."); } } diff --git a/src/Serval.Shared/Services/FileSystem.cs b/src/Serval.Shared/Services/FileSystem.cs index d7683d0a..00b96a5c 100644 --- a/src/Serval.Shared/Services/FileSystem.cs +++ b/src/Serval.Shared/Services/FileSystem.cs @@ -22,4 +22,9 @@ public Stream OpenRead(string path) { return File.OpenRead(path); } + + public IZipContainer OpenZipFile(string path) + { + return new ZipContainer(path); + } } diff --git a/src/Serval.Shared/Services/IFileSystem.cs b/src/Serval.Shared/Services/IFileSystem.cs index 429609b1..55d93f5a 100644 --- a/src/Serval.Shared/Services/IFileSystem.cs +++ b/src/Serval.Shared/Services/IFileSystem.cs @@ -6,4 +6,5 @@ public interface IFileSystem void CreateDirectory(string path); Stream OpenWrite(string path); Stream OpenRead(string path); + IZipContainer OpenZipFile(string path); } diff --git a/src/Serval.Shared/Services/IScriptureDataFileService.cs b/src/Serval.Shared/Services/IScriptureDataFileService.cs new file mode 100644 index 00000000..92fe96d9 --- /dev/null +++ b/src/Serval.Shared/Services/IScriptureDataFileService.cs @@ -0,0 +1,7 @@ +namespace Serval.Shared.Services; + +public interface IScriptureDataFileService +{ + ParatextProjectSettings GetParatextProjectSettings(string filename); + Task ReadParatextProjectBookAsync(string filename, string book); +} diff --git a/src/Serval.Shared/Services/IZipContainer.cs b/src/Serval.Shared/Services/IZipContainer.cs new file mode 100644 index 00000000..7c0beac1 --- /dev/null +++ b/src/Serval.Shared/Services/IZipContainer.cs @@ -0,0 +1,8 @@ +namespace Serval.Shared.Services; + +public interface IZipContainer : IDisposable +{ + bool EntryExists(string name); + Stream OpenEntry(string name); + IEnumerable Entries { get; } +} diff --git a/src/Serval.Shared/Services/ScriptureDataFileService.cs b/src/Serval.Shared/Services/ScriptureDataFileService.cs new file mode 100644 index 00000000..1a5a9753 --- /dev/null +++ b/src/Serval.Shared/Services/ScriptureDataFileService.cs @@ -0,0 +1,36 @@ +namespace Serval.Shared.Services; + +public class ScriptureDataFileService(IFileSystem fileSystem, IOptionsMonitor dataFileOptions) + : IScriptureDataFileService +{ + private readonly IFileSystem _fileSystem = fileSystem; + private readonly IOptionsMonitor _dataFileOptions = dataFileOptions; + + public ParatextProjectSettings GetParatextProjectSettings(string filename) + { + using IZipContainer container = _fileSystem.OpenZipFile(GetFilePath(filename)); + return ParseProjectSettings(container); + } + + public async Task ReadParatextProjectBookAsync(string filename, string book) + { + using IZipContainer container = _fileSystem.OpenZipFile(GetFilePath(filename)); + ParatextProjectSettings settings = ParseProjectSettings(container); + string entryName = settings.GetBookFileName(book); + if (!container.EntryExists(entryName)) + return null; + using StreamReader reader = new(container.OpenEntry(entryName)); + return await reader.ReadToEndAsync(); + } + + private string GetFilePath(string filename) + { + return Path.Combine(_dataFileOptions.CurrentValue.FilesDirectory, filename); + } + + private static ParatextProjectSettings ParseProjectSettings(IZipContainer container) + { + ZipParatextProjectSettingsParser settingsParser = new(container); + return settingsParser.Parse(); + } +} diff --git a/src/Serval.Shared/Services/ZipContainer.cs b/src/Serval.Shared/Services/ZipContainer.cs new file mode 100644 index 00000000..e7c0eff0 --- /dev/null +++ b/src/Serval.Shared/Services/ZipContainer.cs @@ -0,0 +1,34 @@ +using System.IO.Compression; +using SIL.ObjectModel; + +namespace Serval.Shared.Services; + +public class ZipContainer : DisposableBase, IZipContainer +{ + private readonly ZipArchive _archive; + + public ZipContainer(string fileName) + { + _archive = ZipFile.OpenRead(fileName); + } + + public IEnumerable Entries => _archive.Entries.Select(e => e.FullName); + + public bool EntryExists(string name) + { + return _archive.GetEntry(name) is not null; + } + + public Stream OpenEntry(string name) + { + ZipArchiveEntry? entry = _archive.GetEntry(name); + if (entry is null) + throw new ArgumentException("The specified entry does not exist.", nameof(name)); + return entry.Open(); + } + + protected override void DisposeManagedResources() + { + _archive.Dispose(); + } +} diff --git a/src/Serval.Shared/Services/ZipParatextProjectSettingsParser.cs b/src/Serval.Shared/Services/ZipParatextProjectSettingsParser.cs new file mode 100644 index 00000000..85fb255c --- /dev/null +++ b/src/Serval.Shared/Services/ZipParatextProjectSettingsParser.cs @@ -0,0 +1,21 @@ +namespace Serval.Shared.Services; + +public class ZipParatextProjectSettingsParser(IZipContainer projectContainer) : ZipParatextProjectSettingsParserBase +{ + private readonly IZipContainer _projectContainer = projectContainer; + + protected override bool Exists(string fileName) + { + return _projectContainer.EntryExists(fileName); + } + + protected override string? Find(string extension) + { + return _projectContainer.Entries.FirstOrDefault(e => e.EndsWith(extension)); + } + + protected override Stream Open(string fileName) + { + return _projectContainer.OpenEntry(fileName); + } +} diff --git a/src/Serval.Shared/Usings.cs b/src/Serval.Shared/Usings.cs index 05799fbf..fec5ec1b 100644 --- a/src/Serval.Shared/Usings.cs +++ b/src/Serval.Shared/Usings.cs @@ -1,4 +1,4 @@ -global using System.Text; +global using System.Diagnostics; global using System.Text.Json; global using System.Text.Json.Serialization; global using Grpc.Core; @@ -7,9 +7,11 @@ global using Microsoft.AspNetCore.Mvc; global using Microsoft.AspNetCore.Mvc.Filters; global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.Diagnostics.HealthChecks; global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; global using Serval.Shared.Configuration; global using Serval.Shared.Models; global using Serval.Shared.Services; +global using Serval.Shared.Utils; global using SIL.DataAccess; +global using SIL.Machine.Corpora; diff --git a/src/Serval.Shared/Utils/EntityNotFoundException.cs b/src/Serval.Shared/Utils/EntityNotFoundException.cs new file mode 100644 index 00000000..1c576bfb --- /dev/null +++ b/src/Serval.Shared/Utils/EntityNotFoundException.cs @@ -0,0 +1,12 @@ +namespace Serval.Shared.Utils; + +public class EntityNotFoundException : Exception +{ + public EntityNotFoundException() { } + + public EntityNotFoundException(string? message) + : base(message) { } + + public EntityNotFoundException(string? message, Exception? innerException) + : base(message, innerException) { } +} diff --git a/src/Serval.Shared/Utils/ForbiddenException.cs b/src/Serval.Shared/Utils/ForbiddenException.cs new file mode 100644 index 00000000..058689c4 --- /dev/null +++ b/src/Serval.Shared/Utils/ForbiddenException.cs @@ -0,0 +1,12 @@ +namespace Serval.Shared.Utils; + +public class ForbiddenException : Exception +{ + public ForbiddenException() { } + + public ForbiddenException(string? message) + : base(message) { } + + public ForbiddenException(string? message, Exception? innerException) + : base(message, innerException) { } +} diff --git a/src/Serval.Shared/Utils/TupleExtensions.cs b/src/Serval.Shared/Utils/TupleExtensions.cs deleted file mode 100644 index fa4f1a5f..00000000 --- a/src/Serval.Shared/Utils/TupleExtensions.cs +++ /dev/null @@ -1,16 +0,0 @@ -using System.Diagnostics.CodeAnalysis; - -namespace Serval.Shared.Utils; - -public static class TupleExtensions -{ - public static bool IsSuccess( - this ValueTuple tuple, - [MaybeNullWhen(true)] out ActionResult errorResult - ) - { - bool success; - (success, errorResult) = tuple; - return success; - } -} diff --git a/src/Serval.Translation/Contracts/PretranslationFormat.cs b/src/Serval.Translation/Contracts/PretranslationFormat.cs new file mode 100644 index 00000000..e0f53ab9 --- /dev/null +++ b/src/Serval.Translation/Contracts/PretranslationFormat.cs @@ -0,0 +1,7 @@ +namespace Serval.Translation.Contracts; + +public enum PretranslationFormat +{ + Json, + Usfm +} diff --git a/src/Serval.Translation/Controllers/TranslationEngineTypesController.cs b/src/Serval.Translation/Controllers/TranslationEngineTypesController.cs index 22880c70..53df27a0 100644 --- a/src/Serval.Translation/Controllers/TranslationEngineTypesController.cs +++ b/src/Serval.Translation/Controllers/TranslationEngineTypesController.cs @@ -3,7 +3,7 @@ [ApiVersion(1.0)] [Route("api/v{version:apiVersion}/translation/engine-types")] [OpenApiTag("Translation Engines")] -public class TranslationController(IAuthorizationService authService, IEngineService engineService) +public class TranslationEngineTypesController(IAuthorizationService authService, IEngineService engineService) : ServalControllerBase(authService) { private readonly IEngineService _engineService = engineService; diff --git a/src/Serval.Translation/Controllers/TranslationEnginesController.cs b/src/Serval.Translation/Controllers/TranslationEnginesController.cs index 8a01d75f..73e397a4 100644 --- a/src/Serval.Translation/Controllers/TranslationEnginesController.cs +++ b/src/Serval.Translation/Controllers/TranslationEnginesController.cs @@ -23,9 +23,9 @@ IUrlService urlService /// /// /// The engines - /// The client is not authenticated - /// The authenticated client cannot perform the operation - /// A necessary service is currently unavailable. Check `/health` for more details. + /// The client is not authenticated. + /// The authenticated client cannot perform the operation. + /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.ReadTranslationEngines)] [HttpGet] [ProducesResponseType(StatusCodes.Status200OK)] @@ -43,10 +43,10 @@ public async Task> GetAllAsync(CancellationTok /// The translation engine id /// /// The translation engine - /// The client is not authenticated - /// The authenticated client cannot perform the operation or does not own the translation engine - /// The engine does not exist - /// A necessary service is currently unavailable. Check `/health` for more details. + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.ReadTranslationEngines)] [HttpGet("{id}", Name = "GetTranslationEngine")] @@ -60,12 +60,8 @@ public async Task> GetAsync( CancellationToken cancellationToken ) { - Engine? engine = await _engineService.GetAsync(id, cancellationToken); - if (engine == null) - return NotFound(); - if (!await AuthorizeIsOwnerAsync(engine)) - return Forbid(); - + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); return Ok(Map(engine)); } @@ -103,20 +99,18 @@ CancellationToken cancellationToken /// The translation engine configuration (see above) /// /// - /// The translation engine was created successfully - /// Bad request. Is the engine type correct? - /// The client is not authenticated - /// The authenticated client cannot perform the operation or does not own the translation engine - /// The engine does not exist - /// The engine was specified incorrectly. Did you use the same language for the source and target? - /// A necessary service is currently unavailable. Check `/health` for more details. + /// The new translation engine + /// Bad request. Is the engine type correct? + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.CreateTranslationEngines)] [HttpPost] [ProducesResponseType(StatusCodes.Status201Created)] [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] - [ProducesResponseType(typeof(void), StatusCodes.Status422UnprocessableEntity)] [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] public async Task> CreateAsync( [FromBody] TranslationEngineConfigDto engineConfig, @@ -126,19 +120,7 @@ CancellationToken cancellationToken { Engine engine = Map(engineConfig); engine.Id = idGenerator.GenerateId(); - try - { - bool success = await _engineService.CreateAsync(engine, cancellationToken); - if (!success) - return BadRequest(); - } - catch (RpcException rpcEx) - { - if (rpcEx.StatusCode == Grpc.Core.StatusCode.InvalidArgument) - return UnprocessableEntity(rpcEx.Message); - throw; - } - + await _engineService.CreateAsync(engine, cancellationToken); TranslationEngineDto dto = Map(engine); return Created(dto.Url, dto); } @@ -148,11 +130,11 @@ CancellationToken cancellationToken /// /// The translation engine id /// - /// The engine was successfully deleted - /// The client is not authenticated - /// The authenticated client cannot perform the operation or does not own the translation engine - /// The engine does not exist and therefore cannot be deleted - /// A necessary service is currently unavailable. Check `/health` for more details. + /// The engine was successfully deleted. + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine does not exist and therefore cannot be deleted. + /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.DeleteTranslationEngines)] [HttpDelete("{id}")] [ProducesResponseType(typeof(void), StatusCodes.Status200OK)] @@ -162,11 +144,8 @@ CancellationToken cancellationToken [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] public async Task DeleteAsync([NotNull] string id, CancellationToken cancellationToken) { - if (!(await AuthorizeAsync(id, cancellationToken)).IsSuccess(out ActionResult? errorResult)) - return errorResult; - - if (!await _engineService.DeleteAsync(id, cancellationToken)) - return NotFound(); + await AuthorizeAsync(id, cancellationToken); + await _engineService.DeleteAsync(id, cancellationToken); return Ok(); } @@ -178,12 +157,12 @@ public async Task DeleteAsync([NotNull] string id, CancellationTok /// /// The translation result /// Bad request - /// The client is not authenticated - /// The authenticated client cannot perform the operation or does not own the translation engine - /// The engine does not exist - /// The method is not supported - /// The engine needs to be built before it can translate segments - /// A necessary service is currently unavailable. Check `/health` for more details. + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine does not exist. + /// The method is not supported. + /// The engine needs to be built before it can translate segments. + /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.ReadTranslationEngines)] [HttpPost("{id}/translate")] [ProducesResponseType(StatusCodes.Status200OK)] @@ -200,12 +179,8 @@ public async Task> TranslateAsync( CancellationToken cancellationToken ) { - if (!(await AuthorizeAsync(id, cancellationToken)).IsSuccess(out ActionResult? errorResult)) - return errorResult; - - TranslationResult? result = await _engineService.TranslateAsync(id, segment, cancellationToken); - if (result == null) - return NotFound(); + await AuthorizeAsync(id, cancellationToken); + TranslationResult result = await _engineService.TranslateAsync(id, segment, cancellationToken); return Ok(Map(result)); } @@ -218,12 +193,12 @@ CancellationToken cancellationToken /// /// The translation results /// Bad request - /// The client is not authenticated - /// The authenticated client cannot perform the operation or does not own the translation engine - /// The engine does not exist - /// The method is not supported - /// The engine needs to be built before it can translate segments - /// A necessary service is currently unavailable. Check `/health` for more details. + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine does not exist. + /// The method is not supported. + /// The engine needs to be built before it can translate segments. + /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.ReadTranslationEngines)] [HttpPost("{id}/translate/{n}")] [ProducesResponseType(StatusCodes.Status200OK)] @@ -241,17 +216,8 @@ public async Task>> TranslateNAsy CancellationToken cancellationToken ) { - if (!(await AuthorizeAsync(id, cancellationToken)).IsSuccess(out ActionResult? errorResult)) - return errorResult; - - IEnumerable? results = await _engineService.TranslateAsync( - id, - n, - segment, - cancellationToken - ); - if (results == null) - return NotFound(); + await AuthorizeAsync(id, cancellationToken); + IEnumerable results = await _engineService.TranslateAsync(id, n, segment, cancellationToken); return Ok(results.Select(Map)); } @@ -263,12 +229,12 @@ CancellationToken cancellationToken /// /// The word graph result /// Bad request - /// The client is not authenticated - /// The authenticated client cannot perform the operation or does not own the translation engine - /// The engine does not exist - /// The method is not supported - /// The engine needs to be built first - /// A necessary service is currently unavailable. Check `/health` for more details. + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine does not exist. + /// The method is not supported. + /// The engine needs to be built first. + /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.ReadTranslationEngines)] [HttpPost("{id}/get-word-graph")] [ProducesResponseType(StatusCodes.Status200OK)] @@ -285,12 +251,8 @@ public async Task> GetWordGraphAsync( CancellationToken cancellationToken ) { - if (!(await AuthorizeAsync(id, cancellationToken)).IsSuccess(out ActionResult? errorResult)) - return errorResult; - - WordGraph? wordGraph = await _engineService.GetWordGraphAsync(id, segment, cancellationToken); - if (wordGraph == null) - return NotFound(); + await AuthorizeAsync(id, cancellationToken); + WordGraph wordGraph = await _engineService.GetWordGraphAsync(id, segment, cancellationToken); return Ok(Map(wordGraph)); } @@ -305,14 +267,14 @@ CancellationToken cancellationToken /// The translation engine id /// The segment pair /// - /// The engine was trained successfully + /// The engine was trained successfully. /// Bad request - /// The client is not authenticated - /// The authenticated client cannot perform the operation or does not own the translation engine - /// The engine does not exist - /// The method is not supported - /// The engine needs to be built first - /// A necessary service is currently unavailable. Check `/health` for more details. + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine does not exist. + /// The method is not supported. + /// The engine needs to be built first. + /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.UpdateTranslationEngines)] [HttpPost("{id}/train-segment")] [ProducesResponseType(StatusCodes.Status200OK)] @@ -329,21 +291,14 @@ public async Task TrainSegmentAsync( CancellationToken cancellationToken ) { - if (!(await AuthorizeAsync(id, cancellationToken)).IsSuccess(out ActionResult? errorResult)) - return errorResult; - - if ( - !await _engineService.TrainSegmentPairAsync( - id, - segmentPair.SourceSegment, - segmentPair.TargetSegment, - segmentPair.SentenceStart, - cancellationToken - ) - ) - { - return NotFound(); - } + await AuthorizeAsync(id, cancellationToken); + await _engineService.TrainSegmentPairAsync( + id, + segmentPair.SourceSegment, + segmentPair.TargetSegment, + segmentPair.SentenceStart, + cancellationToken + ); return Ok(); } @@ -372,13 +327,12 @@ CancellationToken cancellationToken /// /// /// - /// The corpus was added successfully + /// The added corpus /// Bad request - /// The client is not authenticated - /// The authenticated client cannot perform the operation or does not own the translation engine - /// The engine does not exist - /// The engine was specified incorrectly. Did you use the same language for the source and target? - /// A necessary service is currently unavailable. Check `/health` for more details. + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.UpdateTranslationEngines)] [HttpPost("{id}/corpora")] [ProducesResponseType(StatusCodes.Status201Created)] @@ -386,7 +340,6 @@ CancellationToken cancellationToken [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] - [ProducesResponseType(typeof(void), StatusCodes.Status422UnprocessableEntity)] [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] public async Task> AddCorpusAsync( [NotNull] string id, @@ -396,17 +349,9 @@ public async Task> AddCorpusAsync( CancellationToken cancellationToken ) { - Engine? engine = await _engineService.GetAsync(id, cancellationToken); - if (engine is null) - return NotFound(); - if (!await AuthorizeIsOwnerAsync(engine)) - return Forbid(); + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); Corpus corpus = await MapAsync(getDataFileClient, idGenerator.GenerateId(), corpusConfig, cancellationToken); - if (engine.SourceLanguage != corpus.SourceLanguage || engine.TargetLanguage != corpus.TargetLanguage) - return UnprocessableEntity( - $"Source and target languages, {corpus.SourceLanguage} & {corpus.TargetLanguage}, do not match engine source and target languages, {engine.SourceLanguage} & {engine.TargetLanguage}" - ); - await _engineService.AddCorpusAsync(id, corpus, cancellationToken); TranslationCorpusDto dto = Map(id, corpus); return Created(dto.Url, dto); @@ -416,8 +361,8 @@ CancellationToken cancellationToken /// Update a corpus with a new set of files /// /// - /// See posting a new corpus for details of use. Will completely replace corpus' file associations. - /// Will not affect jobs already queued or running. Will not affect existing pretranslations until new build is complete. + /// See posting a new corpus for details of use. Will completely replace corpus' file associations. + /// Will not affect jobs already queued or running. Will not affect existing pretranslations until new build is complete. /// /// The translation engine id /// The corpus id @@ -426,10 +371,10 @@ CancellationToken cancellationToken /// /// The corpus was updated successfully /// Bad request - /// The client is not authenticated - /// The authenticated client cannot perform the operation or does not own the translation engine - /// The engine or corpus does not exist - /// A necessary service is currently unavailable. Check `/health` for more details. + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine or corpus does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.UpdateTranslationEngines)] [HttpPatch("{id}/corpora/{corpusId}")] [ProducesResponseType(StatusCodes.Status200OK)] @@ -446,10 +391,8 @@ public async Task> UpdateCorpusAsync( CancellationToken cancellationToken ) { - if (!(await AuthorizeAsync(id, cancellationToken)).IsSuccess(out ActionResult? errorResult)) - return errorResult; - - Corpus? corpus = await _engineService.UpdateCorpusAsync( + await AuthorizeAsync(id, cancellationToken); + Corpus corpus = await _engineService.UpdateCorpusAsync( id, corpusId, corpusConfig.SourceFiles is null @@ -460,8 +403,6 @@ corpusConfig.TargetFiles is null : await MapAsync(getDataFileClient, corpusConfig.TargetFiles, cancellationToken), cancellationToken ); - if (corpus is null) - return NotFound(); return Ok(Map(id, corpus)); } @@ -487,12 +428,8 @@ public async Task>> GetAllCorpora CancellationToken cancellationToken ) { - Engine? engine = await _engineService.GetAsync(id, cancellationToken); - if (engine == null) - return NotFound(); - if (!await AuthorizeIsOwnerAsync(engine)) - return Forbid(); - + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); return Ok(engine.Corpora.Select(c => Map(id, c))); } @@ -503,10 +440,10 @@ CancellationToken cancellationToken /// The corpus id /// /// The corpus configuration - /// The client is not authenticated - /// The authenticated client cannot perform the operation or does not own the translation engine - /// The engine or corpus does not exist - /// A necessary service is currently unavailable. Check `/health` for more details. + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine or corpus does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.ReadTranslationEngines)] [HttpGet("{id}/corpora/{corpusId}", Name = "GetTranslationCorpus")] [ProducesResponseType(StatusCodes.Status200OK)] @@ -520,15 +457,11 @@ public async Task> GetCorpusAsync( CancellationToken cancellationToken ) { - Engine? engine = await _engineService.GetAsync(id, cancellationToken); - if (engine == null) - return NotFound(); - if (!await AuthorizeIsOwnerAsync(engine)) - return Forbid(); + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); Corpus? corpus = engine.Corpora.FirstOrDefault(f => f.Id == corpusId); if (corpus == null) return NotFound(); - return Ok(Map(id, corpus)); } @@ -541,11 +474,11 @@ CancellationToken cancellationToken /// The translation engine id /// The corpus id /// - /// The data file was deleted successfully - /// The client is not authenticated - /// The authenticated client cannot perform the operation or does not own the translation engine - /// The engine or corpus does not exist - /// A necessary service is currently unavailable. Check `/health` for more details. + /// The data file was deleted successfully. + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine or corpus does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.UpdateTranslationEngines)] [HttpDelete("{id}/corpora/{corpusId}")] [ProducesResponseType(typeof(void), StatusCodes.Status200OK)] @@ -559,12 +492,8 @@ public async Task DeleteCorpusAsync( CancellationToken cancellationToken ) { - if (!(await AuthorizeAsync(id, cancellationToken)).IsSuccess(out ActionResult? errorResult)) - return errorResult; - - if (!await _engineService.DeleteCorpusAsync(id, corpusId, cancellationToken)) - return NotFound(); - + await AuthorizeAsync(id, cancellationToken); + await _engineService.DeleteCorpusAsync(id, corpusId, cancellationToken); return Ok(); } @@ -586,11 +515,11 @@ CancellationToken cancellationToken /// The text id (optional) /// /// The pretranslations - /// The client is not authenticated - /// The authenticated client cannot perform the operation or does not own the translation engine - /// The engine or corpus does not exist - /// The engine needs to be built first - /// A necessary service is currently unavailable. Check `/health` for more details. + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine or corpus does not exist. + /// The engine needs to be built first. + /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.ReadTranslationEngines)] [HttpGet("{id}/corpora/{corpusId}/pretranslations")] [ProducesResponseType(StatusCodes.Status200OK)] @@ -606,11 +535,61 @@ public async Task>> GetAllPretransla CancellationToken cancellationToken ) { - Engine? engine = await _engineService.GetAsync(id, cancellationToken); - if (engine == null) + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); + if (!engine.Corpora.Any(c => c.Id == corpusId)) return NotFound(); - if (!await AuthorizeIsOwnerAsync(engine)) - return Forbid(); + if (engine.ModelRevision == 0) + return Conflict(); + + IEnumerable pretranslations = await _pretranslationService.GetAllAsync( + id, + engine.ModelRevision, + corpusId, + textId, + cancellationToken + ); + return Ok(pretranslations.Select(Map)); + } + + /// + /// Get all pretranslations for the specified text in a corpus of a translation engine + /// + /// + /// Pretranslations are arranged in a list of dictionaries with the following fields per pretranslation: + /// * **TextId**: The TextId of the SourceFile defined when the corpus was created. + /// * **Refs** (a list of strings): A list of references including: + /// * The references defined in the SourceFile per line, if any. + /// * An auto-generated reference of `[TextId]:[lineNumber]`, 1 indexed. + /// * **Translation**: the text of the pretranslation + /// + /// The translation engine id + /// The corpus id + /// The text id + /// + /// The pretranslations + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine or corpus does not exist. + /// The engine needs to be built first. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadTranslationEngines)] + [HttpGet("{id}/corpora/{corpusId}/pretranslations/{textId}")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status409Conflict)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task>> GetPretranslationsByTextIdAsync( + [NotNull] string id, + [NotNull] string corpusId, + [NotNull] string textId, + CancellationToken cancellationToken + ) + { + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); if (!engine.Corpora.Any(c => c.Id == corpusId)) return NotFound(); if (engine.ModelRevision == 0) @@ -626,16 +605,73 @@ CancellationToken cancellationToken return Ok(pretranslations.Select(Map)); } + /// + /// Get a pretranslated Scripture book in USFM format. + /// + /// + /// If the USFM book exists in the target corpus, then the pretranslated text will be inserted into any empty + /// segments in the the target book and returned. If the USFM book does not exist in the target corpus, then the + /// pretranslated text will be inserted into an empty template created from the source USFM book and returned. + /// + /// The translation engine id + /// The corpus id + /// The text id + /// + /// The book in USFM format + /// The specified book does not exist in the source or target corpus. + /// The corpus is not a valid Scripture corpus. + /// The client is not authenticated + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine or corpus does not exist. + /// The engine needs to be built first. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadTranslationEngines)] + [HttpGet("{id}/corpora/{corpusId}/pretranslations/{textId}/usfm")] + [Produces("text/plain")] + [ProducesResponseType(typeof(string), StatusCodes.Status200OK, "text/plain")] + [ProducesResponseType(typeof(void), StatusCodes.Status204NoContent)] + [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status409Conflict)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task GetPretranslatedUsfmAsync( + [NotNull] string id, + [NotNull] string corpusId, + [NotNull] string textId, + CancellationToken cancellationToken + ) + { + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); + if (!engine.Corpora.Any(c => c.Id == corpusId)) + return NotFound(); + if (engine.ModelRevision == 0) + return Conflict(); + + string usfm = await _pretranslationService.GetUsfmAsync( + id, + engine.ModelRevision, + corpusId, + textId, + cancellationToken + ); + if (usfm == "") + return NoContent(); + return Content(usfm, "text/plain"); + } + /// /// Get all build jobs for a translation engine /// /// The translation engine id /// /// The build jobs - /// The client is not authenticated - /// The authenticated client cannot perform the operation or does not own the translation engine - /// The engine does not exist - /// A necessary service is currently unavailable. Check `/health` for more details. + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the translation engine. + /// The engine does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.ReadTranslationEngines)] [HttpGet("{id}/builds")] [ProducesResponseType(StatusCodes.Status200OK)] @@ -648,9 +684,7 @@ public async Task>> GetAllBuildsAs CancellationToken cancellationToken ) { - if (!(await AuthorizeAsync(id, cancellationToken)).IsSuccess(out ActionResult? errorResult)) - return errorResult; - + await AuthorizeAsync(id, cancellationToken); return Ok((await _buildService.GetAllAsync(id, cancellationToken)).Select(Map)); } @@ -672,16 +706,14 @@ CancellationToken cancellationToken /// The minimum revision /// /// The build job - /// Bad request - /// The client is not authenticated - /// The authenticated client does not own the translation engine - /// The engine or build does not exist - /// The long polling request timed out. This is expected behavior if you're using long-polling with the minRevision strategy specified in the docs - /// A necessary service is currently unavailable. Check `/health` for more details. + /// The client is not authenticated. + /// The authenticated client does not own the translation engine. + /// The engine or build does not exist. + /// The long polling request timed out. This is expected behavior if you're using long-polling with the minRevision strategy specified in the docs. + /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.ReadTranslationEngines)] [HttpGet("{id}/builds/{buildId}", Name = "GetTranslationBuild")] [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] @@ -694,9 +726,7 @@ public async Task> GetBuildAsync( CancellationToken cancellationToken ) { - if (!(await AuthorizeAsync(id, cancellationToken)).IsSuccess(out ActionResult? errorResult)) - return errorResult; - + await AuthorizeAsync(id, cancellationToken); if (minRevision != null) { EntityChange change = await TaskEx.Timeout( @@ -713,10 +743,7 @@ CancellationToken cancellationToken } else { - Build? build = await _buildService.GetAsync(buildId, cancellationToken); - if (build == null) - return NotFound(); - + Build build = await _buildService.GetAsync(buildId, cancellationToken); return Ok(Map(build)); } } @@ -743,12 +770,12 @@ CancellationToken cancellationToken /// The translation engine id /// The build config (see remarks) /// - /// The build job was started successfully - /// A corpus id is invalid - /// The client is not authenticated - /// The authenticated client does not own the translation engine - /// The engine does not exist - /// There is already an active or pending build or a build in the process of being canceled + /// The new build job + /// A corpus id is invalid. + /// The client is not authenticated. + /// The authenticated client does not own the translation engine. + /// The engine does not exist. + /// There is already an active or pending build or a build in the process of being canceled. /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.UpdateTranslationEngines)] [HttpPost("{id}/builds")] @@ -765,30 +792,10 @@ public async Task> StartBuildAsync( CancellationToken cancellationToken ) { - Engine? engine = await _engineService.GetAsync(id, cancellationToken); - if (engine is null) - return NotFound(); - if (!await AuthorizeIsOwnerAsync(engine)) - return Forbid(); - - if (await _buildService.GetActiveAsync(id) is not null) - return Conflict(); - - Build build; - try - { - build = Map(engine, buildConfig); - } - catch (InvalidOperationException ioe) - { - return BadRequest(ioe.Message); - } - catch (ArgumentException ae) - { - return BadRequest(ae.Message); - } - if (!await _engineService.StartBuildAsync(build, cancellationToken)) - return NotFound(); + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); + Build build = Map(engine, buildConfig); + await _engineService.StartBuildAsync(build, cancellationToken); var dto = Map(build); return Created(dto.Url, dto); @@ -804,13 +811,13 @@ CancellationToken cancellationToken /// The minimum revision /// /// The build job - /// There is no build currently running + /// There is no build currently running. /// Bad request - /// The client is not authenticated - /// The authenticated client does not own the translation engine - /// The engine does not exist - /// The long polling request timed out. This is expected behavior if you're using long-polling with the minRevision strategy specified in the docs - /// A necessary service is currently unavailable. Check `/health` for more details. + /// The client is not authenticated. + /// The authenticated client does not own the translation engine. + /// The engine does not exist. + /// The long polling request timed out. This is expected behavior if you're using long-polling with the minRevision strategy specified in the docs. + /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.ReadTranslationEngines)] [HttpGet("{id}/current-build")] [ProducesResponseType(StatusCodes.Status200OK)] @@ -827,9 +834,7 @@ public async Task> GetCurrentBuildAsync( CancellationToken cancellationToken ) { - if (!(await AuthorizeAsync(id, cancellationToken)).IsSuccess(out ActionResult? errorResult)) - return errorResult; - + await AuthorizeAsync(id, cancellationToken); if (minRevision != null) { EntityChange change = await TaskEx.Timeout( @@ -861,15 +866,17 @@ CancellationToken cancellationToken /// /// The translation engine id /// - /// The build job was cancelled successfully - /// The client is not authenticated - /// The authenticated client does not own the translation engine - /// The engine does not exist or there is no active build - /// The translation engine does not support cancelling builds - /// A necessary service is currently unavailable. Check `/health` for more details. + /// The build job was cancelled successfully. + /// There is no active build job. + /// The client is not authenticated. + /// The authenticated client does not own the translation engine. + /// The engine does not exist. + /// The translation engine does not support cancelling builds. + /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.UpdateTranslationEngines)] [HttpPost("{id}/current-build/cancel")] [ProducesResponseType(typeof(void), StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status204NoContent)] [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] @@ -877,24 +884,16 @@ CancellationToken cancellationToken [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] public async Task CancelBuildAsync([NotNull] string id, CancellationToken cancellationToken) { - if (!(await AuthorizeAsync(id, cancellationToken)).IsSuccess(out ActionResult? errorResult)) - return errorResult; - - if (await _buildService.GetActiveAsync(id) == null) - return NotFound(); - - await _engineService.CancelBuildAsync(id, cancellationToken); + await AuthorizeAsync(id, cancellationToken); + if (!await _engineService.CancelBuildAsync(id, cancellationToken)) + return NoContent(); return Ok(); } - private async Task<(bool, ActionResult?)> AuthorizeAsync(string id, CancellationToken cancellationToken) + private async Task AuthorizeAsync(string id, CancellationToken cancellationToken) { - Engine? engine = await _engineService.GetAsync(id, cancellationToken); - if (engine == null) - return (false, NotFound()); - if (!await AuthorizeIsOwnerAsync(engine)) - return (false, Forbid()); - return (true, null); + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); } private async Task MapAsync( @@ -1001,7 +1000,7 @@ private static Build Map(Engine engine, TranslationBuildConfigDto source) } catch (Exception e) { - throw new ArgumentException($"Unable to parse field 'options' : {e.Message}"); + throw new InvalidOperationException($"Unable to parse field 'options' : {e.Message}", e); } return build; } diff --git a/src/Serval.Translation/Services/EngineService.cs b/src/Serval.Translation/Services/EngineService.cs index 8113b2f9..2159e42b 100644 --- a/src/Serval.Translation/Services/EngineService.cs +++ b/src/Serval.Translation/Services/EngineService.cs @@ -19,15 +19,13 @@ ILoggerFactory loggerFactory private readonly IDataAccessContext _dataAccessContext = dataAccessContext; private readonly ILogger _logger = loggerFactory.CreateLogger(); - public async Task TranslateAsync( + public async Task TranslateAsync( string engineId, string segment, CancellationToken cancellationToken = default ) { - Engine? engine = await GetAsync(engineId, cancellationToken); - if (engine == null) - return null; + Engine engine = await GetAsync(engineId, cancellationToken); var client = _grpcClientFactory.CreateClient(engine.Type); TranslateResponse response = await client.TranslateAsync( @@ -43,16 +41,14 @@ ILoggerFactory loggerFactory return Map(response.Results[0]); } - public async Task?> TranslateAsync( + public async Task> TranslateAsync( string engineId, int n, string segment, CancellationToken cancellationToken = default ) { - Engine? engine = await GetAsync(engineId, cancellationToken); - if (engine == null) - return null; + Engine engine = await GetAsync(engineId, cancellationToken); var client = _grpcClientFactory.CreateClient(engine.Type); TranslateResponse response = await client.TranslateAsync( @@ -68,15 +64,13 @@ ILoggerFactory loggerFactory return response.Results.Select(Map); } - public async Task GetWordGraphAsync( + public async Task GetWordGraphAsync( string engineId, string segment, CancellationToken cancellationToken = default ) { - Engine? engine = await GetAsync(engineId, cancellationToken); - if (engine == null) - return null; + Engine engine = await GetAsync(engineId, cancellationToken); var client = _grpcClientFactory.CreateClient(engine.Type); GetWordGraphResponse response = await client.GetWordGraphAsync( @@ -91,7 +85,7 @@ ILoggerFactory loggerFactory return Map(response.WordGraph); } - public async Task TrainSegmentPairAsync( + public async Task TrainSegmentPairAsync( string engineId, string sourceSegment, string targetSegment, @@ -99,9 +93,7 @@ public async Task TrainSegmentPairAsync( CancellationToken cancellationToken = default ) { - Engine? engine = await GetAsync(engineId, cancellationToken); - if (engine == null) - return false; + Engine engine = await GetAsync(engineId, cancellationToken); var client = _grpcClientFactory.CreateClient(engine.Type); await client.TrainSegmentPairAsync( @@ -115,7 +107,6 @@ await client.TrainSegmentPairAsync( }, cancellationToken: cancellationToken ); - return true; } public async Task> GetAllAsync(string owner, CancellationToken cancellationToken = default) @@ -123,7 +114,7 @@ public async Task> GetAllAsync(string owner, CancellationTok return await Entities.GetAllAsync(e => e.Owner == owner, cancellationToken); } - public override async Task CreateAsync(Engine engine, CancellationToken cancellationToken = default) + public override async Task CreateAsync(Engine engine, CancellationToken cancellationToken = default) { await Entities.InsertAsync(engine, cancellationToken); try @@ -131,7 +122,7 @@ public override async Task CreateAsync(Engine engine, CancellationToken ca TranslationEngineApi.TranslationEngineApiClient? client = _grpcClientFactory.CreateClient(engine.Type); if (client is null) - return false; + throw new InvalidOperationException($"'{engine.Type}' is an invalid engine type."); var request = new CreateRequest { EngineType = engine.Type, @@ -142,7 +133,6 @@ public override async Task CreateAsync(Engine engine, CancellationToken ca if (engine.Name is not null) request.EngineName = engine.Name; await client.CreateAsync(request, cancellationToken: cancellationToken); - return true; } catch { @@ -151,11 +141,11 @@ public override async Task CreateAsync(Engine engine, CancellationToken ca } } - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) { Engine? engine = await Entities.GetAsync(engineId, cancellationToken); - if (engine == null) - return false; + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{engineId}'."); var client = _grpcClientFactory.CreateClient(engine.Type); await client.DeleteAsync( @@ -168,16 +158,11 @@ await client.DeleteAsync( await _builds.DeleteAllAsync(b => b.EngineRef == engineId, CancellationToken.None); await _pretranslations.DeleteAllAsync(pt => pt.EngineRef == engineId, CancellationToken.None); await _dataAccessContext.CommitTransactionAsync(CancellationToken.None); - - return true; } - public async Task StartBuildAsync(Build build, CancellationToken cancellationToken = default) + public async Task StartBuildAsync(Build build, CancellationToken cancellationToken = default) { - Engine? engine = await GetAsync(build.EngineRef, cancellationToken); - if (engine == null) - return false; - + Engine engine = await GetAsync(build.EngineRef, cancellationToken); await _builds.InsertAsync(build, cancellationToken); try @@ -253,21 +238,29 @@ public async Task StartBuildAsync(Build build, CancellationToken cancellat await _builds.DeleteAsync(build, CancellationToken.None); throw; } - - return true; } - public async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + public async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) { Engine? engine = await GetAsync(engineId, cancellationToken); - if (engine == null) - return; + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{engineId}'."); var client = _grpcClientFactory.CreateClient(engine.Type); - await client.CancelBuildAsync( - new CancelBuildRequest { EngineType = engine.Type, EngineId = engine.Id }, - cancellationToken: cancellationToken - ); + try + { + await client.CancelBuildAsync( + new CancelBuildRequest { EngineType = engine.Type, EngineId = engine.Id }, + cancellationToken: cancellationToken + ); + } + catch (RpcException re) + { + if (re.StatusCode is StatusCode.Aborted) + return false; + throw; + } + return true; } public Task AddCorpusAsync(string engineId, Models.Corpus corpus, CancellationToken cancellationToken = default) @@ -275,7 +268,7 @@ public Task AddCorpusAsync(string engineId, Models.Corpus corpus, CancellationTo return Entities.UpdateAsync(engineId, u => u.Add(e => e.Corpora, corpus), cancellationToken: cancellationToken); } - public async Task UpdateCorpusAsync( + public async Task UpdateCorpusAsync( string engineId, string corpusId, IList? sourceFiles, @@ -294,14 +287,12 @@ public Task AddCorpusAsync(string engineId, Models.Corpus corpus, CancellationTo }, cancellationToken: cancellationToken ); - return engine?.Corpora.FirstOrDefault(c => c.Id == corpusId); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Corpus '{corpusId}' in Engine '{engineId}'."); + return engine.Corpora.First(c => c.Id == corpusId); } - public async Task DeleteCorpusAsync( - string engineId, - string corpusId, - CancellationToken cancellationToken = default - ) + public async Task DeleteCorpusAsync(string engineId, string corpusId, CancellationToken cancellationToken = default) { await _dataAccessContext.BeginTransactionAsync(cancellationToken); Engine? originalEngine = await Entities.UpdateAsync( @@ -310,9 +301,10 @@ public async Task DeleteCorpusAsync( returnOriginal: true, cancellationToken: cancellationToken ); + if (originalEngine is null || !originalEngine.Corpora.Any(c => c.Id == corpusId)) + throw new EntityNotFoundException($"Could not find the Corpus '{corpusId}' in Engine '{engineId}'."); await _pretranslations.DeleteAllAsync(pt => pt.CorpusRef == corpusId, cancellationToken); await _dataAccessContext.CommitTransactionAsync(cancellationToken); - return originalEngine is not null && originalEngine.Corpora.Any(c => c.Id == corpusId); } public Task DeleteAllCorpusFilesAsync(string dataFileId, CancellationToken cancellationToken = default) diff --git a/src/Serval.Translation/Services/IBuildService.cs b/src/Serval.Translation/Services/IBuildService.cs index e3ab12a6..4fc13dee 100644 --- a/src/Serval.Translation/Services/IBuildService.cs +++ b/src/Serval.Translation/Services/IBuildService.cs @@ -3,7 +3,7 @@ public interface IBuildService { Task> GetAllAsync(string parentId, CancellationToken cancellationToken = default); - Task GetAsync(string id, CancellationToken cancellationToken = default); + Task GetAsync(string id, CancellationToken cancellationToken = default); Task GetActiveAsync(string parentId, CancellationToken cancellationToken = default); Task> GetNewerRevisionAsync( string id, diff --git a/src/Serval.Translation/Services/IEngineService.cs b/src/Serval.Translation/Services/IEngineService.cs index fa88dfa7..49cafbf1 100644 --- a/src/Serval.Translation/Services/IEngineService.cs +++ b/src/Serval.Translation/Services/IEngineService.cs @@ -3,27 +3,27 @@ public interface IEngineService { Task> GetAllAsync(string owner, CancellationToken cancellationToken = default); - Task GetAsync(string engineId, CancellationToken cancellationToken = default); + Task GetAsync(string engineId, CancellationToken cancellationToken = default); - Task CreateAsync(Engine engine, CancellationToken cancellationToken = default); - Task DeleteAsync(string engineId, CancellationToken cancellationToken = default); + Task CreateAsync(Engine engine, CancellationToken cancellationToken = default); + Task DeleteAsync(string engineId, CancellationToken cancellationToken = default); - Task TranslateAsync( + Task TranslateAsync( string engineId, string segment, CancellationToken cancellationToken = default ); - Task?> TranslateAsync( + Task> TranslateAsync( string engineId, int n, string segment, CancellationToken cancellationToken = default ); - Task GetWordGraphAsync(string engineId, string segment, CancellationToken cancellationToken = default); + Task GetWordGraphAsync(string engineId, string segment, CancellationToken cancellationToken = default); - Task TrainSegmentPairAsync( + Task TrainSegmentPairAsync( string engineId, string sourceSegment, string targetSegment, @@ -31,19 +31,19 @@ Task TrainSegmentPairAsync( CancellationToken cancellationToken = default ); - Task StartBuildAsync(Build build, CancellationToken cancellationToken = default); + Task StartBuildAsync(Build build, CancellationToken cancellationToken = default); - Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default); + Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default); Task AddCorpusAsync(string engineId, Corpus corpus, CancellationToken cancellationToken = default); - Task UpdateCorpusAsync( + Task UpdateCorpusAsync( string engineId, string corpusId, IList? sourceFiles, IList? targetFiles, CancellationToken cancellationToken = default ); - Task DeleteCorpusAsync(string engineId, string corpusId, CancellationToken cancellationToken = default); + Task DeleteCorpusAsync(string engineId, string corpusId, CancellationToken cancellationToken = default); Task DeleteAllCorpusFilesAsync(string dataFileId, CancellationToken cancellationToken = default); diff --git a/src/Serval.Translation/Services/IPretranslationService.cs b/src/Serval.Translation/Services/IPretranslationService.cs index 40ea2920..146c945b 100644 --- a/src/Serval.Translation/Services/IPretranslationService.cs +++ b/src/Serval.Translation/Services/IPretranslationService.cs @@ -9,4 +9,12 @@ Task> GetAllAsync( string? textId = null, CancellationToken cancellationToken = default ); + + Task GetUsfmAsync( + string engineId, + int modelRevision, + string corpusId, + string textId, + CancellationToken cancellationToken = default + ); } diff --git a/src/Serval.Translation/Services/PretranslationService.cs b/src/Serval.Translation/Services/PretranslationService.cs index 917f1239..589b7452 100644 --- a/src/Serval.Translation/Services/PretranslationService.cs +++ b/src/Serval.Translation/Services/PretranslationService.cs @@ -1,9 +1,15 @@ -namespace Serval.Translation.Services; +using SIL.Machine.Corpora; -public class PretranslationService : EntityServiceBase, IPretranslationService +namespace Serval.Translation.Services; + +public class PretranslationService( + IRepository pretranslations, + IRepository engines, + IScriptureDataFileService scriptureDataFileService +) : EntityServiceBase(pretranslations), IPretranslationService { - public PretranslationService(IRepository pretranslations) - : base(pretranslations) { } + private readonly IRepository _engines = engines; + private readonly IScriptureDataFileService _scriptureDataFileService = scriptureDataFileService; public async Task> GetAllAsync( string engineId, @@ -22,4 +28,71 @@ public async Task> GetAllAsync( cancellationToken ); } + + public async Task GetUsfmAsync( + string engineId, + int modelRevision, + string corpusId, + string textId, + CancellationToken cancellationToken = default + ) + { + Engine? engine = await _engines.GetAsync(engineId, cancellationToken); + Corpus? corpus = engine?.Corpora.SingleOrDefault(c => c.Id == corpusId); + if (corpus is null) + throw new EntityNotFoundException($"Could not find the Corpus '{corpusId}' in Engine '{engineId}'."); + + CorpusFile sourceFile = corpus.SourceFiles.First(); + CorpusFile targetFile = corpus.TargetFiles.First(); + if (sourceFile.Format is not FileFormat.Paratext || targetFile.Format is not FileFormat.Paratext) + throw new InvalidOperationException("USFM format is not valid for non-Scripture corpora."); + + ParatextProjectSettings sourceSettings = _scriptureDataFileService.GetParatextProjectSettings( + sourceFile.Filename + ); + ParatextProjectSettings targetSettings = _scriptureDataFileService.GetParatextProjectSettings( + targetFile.Filename + ); + + IReadOnlyList<(IReadOnlyList, string)> pretranslations = ( + await GetAllAsync(engineId, modelRevision, corpusId, textId, cancellationToken) + ) + .Select(p => + ( + (IReadOnlyList)p.Refs.Select(r => new VerseRef(r, targetSettings.Versification)).ToList(), + p.Translation + ) + ) + .OrderBy(p => p.Item1[0]) + .ToList(); + + // Update the target book if it exists + string? usfm = await _scriptureDataFileService.ReadParatextProjectBookAsync(targetFile.Filename, textId); + if (usfm is not null) + return UpdateUsfm(targetSettings, usfm, pretranslations); + + // Copy and update the source book if it exists + usfm = await _scriptureDataFileService.ReadParatextProjectBookAsync(sourceFile.Filename, textId); + if (usfm is not null) + return UpdateUsfm(sourceSettings, usfm, pretranslations, targetSettings.FullName, stripAllText: true); + + return ""; + } + + private static string UpdateUsfm( + ParatextProjectSettings settings, + string usfm, + IReadOnlyList<(IReadOnlyList, string)> pretranslations, + string? fullName = null, + bool stripAllText = false + ) + { + var updater = new UsfmVerseTextUpdater( + pretranslations, + fullName is null ? null : $"- {fullName}", + stripAllText + ); + UsfmParser.Parse(usfm, updater, settings.Stylesheet, settings.Versification); + return updater.GetUsfm(settings.Stylesheet); + } } diff --git a/src/Serval.Translation/Usings.cs b/src/Serval.Translation/Usings.cs index 77bb4439..1fda2ed5 100644 --- a/src/Serval.Translation/Usings.cs +++ b/src/Serval.Translation/Usings.cs @@ -27,3 +27,4 @@ global using Serval.Translation.Models; global using Serval.Translation.Services; global using SIL.DataAccess; +global using SIL.Scripture; diff --git a/src/Serval.Webhooks/Controllers/WebhooksController.cs b/src/Serval.Webhooks/Controllers/WebhooksController.cs index d7bf81a7..83cd5ba8 100644 --- a/src/Serval.Webhooks/Controllers/WebhooksController.cs +++ b/src/Serval.Webhooks/Controllers/WebhooksController.cs @@ -45,12 +45,8 @@ public async Task> GetAllAsync(CancellationToken cancell [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] public async Task> GetAsync([NotNull] string id, CancellationToken cancellationToken) { - Webhook? hook = await _hookService.GetAsync(id, cancellationToken); - if (hook == null) - return NotFound(); - if (!await AuthorizeIsOwnerAsync(hook)) - return Forbid(); - + Webhook hook = await _hookService.GetAsync(id, cancellationToken); + await AuthorizeAsync(hook); return Ok(Map(hook)); } @@ -93,17 +89,17 @@ CancellationToken cancellationToken [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] public async Task DeleteAsync([NotNull] string id, CancellationToken cancellationToken) { - Webhook? hook = await _hookService.GetAsync(id, cancellationToken); - if (hook == null) - return NotFound(); - if (!await AuthorizeIsOwnerAsync(hook)) - return Forbid(); - - if (!await _hookService.DeleteAsync(id, cancellationToken)) - return NotFound(); + await AuthorizeAsync(id, cancellationToken); + await _hookService.DeleteAsync(id, cancellationToken); return Ok(); } + private async Task AuthorizeAsync(string id, CancellationToken cancellationToken) + { + Webhook hook = await _hookService.GetAsync(id, cancellationToken); + await AuthorizeAsync(hook); + } + private WebhookDto Map(Webhook source) { return new WebhookDto diff --git a/src/Serval.Webhooks/Services/IWebhookService.cs b/src/Serval.Webhooks/Services/IWebhookService.cs index 03e479a6..30a8e91c 100644 --- a/src/Serval.Webhooks/Services/IWebhookService.cs +++ b/src/Serval.Webhooks/Services/IWebhookService.cs @@ -3,10 +3,10 @@ public interface IWebhookService { Task> GetAllAsync(string owner, CancellationToken cancellationToken = default); - Task GetAsync(string id, CancellationToken cancellationToken = default); + Task GetAsync(string id, CancellationToken cancellationToken = default); Task CreateAsync(Webhook hook, CancellationToken cancellationToken = default); - Task DeleteAsync(string id, CancellationToken cancellationToken = default); + Task DeleteAsync(string id, CancellationToken cancellationToken = default); Task SendEventAsync( WebhookEvent webhookEvent, diff --git a/tests/SIL.DataAccess.Tests/SIL.DataAccess.Tests.csproj b/tests/SIL.DataAccess.Tests/SIL.DataAccess.Tests.csproj index 9f7b61a8..8848ce87 100644 --- a/tests/SIL.DataAccess.Tests/SIL.DataAccess.Tests.csproj +++ b/tests/SIL.DataAccess.Tests/SIL.DataAccess.Tests.csproj @@ -13,15 +13,15 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/Serval.ApiServer.IntegrationTests/DataFilesTests.cs b/tests/Serval.ApiServer.IntegrationTests/DataFilesTests.cs index d3737abd..6828234a 100644 --- a/tests/Serval.ApiServer.IntegrationTests/DataFilesTests.cs +++ b/tests/Serval.ApiServer.IntegrationTests/DataFilesTests.cs @@ -4,7 +4,7 @@ namespace Serval.ApiServer; [Category("Integration")] public class DataFilesTests { - TestEnvironment? _env; + TestEnvironment _env; const string ID1 = "000000000000000000000001"; const string NAME1 = "sample1.txt"; @@ -53,7 +53,7 @@ public async Task SetUp() [TestCase(new[] { Scopes.CreateTranslationEngines }, 403)] public async Task GetAllAsync(IEnumerable scope, int expectedStatusCode) { - DataFilesClient client = _env!.CreateClient(scope); + DataFilesClient client = _env.CreateClient(scope); switch (expectedStatusCode) { case 200: @@ -72,7 +72,7 @@ public async Task GetAllAsync(IEnumerable scope, int expectedStatusCode) await client.GetAllAsync(); }); Assert.That(ex, Is.Not.Null); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; default: @@ -89,7 +89,7 @@ public async Task GetAllAsync(IEnumerable scope, int expectedStatusCode) [TestCase(new[] { Scopes.ReadFiles }, 404, DOES_NOT_EXIST_ID)] public async Task GetByIDAsync(IEnumerable scope, int expectedStatusCode, string fileId) { - DataFilesClient client = _env!.CreateClient(scope); + DataFilesClient client = _env.CreateClient(scope); switch (expectedStatusCode) { case 200: @@ -105,7 +105,7 @@ public async Task GetByIDAsync(IEnumerable scope, int expectedStatusCode { await client.GetAsync(fileId); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; default: Assert.Fail("Invalid expectedStatusCode. Check test case for typo."); @@ -120,8 +120,7 @@ public async Task GetByIDAsync(IEnumerable scope, int expectedStatusCode [TestCase(new[] { Scopes.ReadFiles }, 403)] public async Task CreateAsync(IEnumerable scope, int expectedStatusCode) { - DataFilesClient client = _env!.CreateClient(scope); - ServalApiException ex; + DataFilesClient client = _env.CreateClient(scope); switch (expectedStatusCode) { case 201: @@ -141,23 +140,23 @@ public async Task CreateAsync(IEnumerable scope, int expectedStatusCode) { var fp = new FileParameter(fs); fp = new FileParameter(fs); - ex = Assert.ThrowsAsync(async () => + var ex = Assert.ThrowsAsync(async () => { - await client.CreateAsync(fp, Client.FileFormat.Text); + await client.CreateAsync(fp, FileFormat.Text); }); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); } - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); break; case 403: using (var fs = new MemoryStream()) { var fp = new FileParameter(fs); - ex = Assert.ThrowsAsync(async () => + var ex = Assert.ThrowsAsync(async () => { await client.CreateAsync(fp, Client.FileFormat.Text); }); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); } - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); break; default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); @@ -172,8 +171,7 @@ public async Task CreateAsync(IEnumerable scope, int expectedStatusCode) [TestCase(new[] { Scopes.CreateFiles, Scopes.ReadFiles }, 404, DOES_NOT_EXIST_ID)] public async Task DownloadAsync(IEnumerable scope, int expectedStatusCode, string fileId) { - DataFilesClient client = _env!.CreateClient(scope); - ServalApiException ex; + DataFilesClient client = _env.CreateClient(scope); string content = "This is a file."; var contentBytes = Encoding.UTF8.GetBytes(content); @@ -206,20 +204,24 @@ public async Task DownloadAsync(IEnumerable scope, int expectedStatusCod } break; case 400: - ex = Assert.ThrowsAsync(async () => + { + var ex = Assert.ThrowsAsync(async () => { await client.DownloadAsync(fileId); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; + } case 403: case 404: - ex = Assert.ThrowsAsync(async () => + { + var ex = Assert.ThrowsAsync(async () => { await client.DownloadAsync(fileId); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; + } default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); break; @@ -234,8 +236,7 @@ public async Task DownloadAsync(IEnumerable scope, int expectedStatusCod [TestCase(new[] { Scopes.UpdateFiles, Scopes.ReadFiles }, 404, DOES_NOT_EXIST_ID)] public async Task UpdateAsync(IEnumerable scope, int expectedStatusCode, string fileId) { - DataFilesClient client = _env!.CreateClient(scope); - ServalApiException ex; + DataFilesClient client = _env.CreateClient(scope); switch (expectedStatusCode) { case 200: @@ -249,20 +250,24 @@ public async Task UpdateAsync(IEnumerable scope, int expectedStatusCode, Assert.That(resultAfterUpdate.Id, Is.EqualTo(ID1)); break; case 400: - ex = Assert.ThrowsAsync(async () => + { + var ex = Assert.ThrowsAsync(async () => { await client.UpdateAsync(fileId, new FileParameter(new MemoryStream(new byte[2_000_000_000]))); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; + } case 403: case 404: - ex = Assert.ThrowsAsync(async () => + { + var ex = Assert.ThrowsAsync(async () => { await client.UpdateAsync(fileId, new FileParameter(new MemoryStream())); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; + } default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); break; @@ -277,7 +282,7 @@ public async Task UpdateAsync(IEnumerable scope, int expectedStatusCode, [TestCase(new[] { Scopes.DeleteFiles, Scopes.ReadFiles }, 404, DOES_NOT_EXIST_ID)] public async Task DeleteAsync(IEnumerable scope, int expectedStatusCode, string fileId) { - DataFilesClient client = _env!.CreateClient(scope); + DataFilesClient client = _env.CreateClient(scope); switch (expectedStatusCode) { case 200: @@ -297,7 +302,7 @@ public async Task DeleteAsync(IEnumerable scope, int expectedStatusCode, { await client.DeleteAsync(fileId); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); ICollection resultsAfterDelete = await client.GetAllAsync(); Assert.That(resultsAfterDelete, Has.Count.EqualTo(2)); break; @@ -310,12 +315,12 @@ public async Task DeleteAsync(IEnumerable scope, int expectedStatusCode, [TearDown] public void TearDown() { - _env!.Dispose(); + _env.Dispose(); } private class TestEnvironment : DisposableBase { - private readonly IMongoClient _mongoClient; + private readonly MongoClient _mongoClient; private readonly IServiceScope _scope; public TestEnvironment() diff --git a/tests/Serval.ApiServer.IntegrationTests/Serval.ApiServer.IntegrationTests.csproj b/tests/Serval.ApiServer.IntegrationTests/Serval.ApiServer.IntegrationTests.csproj index a05921d4..89dcc1e1 100644 --- a/tests/Serval.ApiServer.IntegrationTests/Serval.ApiServer.IntegrationTests.csproj +++ b/tests/Serval.ApiServer.IntegrationTests/Serval.ApiServer.IntegrationTests.csproj @@ -17,16 +17,16 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs b/tests/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs index 85257e29..1fadc355 100644 --- a/tests/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs +++ b/tests/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs @@ -7,7 +7,7 @@ namespace Serval.ApiServer; [Category("Integration")] public class TranslationEngineTests { - private readonly TranslationCorpusConfig TestCorpusConfig = + private static readonly TranslationCorpusConfig TestCorpusConfig = new() { Name = "TestCorpus", @@ -22,7 +22,7 @@ public class TranslationEngineTests new TranslationCorpusFileConfig { FileId = FILE2_ID, TextId = "all" } } }; - private readonly TranslationCorpusConfig TestCorpusConfigNonEcho = + private static readonly TranslationCorpusConfig TestCorpusConfigNonEcho = new() { Name = "TestCorpus", @@ -38,6 +38,16 @@ public class TranslationEngineTests } }; + private static readonly TranslationCorpusConfig TestCorpusConfigScripture = + new() + { + Name = "TestCorpus", + SourceLanguage = "en", + TargetLanguage = "en", + SourceFiles = { new TranslationCorpusFileConfig { FileId = FILE3_ID } }, + TargetFiles = { new TranslationCorpusFileConfig { FileId = FILE4_ID } } + }; + private const string ECHO_ENGINE1_ID = "e00000000000000000000001"; private const string ECHO_ENGINE2_ID = "e00000000000000000000002"; private const string ECHO_ENGINE3_ID = "e00000000000000000000003"; @@ -47,10 +57,15 @@ public class TranslationEngineTests private const string FILE1_FILENAME = "abcd"; private const string FILE2_ID = "f00000000000000000000002"; private const string FILE2_FILENAME = "efgh"; + private const string FILE3_ID = "f00000000000000000000003"; + private const string FILE3_FILENAME = "ijkl"; + private const string FILE4_ID = "f00000000000000000000004"; + private const string FILE4_FILENAME = "mnop"; + private const string DOES_NOT_EXIST_ENGINE_ID = "e00000000000000000000004"; private const string DOES_NOT_EXIST_CORPUS_ID = "c00000000000000000000001"; - private TestEnvironment? _env; + private TestEnvironment _env; [SetUp] public async Task SetUp() @@ -64,7 +79,7 @@ public async Task SetUp() TargetLanguage = "en", Type = "Echo", Owner = "client1", - Corpora = new List() + Corpora = [] }; var e1 = new Engine { @@ -74,7 +89,7 @@ public async Task SetUp() TargetLanguage = "en", Type = "Echo", Owner = "client1", - Corpora = new List() + Corpora = [] }; var e2 = new Engine { @@ -84,7 +99,7 @@ public async Task SetUp() TargetLanguage = "en", Type = "Echo", Owner = "client2", - Corpora = new List() + Corpora = [] }; var be0 = new Engine { @@ -94,7 +109,7 @@ public async Task SetUp() TargetLanguage = "es", Type = "SMTTransfer", Owner = "client1", - Corpora = new List() + Corpora = [] }; var ce0 = new Engine { @@ -104,7 +119,7 @@ public async Task SetUp() TargetLanguage = "es", Type = "Nmt", Owner = "client1", - Corpora = new List() + Corpora = [] }; await _env.Engines.InsertAllAsync(new[] { e0, e1, e2, be0, ce0 }); @@ -125,7 +140,23 @@ public async Task SetUp() Filename = FILE2_FILENAME, Format = Shared.Contracts.FileFormat.Text }; - await _env.DataFiles.InsertAllAsync(new[] { srcFile, trgFile }); + var srcParatextFile = new DataFiles.Models.DataFile + { + Id = FILE3_ID, + Owner = "client1", + Name = "src.zip", + Filename = FILE3_FILENAME, + Format = Shared.Contracts.FileFormat.Paratext + }; + var trgParatextFile = new DataFiles.Models.DataFile + { + Id = FILE4_ID, + Owner = "client1", + Name = "trg.zip", + Filename = FILE4_FILENAME, + Format = Shared.Contracts.FileFormat.Paratext + }; + await _env.DataFiles.InsertAllAsync([srcFile, trgFile, srcParatextFile, trgParatextFile]); } [Test] @@ -133,7 +164,7 @@ public async Task SetUp() [TestCase(new[] { Scopes.ReadFiles }, 403)] //Arbitrary unrelated privilege public async Task GetAllAsync(IEnumerable scope, int expectedStatusCode) { - ITranslationEnginesClient client = _env!.CreateClient(scope); + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); switch (expectedStatusCode) { case 200: @@ -146,7 +177,7 @@ public async Task GetAllAsync(IEnumerable scope, int expectedStatusCode) { await client.GetAllAsync(); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); @@ -161,7 +192,7 @@ public async Task GetAllAsync(IEnumerable scope, int expectedStatusCode) [TestCase(new[] { Scopes.ReadTranslationEngines }, 404, DOES_NOT_EXIST_ENGINE_ID)] public async Task GetByIdAsync(IEnumerable scope, int expectedStatusCode, string engineId) { - ITranslationEnginesClient client = _env!.CreateClient(scope); + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); switch (expectedStatusCode) { case 200: @@ -174,7 +205,7 @@ public async Task GetByIdAsync(IEnumerable scope, int expectedStatusCode { await client.GetAsync(engineId); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); @@ -188,8 +219,7 @@ public async Task GetByIdAsync(IEnumerable scope, int expectedStatusCode [TestCase(new[] { Scopes.ReadFiles }, 403, "Echo")] //Arbitrary unrelated privilege public async Task CreateEngineAsync(IEnumerable scope, int expectedStatusCode, string engineType) { - ITranslationEnginesClient client = _env!.CreateClient(scope); - ServalApiException? ex; + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); switch (expectedStatusCode) { case 201: @@ -208,7 +238,8 @@ public async Task CreateEngineAsync(IEnumerable scope, int expectedStatu Assert.That(engine.Name, Is.EqualTo("test")); break; case 400: - ex = Assert.ThrowsAsync(async () => + { + var ex = Assert.ThrowsAsync(async () => { await client.CreateAsync( new TranslationEngineConfig @@ -220,10 +251,12 @@ await client.CreateAsync( } ); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; + } case 403: - ex = Assert.ThrowsAsync(async () => + { + var ex = Assert.ThrowsAsync(async () => { await client.CreateAsync( new TranslationEngineConfig @@ -235,8 +268,9 @@ await client.CreateAsync( } ); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; + } default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); break; @@ -249,7 +283,7 @@ await client.CreateAsync( [TestCase(new[] { Scopes.DeleteTranslationEngines }, 404, DOES_NOT_EXIST_ENGINE_ID)] public async Task DeleteEngineByIdAsync(IEnumerable scope, int expectedStatusCode, string engineId) { - ITranslationEnginesClient client = _env!.CreateClient(scope); + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); switch (expectedStatusCode) { case 200: @@ -264,7 +298,7 @@ public async Task DeleteEngineByIdAsync(IEnumerable scope, int expectedS { await client.DeleteAsync(engineId); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); @@ -283,8 +317,7 @@ public async Task TranslateSegmentWithEngineByIdAsync( string engineId ) { - ITranslationEnginesClient client = _env!.CreateClient(scope); - ServalApiException ex; + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); switch (expectedStatusCode) { case 200: @@ -300,22 +333,26 @@ await _env.Builds.InsertAsync( ); break; case 409: + { _env.EchoClient.TranslateAsync(Arg.Any(), null, null, Arg.Any()) .Returns(CreateAsyncUnaryCall(StatusCode.Aborted)); - ex = Assert.ThrowsAsync(async () => + var ex = Assert.ThrowsAsync(async () => { await client.TranslateAsync(engineId, "This is a test ."); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; + } case 403: case 404: - ex = Assert.ThrowsAsync(async () => + { + var ex = Assert.ThrowsAsync(async () => { await client.TranslateAsync(engineId, "This is a test ."); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; + } default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); break; @@ -333,8 +370,7 @@ public async Task TranslateNSegmentWithEngineByIdAsync( string engineId ) { - ITranslationEnginesClient client = _env!.CreateClient(scope); - ServalApiException ex; + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); switch (expectedStatusCode) { case 200: @@ -355,22 +391,26 @@ await _env.Builds.InsertAsync( ); break; case 409: + { _env.EchoClient.TranslateAsync(Arg.Any(), null, null, Arg.Any()) .Returns(CreateAsyncUnaryCall(StatusCode.Aborted)); - ex = Assert.ThrowsAsync(async () => + var ex = Assert.ThrowsAsync(async () => { await client.TranslateNAsync(engineId, 1, "This is a test ."); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; + } case 403: case 404: - ex = Assert.ThrowsAsync(async () => + { + var ex = Assert.ThrowsAsync(async () => { await client.TranslateNAsync(engineId, 1, "This is a test ."); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; + } default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); break; @@ -388,8 +428,7 @@ public async Task GetWordGraphForSegmentByIdAsync( string engineId ) { - ITranslationEnginesClient client = _env!.CreateClient(scope); - ServalApiException ex; + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); switch (expectedStatusCode) { case 200: @@ -405,6 +444,7 @@ await _env.Builds.InsertAsync( }); break; case 409: + { _env.EchoClient.GetWordGraphAsync( Arg.Any(), null, @@ -412,20 +452,23 @@ await _env.Builds.InsertAsync( Arg.Any() ) .Returns(CreateAsyncUnaryCall(StatusCode.Aborted)); - ex = Assert.ThrowsAsync(async () => + var ex = Assert.ThrowsAsync(async () => { await client.GetWordGraphAsync(engineId, "This is a test ."); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; + } case 403: case 404: - ex = Assert.ThrowsAsync(async () => + { + var ex = Assert.ThrowsAsync(async () => { await client.GetWordGraphAsync(engineId, "This is a test ."); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; + } default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); break; @@ -443,14 +486,13 @@ public async Task TrainEngineByIdOnSegmentPairAsync( string engineId ) { - ITranslationEnginesClient client = _env!.CreateClient(scope); + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); var sp = new SegmentPair { SourceSegment = "This is a test .", TargetSegment = "This is a test .", SentenceStart = true }; - ServalApiException ex; switch (expectedStatusCode) { case 200: @@ -461,6 +503,7 @@ await _env.Builds.InsertAsync( await client.TrainSegmentAsync(engineId, sp); break; case 409: + { _env.EchoClient.TrainSegmentPairAsync( Arg.Any(), null, @@ -468,20 +511,23 @@ await _env.Builds.InsertAsync( Arg.Any() ) .Returns(CreateAsyncUnaryCall(StatusCode.Aborted)); - ex = Assert.ThrowsAsync(async () => + var ex = Assert.ThrowsAsync(async () => { await client.TrainSegmentAsync(engineId, sp); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; + } case 403: case 404: - ex = Assert.ThrowsAsync(async () => + { + var ex = Assert.ThrowsAsync(async () => { await client.TrainSegmentAsync(engineId, sp); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; + } default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); break; @@ -494,10 +540,11 @@ await _env.Builds.InsertAsync( [TestCase(new[] { Scopes.ReadFiles }, 403, ECHO_ENGINE1_ID)] //Arbitrary unrelated privilege public async Task AddCorpusToEngineByIdAsync(IEnumerable scope, int expectedStatusCode, string engineId) { - ITranslationEnginesClient client = _env!.CreateClient(scope); + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); switch (expectedStatusCode) { case 201: + { TranslationCorpus result = await client.AddCorpusAsync(engineId, TestCorpusConfig); Assert.Multiple(() => { @@ -509,17 +556,18 @@ public async Task AddCorpusToEngineByIdAsync(IEnumerable scope, int expe Assert.That(engine, Is.Not.Null); Assert.Multiple(() => { - Assert.That(engine!.Corpora[0].SourceFiles[0].Filename, Is.EqualTo(FILE1_FILENAME)); + Assert.That(engine.Corpora[0].SourceFiles[0].Filename, Is.EqualTo(FILE1_FILENAME)); Assert.That(engine.Corpora[0].TargetFiles[0].Filename, Is.EqualTo(FILE2_FILENAME)); }); break; + } case 403: case 404: var ex = Assert.ThrowsAsync(async () => { - result = await client.AddCorpusAsync(engineId, TestCorpusConfig); + await client.AddCorpusAsync(engineId, TestCorpusConfig); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); @@ -550,10 +598,11 @@ public async Task UpdateCorpusByIdForEngineByIdAsync( string engineId ) { - TranslationEnginesClient client = _env!.CreateClient(scope); + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); switch (expectedStatusCode) { case 200: + { TranslationCorpus result = await client.AddCorpusAsync(engineId, TestCorpusConfig); var src = new[] { @@ -569,10 +618,11 @@ string engineId Assert.That(engine, Is.Not.Null); Assert.Multiple(() => { - Assert.That(engine!.Corpora[0].SourceFiles[0].Filename, Is.EqualTo(FILE2_FILENAME)); + Assert.That(engine.Corpora[0].SourceFiles[0].Filename, Is.EqualTo(FILE2_FILENAME)); Assert.That(engine.Corpora[0].TargetFiles[0].Filename, Is.EqualTo(FILE1_FILENAME)); }); break; + } case 400: case 403: case 404: @@ -589,7 +639,7 @@ string engineId var updateConfig = new TranslationCorpusUpdateConfig { SourceFiles = src, TargetFiles = trg }; await client.UpdateCorpusAsync(engineId, DOES_NOT_EXIST_CORPUS_ID, updateConfig); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); @@ -615,8 +665,7 @@ public async Task GetAllCorporaForEngineByIdAsync( string engineId ) { - ITranslationEnginesClient client = _env!.CreateClient(scope); - ServalApiException? ex; + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); switch (expectedStatusCode) { case 200: @@ -631,11 +680,11 @@ string engineId break; case 403: case 404: - ex = Assert.ThrowsAsync(async () => + var ex = Assert.ThrowsAsync(async () => { TranslationCorpus result = (await client.GetAllCorporaAsync(engineId)).First(); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; default: @@ -657,8 +706,7 @@ public async Task GetCorpusByIdForEngineByIdAsync( bool addCorpus = false ) { - ITranslationEnginesClient client = _env!.CreateClient(scope); - ServalApiException? ex; + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); TranslationCorpus? result = null; if (addCorpus) { @@ -667,9 +715,9 @@ public async Task GetCorpusByIdForEngineByIdAsync( switch (expectedStatusCode) { case 200: - if (result is null) - Assert.Fail(); - TranslationCorpus resultAfterAdd = await client.GetCorpusAsync(engineId, result!.Id); + { + Assert.That(result, Is.Not.Null); + TranslationCorpus resultAfterAdd = await client.GetCorpusAsync(engineId, result.Id); Assert.Multiple(() => { Assert.That(resultAfterAdd.Name, Is.EqualTo(result.Name)); @@ -677,13 +725,14 @@ public async Task GetCorpusByIdForEngineByIdAsync( Assert.That(resultAfterAdd.TargetLanguage, Is.EqualTo(result.TargetLanguage)); }); break; + } case 403: case 404: - ex = Assert.ThrowsAsync(async () => + var ex = Assert.ThrowsAsync(async () => { TranslationCorpus result_afterAdd = await client.GetCorpusAsync(engineId, DOES_NOT_EXIST_CORPUS_ID); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; default: @@ -703,8 +752,7 @@ public async Task DeleteCorpusByIdForEngineByIdAsync( string engineId ) { - ITranslationEnginesClient client = _env!.CreateClient(scope); - ServalApiException? ex; + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); switch (expectedStatusCode) { case 200: @@ -715,11 +763,11 @@ string engineId break; case 403: case 404: - ex = Assert.ThrowsAsync(async () => + var ex = Assert.ThrowsAsync(async () => { await client.DeleteCorpusAsync(engineId, DOES_NOT_EXIST_CORPUS_ID); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; default: @@ -731,7 +779,7 @@ string engineId [Test] public async Task GetAllPretranslationsAsync_Exists() { - ITranslationEnginesClient client = _env!.CreateClient(); + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(); TranslationCorpus addedCorpus = await client.AddCorpusAsync(ECHO_ENGINE1_ID, TestCorpusConfig); await _env.Engines.UpdateAsync(ECHO_ENGINE1_ID, u => u.Set(e => e.ModelRevision, 1)); @@ -740,7 +788,7 @@ public async Task GetAllPretranslationsAsync_Exists() CorpusRef = addedCorpus.Id, TextId = "all", EngineRef = ECHO_ENGINE1_ID, - Refs = new List { "ref1", "ref2" }, + Refs = ["ref1", "ref2"], Translation = "translation", ModelRevision = 1 }; @@ -756,41 +804,41 @@ public async Task GetAllPretranslationsAsync_Exists() [Test] public void GetAllPretranslationsAsync_EngineDoesNotExist() { - ITranslationEnginesClient client = _env!.CreateClient(); + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(); var ex = Assert.ThrowsAsync( () => client.GetAllPretranslationsAsync(DOES_NOT_EXIST_ENGINE_ID, "cccccccccccccccccccccccc") ); - Assert.That(ex!.StatusCode, Is.EqualTo(404)); + Assert.That(ex?.StatusCode, Is.EqualTo(404)); } [Test] public void GetAllPretranslationsAsync_CorpusDoesNotExist() { - ITranslationEnginesClient client = _env!.CreateClient(); + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(); var ex = Assert.ThrowsAsync( () => client.GetAllPretranslationsAsync(ECHO_ENGINE1_ID, "cccccccccccccccccccccccc") ); - Assert.That(ex!.StatusCode, Is.EqualTo(404)); + Assert.That(ex?.StatusCode, Is.EqualTo(404)); } [Test] public async Task GetAllPretranslationsAsync_EngineNotBuilt() { - ITranslationEnginesClient client = _env!.CreateClient(); + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(); TranslationCorpus addedCorpus = await client.AddCorpusAsync(ECHO_ENGINE1_ID, TestCorpusConfig); var ex = Assert.ThrowsAsync( () => client.GetAllPretranslationsAsync(ECHO_ENGINE1_ID, addedCorpus.Id) ); - Assert.That(ex!.StatusCode, Is.EqualTo(409)); + Assert.That(ex?.StatusCode, Is.EqualTo(409)); } [Test] public async Task GetAllPretranslationsAsync_TextIdExists() { - ITranslationEnginesClient client = _env!.CreateClient(); + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(); TranslationCorpus addedCorpus = await client.AddCorpusAsync(ECHO_ENGINE1_ID, TestCorpusConfig); await _env.Engines.UpdateAsync(ECHO_ENGINE1_ID, u => u.Set(e => e.ModelRevision, 1)); @@ -799,7 +847,7 @@ public async Task GetAllPretranslationsAsync_TextIdExists() CorpusRef = addedCorpus.Id, TextId = "all", EngineRef = ECHO_ENGINE1_ID, - Refs = new List { "ref1", "ref2" }, + Refs = ["ref1", "ref2"], Translation = "translation", ModelRevision = 1 }; @@ -816,7 +864,7 @@ public async Task GetAllPretranslationsAsync_TextIdExists() [Test] public async Task GetAllPretranslationsAsync_TextIdDoesNotExist() { - ITranslationEnginesClient client = _env!.CreateClient(); + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(); TranslationCorpus addedCorpus = await client.AddCorpusAsync(ECHO_ENGINE1_ID, TestCorpusConfig); await _env.Engines.UpdateAsync(ECHO_ENGINE1_ID, u => u.Set(e => e.ModelRevision, 1)); @@ -825,7 +873,7 @@ public async Task GetAllPretranslationsAsync_TextIdDoesNotExist() CorpusRef = addedCorpus.Id, TextId = "all", EngineRef = ECHO_ENGINE1_ID, - Refs = new List { "ref1", "ref2" }, + Refs = ["ref1", "ref2"], Translation = "translation", ModelRevision = 1 }; @@ -850,7 +898,7 @@ public async Task GetAllBuildsForEngineByIdAsync( bool addBuild = true ) { - ITranslationEnginesClient client = _env!.CreateClient(scope); + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); Build? build = null; if (addBuild) { @@ -865,8 +913,8 @@ public async Task GetAllBuildsForEngineByIdAsync( Assert.Multiple(() => { Assert.That(results.First().Revision, Is.EqualTo(1)); - Assert.That(results.First().Id, Is.EqualTo(build!.Id)); - Assert.That(results.First().State, Is.EqualTo(Client.JobState.Pending)); + Assert.That(results.First().Id, Is.EqualTo(build?.Id)); + Assert.That(results.First().State, Is.EqualTo(JobState.Pending)); }); break; case 403: @@ -895,42 +943,48 @@ public async Task GetBuildByIdForEngineByIdAsync( bool addBuild = true ) { - ITranslationEnginesClient client = _env!.CreateClient(scope); + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); Build? build = null; if (addBuild) { build = new Build { EngineRef = engineId }; await _env.Builds.InsertAsync(build); } - ServalApiException ex; + switch (expectedStatusCode) { case 200: + { Assert.That(build, Is.Not.Null); - TranslationBuild result = await client.GetBuildAsync(engineId, build!.Id); + TranslationBuild result = await client.GetBuildAsync(engineId, build.Id); Assert.Multiple(() => { Assert.That(result.Revision, Is.EqualTo(1)); Assert.That(result.Id, Is.EqualTo(build.Id)); - Assert.That(result.State, Is.EqualTo(Client.JobState.Pending)); + Assert.That(result.State, Is.EqualTo(JobState.Pending)); }); break; + } case 403: case 404: - ex = Assert.ThrowsAsync(async () => + { + var ex = Assert.ThrowsAsync(async () => { await client.GetBuildAsync(engineId, "bbbbbbbbbbbbbbbbbbbbbbbb"); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; + } case 408: + { Assert.That(build, Is.Not.Null); - ex = Assert.ThrowsAsync(async () => + var ex = Assert.ThrowsAsync(async () => { await client.GetBuildAsync(engineId, build!.Id, 3); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; + } default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); break; @@ -956,7 +1010,7 @@ public async Task GetBuildByIdForEngineByIdAsync( [TestCase(new[] { Scopes.ReadFiles }, 403, ECHO_ENGINE1_ID)] //Arbitrary unrelated privilege public async Task StartBuildForEngineByIdAsync(IEnumerable scope, int expectedStatusCode, string engineId) { - ITranslationEnginesClient client = _env!.CreateClient(scope); + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); PretranslateCorpusConfig ptcc; TrainingCorpusConfig tcc; TranslationBuildConfig tbc; @@ -1006,7 +1060,7 @@ public async Task StartBuildForEngineByIdAsync(IEnumerable scope, int ex { await client.StartBuildAsync(engineId, tbc); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); @@ -1027,7 +1081,7 @@ public async Task GetCurrentBuildForEngineByIdAsync( bool addBuild = true ) { - ITranslationEnginesClient client = _env!.CreateClient(scope); + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); Build? build = null; if (addBuild) { @@ -1035,31 +1089,35 @@ public async Task GetCurrentBuildForEngineByIdAsync( await _env.Builds.InsertAsync(build); } - ServalApiException ex; switch (expectedStatusCode) { case 200: + { Assert.That(build, Is.Not.Null); TranslationBuild result = await client.GetCurrentBuildAsync(engineId); - Assert.That(result.Id, Is.EqualTo(build!.Id)); + Assert.That(result.Id, Is.EqualTo(build.Id)); break; + } case 204: case 403: case 404: - ex = Assert.ThrowsAsync(async () => + { + var ex = Assert.ThrowsAsync(async () => { await client.GetCurrentBuildAsync(engineId); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; + } case 408: - ex = Assert.ThrowsAsync(async () => + { + var ex = Assert.ThrowsAsync(async () => { await client.GetCurrentBuildAsync(engineId, minRevision: 3); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); - + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; + } default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); break; @@ -1069,7 +1127,7 @@ public async Task GetCurrentBuildForEngineByIdAsync( [Test] [TestCase(new[] { Scopes.UpdateTranslationEngines }, 200, ECHO_ENGINE1_ID)] [TestCase(new[] { Scopes.UpdateTranslationEngines }, 404, DOES_NOT_EXIST_ENGINE_ID, false)] - [TestCase(new[] { Scopes.UpdateTranslationEngines }, 404, ECHO_ENGINE1_ID, false)] + [TestCase(new[] { Scopes.UpdateTranslationEngines }, 204, ECHO_ENGINE1_ID, false)] [TestCase(new[] { Scopes.ReadFiles }, 403, ECHO_ENGINE1_ID, false)] //Arbitrary unrelated privilege public async Task CancelCurrentBuildForEngineByIdAsync( IEnumerable scope, @@ -1078,17 +1136,19 @@ public async Task CancelCurrentBuildForEngineByIdAsync( bool addBuild = true ) { - ITranslationEnginesClient client = _env!.CreateClient(scope); - Build? build = null; - if (addBuild) + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(scope); + if (!addBuild) { - build = new Build { EngineRef = engineId }; + var build = new Build { EngineRef = engineId }; await _env.Builds.InsertAsync(build); + _env.NmtClient.CancelBuildAsync(Arg.Any(), null, null, Arg.Any()) + .Returns(CreateAsyncUnaryCall(StatusCode.Aborted)); } switch (expectedStatusCode) { case 200: + case 204: await client.CancelBuildAsync(engineId); break; case 403: @@ -1097,7 +1157,7 @@ public async Task CancelCurrentBuildForEngineByIdAsync( { await client.CancelBuildAsync(engineId); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); @@ -1108,7 +1168,7 @@ public async Task CancelCurrentBuildForEngineByIdAsync( [Test] public async Task TryToQueueMultipleBuildsPerSingleUser() { - ITranslationEnginesClient client = _env!.CreateClient(); + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(); var engineId = NMT_ENGINE1_ID; var expectedStatusCode = 409; TranslationCorpus addedCorpus = await client.AddCorpusAsync(engineId, TestCorpusConfigNonEcho); @@ -1119,6 +1179,8 @@ public async Task TryToQueueMultipleBuildsPerSingleUser() }; var tbc = new TranslationBuildConfig { Pretranslate = new List { ptcc } }; TranslationBuild build = await client.StartBuildAsync(engineId, tbc); + _env.NmtClient.StartBuildAsync(Arg.Any(), null, null, Arg.Any()) + .Returns(CreateAsyncUnaryCall(StatusCode.Aborted)); var ex = Assert.ThrowsAsync(async () => { build = await client.StartBuildAsync(engineId, tbc); @@ -1128,15 +1190,50 @@ public async Task TryToQueueMultipleBuildsPerSingleUser() } [Test] - public void AddCorpusWithSameSourceAndTargetLangs() + public async Task GetPretranslatedUsfmAsync_BookExists() { - ITranslationEnginesClient client = _env!.CreateClient(); - var ex = Assert.ThrowsAsync(async () => + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(); + TranslationCorpus addedCorpus = await client.AddCorpusAsync(ECHO_ENGINE1_ID, TestCorpusConfigScripture); + + await _env.Engines.UpdateAsync(ECHO_ENGINE1_ID, u => u.Set(e => e.ModelRevision, 1)); + var pret = new Translation.Models.Pretranslation { - await client.AddCorpusAsync(NMT_ENGINE1_ID, TestCorpusConfig); - }); - Assert.That(ex, Is.Not.Null); - Assert.That(ex.StatusCode, Is.EqualTo(422)); + CorpusRef = addedCorpus.Id, + TextId = "MAT", + EngineRef = ECHO_ENGINE1_ID, + Refs = ["MAT 1:1"], + Translation = "translation", + ModelRevision = 1 + }; + await _env.Pretranslations.InsertAsync(pret); + + string usfm = await client.GetPretranslatedUsfmAsync(ECHO_ENGINE1_ID, addedCorpus.Id, "MAT"); + Assert.That( + usfm.Replace("\r\n", "\n"), + Is.EqualTo( + @"\id MAT - TRG +\h +\c 1 +\p +\v 1 translation +\v 2 +".Replace("\r\n", "\n") + ) + ); + } + + [Test] + public async Task GetPretranslatedUsfmAsync_BookDoesNotExist() + { + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(); + TranslationCorpus addedCorpus = await client.AddCorpusAsync(ECHO_ENGINE1_ID, TestCorpusConfigScripture); + + await _env.Engines.UpdateAsync(ECHO_ENGINE1_ID, u => u.Set(e => e.ModelRevision, 1)); + + var ex = Assert.ThrowsAsync( + () => client.GetPretranslatedUsfmAsync(ECHO_ENGINE1_ID, addedCorpus.Id, "MRK") + ); + Assert.That(ex?.StatusCode, Is.EqualTo(204)); } [Test] @@ -1144,7 +1241,7 @@ public void AddCorpusWithSameSourceAndTargetLangs() [TestCase("Echo")] public async Task GetQueueAsync(string engineType) { - TranslationClient client = _env!.CreateTranslationClient(); + TranslationEngineTypesClient client = _env.CreateTranslationEngineTypesClient(); Client.Queue queue = await client.GetQueueAsync(engineType); Assert.That(queue.Size, Is.EqualTo(0)); } @@ -1152,7 +1249,7 @@ public async Task GetQueueAsync(string engineType) [Test] public void GetQueueAsync_NotAuthorized() { - TranslationClient client = _env!.CreateTranslationClient([Scopes.ReadFiles]); + TranslationEngineTypesClient client = _env.CreateTranslationEngineTypesClient([Scopes.ReadFiles]); ServalApiException? ex = Assert.ThrowsAsync(async () => { Client.Queue queue = await client.GetQueueAsync("Echo"); @@ -1164,7 +1261,7 @@ public void GetQueueAsync_NotAuthorized() [Test] public async Task GetLanguageInfoAsync() { - TranslationClient client = _env!.CreateTranslationClient(); + TranslationEngineTypesClient client = _env.CreateTranslationEngineTypesClient(); Client.LanguageInfo languageInfo = await client.GetLanguageInfoAsync("Nmt", "Alphabet"); Assert.Multiple(() => { @@ -1176,7 +1273,7 @@ public async Task GetLanguageInfoAsync() [Test] public void GetLanguageInfo_Error() { - TranslationClient client = _env!.CreateTranslationClient([Scopes.ReadFiles]); + TranslationEngineTypesClient client = _env.CreateTranslationEngineTypesClient([Scopes.ReadFiles]); ServalApiException? ex = Assert.ThrowsAsync(async () => { Client.LanguageInfo languageInfo = await client.GetLanguageInfoAsync("Nmt", "abc"); @@ -1188,7 +1285,7 @@ public void GetLanguageInfo_Error() [TearDown] public void TearDown() { - _env!.Dispose(); + _env.Dispose(); } private static AsyncUnaryCall CreateAsyncUnaryCall(StatusCode statusCode) @@ -1217,7 +1314,7 @@ private static AsyncUnaryCall CreateAsyncUnaryCall(TRespon private class TestEnvironment : DisposableBase { private readonly IServiceScope _scope; - private readonly IMongoClient _mongoClient; + private readonly MongoClient _mongoClient; public TestEnvironment() { @@ -1383,7 +1480,7 @@ public TestEnvironment() .Returns(CreateAsyncUnaryCall(StatusCode.Unimplemented)); } - ServalWebApplicationFactory Factory { get; } + public ServalWebApplicationFactory Factory { get; } public IRepository Engines { get; } public IRepository DataFiles { get; } public IRepository Pretranslations { get; } @@ -1391,7 +1488,7 @@ public TestEnvironment() public TranslationEngineApi.TranslationEngineApiClient EchoClient { get; } public TranslationEngineApi.TranslationEngineApiClient NmtClient { get; } - public TranslationEnginesClient CreateClient(IEnumerable? scope = null) + public TranslationEnginesClient CreateTranslationEnginesClient(IEnumerable? scope = null) { scope ??= new[] { @@ -1413,6 +1510,7 @@ public TranslationEnginesClient CreateClient(IEnumerable? scope = null) .CreateClient("Nmt") .Returns(NmtClient); services.AddSingleton(grpcClientFactory); + services.AddTransient(CreateFileSystem); }); }) .CreateClient(); @@ -1420,7 +1518,7 @@ public TranslationEnginesClient CreateClient(IEnumerable? scope = null) return new TranslationEnginesClient(httpClient); } - public TranslationClient CreateTranslationClient(IEnumerable? scope = null) + public TranslationEngineTypesClient CreateTranslationEngineTypesClient(IEnumerable? scope = null) { scope ??= new[] { @@ -1454,7 +1552,7 @@ public TranslationClient CreateTranslationClient(IEnumerable? scope = nu CreateAsyncUnaryCall(new GetLanguageInfoResponse() { InternalCode = "abc_123", IsNative = true }) ); httpClient.DefaultRequestHeaders.Add("Scope", string.Join(" ", scope)); - return new TranslationClient(httpClient); + return new TranslationEngineTypesClient(httpClient); } public void ResetDatabases() @@ -1463,6 +1561,71 @@ public void ResetDatabases() _mongoClient.DropDatabase("serval_test_jobs"); } + private static IFileSystem CreateFileSystem(IServiceProvider sp) + { + var fileSystem = Substitute.For(); + var dataFileOptions = sp.GetRequiredService>(); + fileSystem + .OpenZipFile(GetFilePath(dataFileOptions, FILE3_FILENAME)) + .Returns(ci => + { + IZipContainer source = CreateZipContainer("SRC"); + source.EntryExists("MATSRC.SFM").Returns(true); + string usfm = + $@"\id MAT - SRC +\h Matthew +\c 1 +\p +\v 1 Chapter one, verse one. +\v 2 Chapter one, verse two. +"; + source.OpenEntry("MATSRC.SFM").Returns(ci => new MemoryStream(Encoding.UTF8.GetBytes(usfm))); + return source; + }); + fileSystem + .OpenZipFile(GetFilePath(dataFileOptions, FILE4_FILENAME)) + .Returns(ci => + { + IZipContainer target = CreateZipContainer("TRG"); + target.EntryExists("MATTRG.SFM").Returns(false); + return target; + }); + return fileSystem; + } + + private static IZipContainer CreateZipContainer(string name) + { + var container = Substitute.For(); + container.EntryExists("Settings.xml").Returns(true); + XElement settingsXml = + new( + "ScriptureText", + new XElement("StyleSheet", "usfm.sty"), + new XElement("Name", name), + new XElement("FullName", name), + new XElement("Encoding", "65001"), + new XElement( + "Naming", + new XAttribute("PrePart", ""), + new XAttribute("PostPart", $"{name}.SFM"), + new XAttribute("BookNameForm", "MAT") + ), + new XElement("BiblicalTermsListSetting", "Major::BiblicalTerms.xml") + ); + container + .OpenEntry("Settings.xml") + .Returns(new MemoryStream(Encoding.UTF8.GetBytes(settingsXml.ToString()))); + container.EntryExists("custom.vrs").Returns(false); + container.EntryExists("usfm.sty").Returns(false); + container.EntryExists("custom.sty").Returns(false); + return container; + } + + private static string GetFilePath(IOptionsMonitor dataFileOptions, string fileName) + { + return Path.Combine(dataFileOptions.CurrentValue.FilesDirectory, fileName); + } + protected override void DisposeManagedResources() { _scope.Dispose(); diff --git a/tests/Serval.ApiServer.IntegrationTests/Usings.cs b/tests/Serval.ApiServer.IntegrationTests/Usings.cs index ad23be4f..fa43f490 100644 --- a/tests/Serval.ApiServer.IntegrationTests/Usings.cs +++ b/tests/Serval.ApiServer.IntegrationTests/Usings.cs @@ -1,6 +1,7 @@ global using System.Security.Claims; global using System.Text; global using System.Text.Encodings.Web; +global using System.Xml.Linq; global using Grpc.Core; global using Grpc.Net.ClientFactory; global using Hangfire; @@ -16,7 +17,9 @@ global using NSubstitute; global using NUnit.Framework; global using Serval.Client; +global using Serval.Shared.Configuration; global using Serval.Shared.Controllers; +global using Serval.Shared.Services; global using Serval.Translation.Models; global using SIL.DataAccess; global using SIL.ObjectModel; diff --git a/tests/Serval.ApiServer.IntegrationTests/WebhooksTests.cs b/tests/Serval.ApiServer.IntegrationTests/WebhooksTests.cs index 7ef693b0..6623f48b 100644 --- a/tests/Serval.ApiServer.IntegrationTests/WebhooksTests.cs +++ b/tests/Serval.ApiServer.IntegrationTests/WebhooksTests.cs @@ -7,7 +7,7 @@ public class WebhooksTests const string ID = "000000000000000000000000"; const string DOES_NOT_EXIST_ID = "000000000000000000000001"; - TestEnvironment? _env; + TestEnvironment _env; [SetUp] public async Task Setup() @@ -19,10 +19,7 @@ public async Task Setup() Owner = "client1", Url = "/a/url", Secret = "s3CreT#", - Events = new List - { - Webhooks.Contracts.WebhookEvent.TranslationBuildStarted - } + Events = [Webhooks.Contracts.WebhookEvent.TranslationBuildStarted] }; await _env.Webhooks.InsertAsync(webhook); } @@ -32,7 +29,7 @@ public async Task Setup() [TestCase(new string[] { Scopes.ReadFiles }, 403)] //Arbitrary unrelated privilege public async Task GetAllWebhooksAsync(IEnumerable? scope, int expectedStatusCode) { - WebhooksClient client = _env!.CreateClient(scope); + WebhooksClient client = _env.CreateClient(scope); switch (expectedStatusCode) { case 200: @@ -40,12 +37,11 @@ public async Task GetAllWebhooksAsync(IEnumerable? scope, int expectedSt Assert.That(result.Id, Is.EqualTo(ID)); break; case 403: - ServalApiException? ex = Assert.ThrowsAsync(async () => + var ex = Assert.ThrowsAsync(async () => { await client.GetAllAsync(); }); - Assert.That(ex, Is.Not.Null); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); @@ -59,7 +55,7 @@ public async Task GetAllWebhooksAsync(IEnumerable? scope, int expectedSt [TestCase(new string[] { Scopes.ReadFiles }, 403, ID)] //Arbitrary unrelated privilege public async Task GetWebhookByIdAsync(IEnumerable? scope, int expectedStatusCode, string webhookId) { - WebhooksClient client = _env!.CreateClient(scope); + WebhooksClient client = _env.CreateClient(scope); switch (expectedStatusCode) { case 200: @@ -72,7 +68,7 @@ public async Task GetWebhookByIdAsync(IEnumerable? scope, int expectedSt { await client.GetAsync(webhookId); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); @@ -86,26 +82,29 @@ public async Task GetWebhookByIdAsync(IEnumerable? scope, int expectedSt [TestCase(new string[] { Scopes.ReadFiles }, 403, ID)] //Arbitrary unrelated privilege public async Task DeleteWebhookByIdAsync(IEnumerable? scope, int expectedStatusCode, string webhookId) { - WebhooksClient client = _env!.CreateClient(scope); - ServalApiException ex; + WebhooksClient client = _env.CreateClient(scope); switch (expectedStatusCode) { case 200: + { await client.DeleteAsync(webhookId); - ex = Assert.ThrowsAsync(async () => + var ex = Assert.ThrowsAsync(async () => { await client.GetAsync(webhookId); }); - Assert.That(ex!.StatusCode, Is.EqualTo(404)); + Assert.That(ex?.StatusCode, Is.EqualTo(404)); break; + } case 403: case 404: - ex = Assert.ThrowsAsync(async () => + { + var ex = Assert.ThrowsAsync(async () => { await client.DeleteAsync(webhookId); }); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; + } default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); break; @@ -115,9 +114,9 @@ public async Task DeleteWebhookByIdAsync(IEnumerable? scope, int expecte [Test] [TestCase(null, 201)] [TestCase(new string[] { Scopes.ReadFiles }, 403)] //Arbitrary unrelated privilege - public async Task CreateWebhookAsync(IEnumerable scope, int expectedStatusCode) + public async Task CreateWebhookAsync(IEnumerable? scope, int expectedStatusCode) { - WebhooksClient client = _env!.CreateClient(scope); + WebhooksClient client = _env.CreateClient(scope); switch (expectedStatusCode) { case 201: @@ -134,7 +133,7 @@ public async Task CreateWebhookAsync(IEnumerable scope, int expectedStat break; case 403: - ServalApiException? ex = Assert.ThrowsAsync(async () => + var ex = Assert.ThrowsAsync(async () => { Webhook result = await client.CreateAsync( new WebhookConfig @@ -145,8 +144,7 @@ public async Task CreateWebhookAsync(IEnumerable scope, int expectedStat } ); }); - Assert.That(ex, Is.Not.Null); - Assert.That(ex!.StatusCode, Is.EqualTo(expectedStatusCode)); + Assert.That(ex?.StatusCode, Is.EqualTo(expectedStatusCode)); break; default: Assert.Fail("Unanticipated expectedStatusCode. Check test case for typo."); @@ -157,7 +155,7 @@ public async Task CreateWebhookAsync(IEnumerable scope, int expectedStat [TearDown] public void TearDown() { - _env!.Dispose(); + _env.Dispose(); } private class TestEnvironment : DisposableBase @@ -167,8 +165,7 @@ private class TestEnvironment : DisposableBase public TestEnvironment() { - var clientSettings = new MongoClientSettings(); - clientSettings.LinqProvider = LinqProvider.V2; + MongoClientSettings clientSettings = new() { LinqProvider = LinqProvider.V2 }; _mongoClient = new MongoClient(clientSettings); ResetDatabases(); @@ -182,10 +179,7 @@ public TestEnvironment() public WebhooksClient CreateClient(IEnumerable? scope) { - if (scope is null) - { - scope = new[] { Scopes.ReadHooks, Scopes.CreateHooks, Scopes.DeleteHooks }; - } + scope ??= new[] { Scopes.ReadHooks, Scopes.CreateHooks, Scopes.DeleteHooks }; var httpClient = Factory.WithWebHostBuilder(_ => { }).CreateClient(); httpClient.DefaultRequestHeaders.Add("Scope", string.Join(" ", scope)); diff --git a/tests/Serval.DataFiles.Tests/Serval.DataFiles.Tests.csproj b/tests/Serval.DataFiles.Tests/Serval.DataFiles.Tests.csproj index 559cd406..d1051479 100644 --- a/tests/Serval.DataFiles.Tests/Serval.DataFiles.Tests.csproj +++ b/tests/Serval.DataFiles.Tests/Serval.DataFiles.Tests.csproj @@ -15,15 +15,15 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/Serval.DataFiles.Tests/Services/DataFileServiceTests.cs b/tests/Serval.DataFiles.Tests/Services/DataFileServiceTests.cs index 9028a79a..5ba2900f 100644 --- a/tests/Serval.DataFiles.Tests/Services/DataFileServiceTests.cs +++ b/tests/Serval.DataFiles.Tests/Services/DataFileServiceTests.cs @@ -1,6 +1,4 @@ -using Serval.Shared.Contracts; - -namespace Serval.DataFiles.Services; +namespace Serval.DataFiles.Services; [TestFixture] public class DataFileServiceTests @@ -59,20 +57,18 @@ public async Task DownloadAsync_Exists() byte[] content = Encoding.UTF8.GetBytes("This is a file."); using var fileStream = new MemoryStream(content); env.FileSystem.OpenRead(Arg.Any()).Returns(fileStream); - Stream? downloadedStream = await env.Service.ReadAsync(DATA_FILE_ID); - Assert.That(downloadedStream, Is.Not.Null); + Stream downloadedStream = await env.Service.ReadAsync(DATA_FILE_ID); Assert.That(new StreamReader(downloadedStream).ReadToEnd(), Is.EqualTo(content)); } [Test] - public async Task DownloadAsync_DoesNotExists() + public void DownloadAsync_DoesNotExists() { var env = new TestEnvironment(); byte[] content = Encoding.UTF8.GetBytes("This is a file."); using var fileStream = new MemoryStream(content); env.FileSystem.OpenRead(Arg.Any()).Returns(fileStream); - Stream? downloadedStream = await env.Service.ReadAsync(DATA_FILE_ID); - Assert.That(downloadedStream, Is.Null); + Assert.ThrowsAsync(() => env.Service.ReadAsync(DATA_FILE_ID)); } [Test] @@ -90,29 +86,26 @@ public async Task UpdateAsync_Exists() using var fileStream = new MemoryStream(); env.FileSystem.OpenWrite(Arg.Any()).Returns(fileStream); string content = "This is a file."; - DataFile? dataFile; + DataFile dataFile; using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(content))) dataFile = await env.Service.UpdateAsync(DATA_FILE_ID, stream); - Assert.That(dataFile, Is.Not.Null); - Assert.That(dataFile!.Revision, Is.EqualTo(2)); + Assert.That(dataFile.Revision, Is.EqualTo(2)); Assert.That(Encoding.UTF8.GetString(fileStream.ToArray()), Is.EqualTo(content)); DeletedFile deletedFile = env.DeletedFiles.Entities.Single(); Assert.That(deletedFile.Filename, Is.EqualTo("file1.txt")); } [Test] - public async Task UpdateAsync_DoesNotExist() + public void UpdateAsync_DoesNotExist() { var env = new TestEnvironment(); using var fileStream = new MemoryStream(); env.FileSystem.OpenWrite(Arg.Any()).Returns(fileStream); string content = "This is a file."; - DataFile? dataFile; using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(content))) - dataFile = await env.Service.UpdateAsync(DATA_FILE_ID, stream); + Assert.ThrowsAsync(() => env.Service.UpdateAsync(DATA_FILE_ID, stream)); - Assert.That(dataFile, Is.Null); env.FileSystem.Received().DeleteFile(Arg.Any()); } @@ -128,9 +121,8 @@ public async Task DeleteAsync_Exists() Filename = "file1.txt" } ); - bool deleted = await env.Service.DeleteAsync(DATA_FILE_ID); + await env.Service.DeleteAsync(DATA_FILE_ID); - Assert.That(deleted, Is.True); Assert.That(env.DataFiles.Contains(DATA_FILE_ID), Is.False); DeletedFile deletedFile = env.DeletedFiles.Entities.Single(); Assert.That(deletedFile.Filename, Is.EqualTo("file1.txt")); @@ -138,12 +130,10 @@ public async Task DeleteAsync_Exists() } [Test] - public async Task DeleteAsync_DoesNotExist() + public void DeleteAsync_DoesNotExist() { var env = new TestEnvironment(); - bool deleted = await env.Service.DeleteAsync(DATA_FILE_ID); - - Assert.That(deleted, Is.False); + Assert.ThrowsAsync(() => env.Service.DeleteAsync(DATA_FILE_ID)); } private class TestEnvironment diff --git a/tests/Serval.DataFiles.Tests/Usings.cs b/tests/Serval.DataFiles.Tests/Usings.cs index 83186c3a..4474dbbf 100644 --- a/tests/Serval.DataFiles.Tests/Usings.cs +++ b/tests/Serval.DataFiles.Tests/Usings.cs @@ -7,5 +7,7 @@ global using NUnit.Framework; global using Serval.DataFiles.Models; global using Serval.Shared.Configuration; +global using Serval.Shared.Contracts; global using Serval.Shared.Services; +global using Serval.Shared.Utils; global using SIL.DataAccess; diff --git a/tests/Serval.E2ETests/MissingServicesTests.cs b/tests/Serval.E2ETests/MissingServicesTests.cs index 38b657d3..9a44ca53 100644 --- a/tests/Serval.E2ETests/MissingServicesTests.cs +++ b/tests/Serval.E2ETests/MissingServicesTests.cs @@ -1,105 +1,102 @@ -namespace Serval.E2ETests +namespace Serval.E2ETests; + +[TestFixture] +[Category("E2EMissingServices")] +public class MissingServicesTests { - [TestFixture] - [Category("E2EMissingServices")] - public class MissingServicesTests - { - private ServalClientHelper? _helperClient; + private ServalClientHelper _helperClient; - [SetUp] - public void Setup() - { - _helperClient = new ServalClientHelper("https://serval-api.org/", ignoreSSLErrors: true); - } + [SetUp] + public async Task Setup() + { + _helperClient = new ServalClientHelper("https://serval-api.org/", ignoreSSLErrors: true); + await _helperClient.InitAsync(); + } - [Test] - [Category("MongoWorking")] - public void UseMongoAndAuth0Async() + [Test] + [Category("MongoWorking")] + public void UseMongoAndAuth0Async() + { + Assert.DoesNotThrowAsync(async () => { - Assert.DoesNotThrowAsync(async () => - { - await _helperClient!.dataFilesClient.GetAllAsync(); - }); - } + await _helperClient.DataFilesClient.GetAllAsync(); + }); + } - [Test] - [Category("EngineServerWorking")] - public void UseEngineServerAsync() + [Test] + [Category("EngineServerWorking")] + public void UseEngineServerAsync() + { + Assert.DoesNotThrowAsync(async () => { - Assert.DoesNotThrowAsync(async () => - { - string engineId = await _helperClient!.CreateNewEngine("SmtTransfer", "es", "en", "SMT3"); - var books = new string[] { "1JN.txt", "2JN.txt", "3JN.txt" }; - await _helperClient.AddTextCorpusToEngine(engineId, books, "es", "en", false); - await _helperClient.BuildEngine(engineId); - }); - } + string engineId = await _helperClient.CreateNewEngineAsync("SmtTransfer", "es", "en", "SMT3"); + string[] books = ["1JN.txt", "2JN.txt", "3JN.txt"]; + await _helperClient.AddTextCorpusToEngineAsync(engineId, books, "es", "en", false); + await _helperClient.BuildEngineAsync(engineId); + }); + } - [Test] - [Category("ClearMLNotWorking")] - public void UseMissingClearMLAsync() + [Test] + [Category("ClearMLNotWorking")] + public void UseMissingClearMLAsync() + { + Assert.ThrowsAsync(async () => { - Assert.ThrowsAsync(async () => - { - string engineId = await _helperClient!.CreateNewEngine("Nmt", "es", "en", "NMT1"); - var books = new string[] { "MAT.txt", "1JN.txt", "2JN.txt" }; - await _helperClient.AddTextCorpusToEngine(engineId, books, "es", "en", false); - var cId = await _helperClient.AddTextCorpusToEngine( - engineId, - new string[] { "3JN.txt" }, - "es", - "en", - true - ); - await _helperClient.BuildEngine(engineId); - IList lTrans = await _helperClient.translationEnginesClient.GetAllPretranslationsAsync( - engineId, - cId - ); - }); - } + string engineId = await _helperClient.CreateNewEngineAsync("Nmt", "es", "en", "NMT1"); + string[] books = ["MAT.txt", "1JN.txt", "2JN.txt"]; + await _helperClient.AddTextCorpusToEngineAsync(engineId, books, "es", "en", false); + var cId = await _helperClient.AddTextCorpusToEngineAsync(engineId, ["3JN.txt"], "es", "en", true); + await _helperClient.BuildEngineAsync(engineId); + IList lTrans = await _helperClient.TranslationEnginesClient.GetAllPretranslationsAsync( + engineId, + cId + ); + }); + } - [Test] - [Category("AWSNotWorking")] - public async Task UseMissingAWSAsync() - { - string engineId = await _helperClient!.CreateNewEngine("Nmt", "es", "en", "NMT1"); - var books = new string[] { "MAT.txt", "1JN.txt", "2JN.txt" }; - await _helperClient.AddTextCorpusToEngine(engineId, books, "es", "en", false); - await _helperClient.AddTextCorpusToEngine(engineId, new string[] { "3JN.txt" }, "es", "en", true); - await _helperClient.BuildEngine(engineId); - IList? builds = await _helperClient.translationEnginesClient.GetAllBuildsAsync(engineId); - Assert.That(builds.First().State, Is.EqualTo(JobState.Faulted)); - } + [Test] + [Category("AWSNotWorking")] + public async Task UseMissingAWSAsync() + { + string engineId = await _helperClient.CreateNewEngineAsync("Nmt", "es", "en", "NMT1"); + string[] books = ["MAT.txt", "1JN.txt", "2JN.txt"]; + await _helperClient.AddTextCorpusToEngineAsync(engineId, books, "es", "en", false); + await _helperClient.AddTextCorpusToEngineAsync(engineId, ["3JN.txt"], "es", "en", true); + await _helperClient.BuildEngineAsync(engineId); + IList builds = await _helperClient.TranslationEnginesClient.GetAllBuildsAsync(engineId); + Assert.That(builds.First().State, Is.EqualTo(JobState.Faulted)); + } - [Test] - [Category("MongoNotWorking")] - public void UseMissingMongoAsync() + [Test] + [Category("MongoNotWorking")] + public void UseMissingMongoAsync() + { + ServalApiException? ex = Assert.ThrowsAsync(async () => { - ServalApiException? ex = Assert.ThrowsAsync(async () => - { - await _helperClient!.dataFilesClient.GetAllAsync(); - }); - Assert.NotNull(ex); - Assert.That(ex!.StatusCode, Is.EqualTo(503)); - } + await _helperClient.DataFilesClient.GetAllAsync(); + }); + Assert.That(ex, Is.Not.Null); + Assert.That(ex.StatusCode, Is.EqualTo(503)); + } - [Test] - [Category("EngineServerNotWorking")] - public void UseMissingEngineServerAsync() + [Test] + [Category("EngineServerNotWorking")] + public void UseMissingEngineServerAsync() + { + ServalApiException? ex = Assert.ThrowsAsync(async () => { - ServalApiException? ex = Assert.ThrowsAsync(async () => - { - string engineId = await _helperClient!.CreateNewEngine("SmtTransfer", "es", "en", "SMT3"); - var books = new string[] { "1JN.txt", "2JN.txt", "3JN.txt" }; - await _helperClient.AddTextCorpusToEngine(engineId, books, "es", "en", false); - await _helperClient.BuildEngine(engineId); - }); - Assert.NotNull(ex); - Assert.That(ex!.StatusCode, Is.EqualTo(503)); - } + string engineId = await _helperClient.CreateNewEngineAsync("SmtTransfer", "es", "en", "SMT3"); + string[] books = ["1JN.txt", "2JN.txt", "3JN.txt"]; + await _helperClient.AddTextCorpusToEngineAsync(engineId, books, "es", "en", false); + await _helperClient.BuildEngineAsync(engineId); + }); + Assert.That(ex, Is.Not.Null); + Assert.That(ex.StatusCode, Is.EqualTo(503)); + } - [TearDown] - public void TearDown() { } + [TearDown] + public async Task TearDown() + { + await _helperClient.DisposeAsync(); } } diff --git a/tests/Serval.E2ETests/Serval.E2ETests.csproj b/tests/Serval.E2ETests/Serval.E2ETests.csproj index 7e5725ec..b58c0a5e 100644 --- a/tests/Serval.E2ETests/Serval.E2ETests.csproj +++ b/tests/Serval.E2ETests/Serval.E2ETests.csproj @@ -7,9 +7,13 @@ - - - + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + diff --git a/tests/Serval.E2ETests/ServalApiSlowTests.cs b/tests/Serval.E2ETests/ServalApiSlowTests.cs index e12500a7..0324ab0b 100644 --- a/tests/Serval.E2ETests/ServalApiSlowTests.cs +++ b/tests/Serval.E2ETests/ServalApiSlowTests.cs @@ -5,22 +5,28 @@ namespace Serval.E2ETests; [Category("slow")] public class ServalApiSlowTests { - private ServalClientHelper? _helperClient; + private ServalClientHelper _helperClient; [SetUp] - public void SetUp() + public async Task SetUp() { _helperClient = new ServalClientHelper("https://serval-api.org/", ignoreSSLErrors: true); + await _helperClient.InitAsync(); } [Test] public async Task GetSmtWholeBible() { - await _helperClient!.ClearEngines(); - string engineId = await _helperClient.CreateNewEngine("SmtTransfer", "es", "en", "SMT2"); - await _helperClient.AddTextCorpusToEngine(engineId, new string[] { "bible.txt" }, "es", "en", false); - await _helperClient.BuildEngine(engineId); - TranslationResult tResult = await _helperClient.translationEnginesClient.TranslateAsync(engineId, "Espíritu"); - Assert.AreEqual(tResult.Translation, "Spirit"); + string engineId = await _helperClient.CreateNewEngineAsync("SmtTransfer", "es", "en", "SMT2"); + await _helperClient.AddTextCorpusToEngineAsync(engineId, ["bible.txt"], "es", "en", false); + await _helperClient.BuildEngineAsync(engineId); + TranslationResult tResult = await _helperClient.TranslationEnginesClient.TranslateAsync(engineId, "Espíritu"); + Assert.That(tResult.Translation, Is.EqualTo("Spirit")); + } + + [TearDown] + public async Task TearDown() + { + await _helperClient.DisposeAsync(); } } diff --git a/tests/Serval.E2ETests/ServalApiTests.cs b/tests/Serval.E2ETests/ServalApiTests.cs index f02668c0..9f46afc5 100644 --- a/tests/Serval.E2ETests/ServalApiTests.cs +++ b/tests/Serval.E2ETests/ServalApiTests.cs @@ -4,70 +4,66 @@ namespace Serval.E2ETests; [Category("E2E")] public class ServalApiTests { - private ServalClientHelper? _helperClient; + private ServalClientHelper _helperClient; [SetUp] - public void SetUp() + public async Task SetUp() { _helperClient = new ServalClientHelper("https://serval-api.org/", ignoreSSLErrors: true); + await _helperClient.InitAsync(); } [Test] public async Task GetEchoSuggestion() { - await _helperClient!.ClearEngines(); - string engineId = await _helperClient.CreateNewEngine("Echo", "es", "es", "Echo1"); - var books = new string[] { "1JN.txt", "2JN.txt", "3JN.txt" }; - await _helperClient.AddTextCorpusToEngine(engineId, books, "es", "es", false); - await _helperClient.BuildEngine(engineId); - TranslationResult tResult = await _helperClient.translationEnginesClient.TranslateAsync(engineId, "Espíritu"); - Assert.AreEqual(tResult.Translation, "Espíritu"); + string engineId = await _helperClient.CreateNewEngineAsync("Echo", "es", "es", "Echo1"); + string[] books = ["1JN.txt", "2JN.txt", "3JN.txt"]; + await _helperClient.AddTextCorpusToEngineAsync(engineId, books, "es", "es", false); + await _helperClient.BuildEngineAsync(engineId); + TranslationResult tResult = await _helperClient.TranslationEnginesClient.TranslateAsync(engineId, "Espíritu"); + Assert.That(tResult.Translation, Is.EqualTo("Espíritu")); } [Test] public async Task GetEchoPretranslate() { - await _helperClient!.ClearEngines(); - string engineId = await _helperClient.CreateNewEngine("Echo", "es", "es", "Echo2"); - var books = new string[] { "1JN.txt", "2JN.txt" }; - await _helperClient.AddTextCorpusToEngine(engineId, books, "es", "es", false); - books = new string[] { "3JN.txt" }; - var corpusId = await _helperClient.AddTextCorpusToEngine(engineId, books, "es", "es", true); - await _helperClient.BuildEngine(engineId); - var corpora = _helperClient.translationEnginesClient.GetAllCorporaAsync(engineId); - var pretranslations = await _helperClient.translationEnginesClient.GetAllPretranslationsAsync( + string engineId = await _helperClient.CreateNewEngineAsync("Echo", "es", "es", "Echo2"); + string[] books = ["1JN.txt", "2JN.txt"]; + await _helperClient.AddTextCorpusToEngineAsync(engineId, books, "es", "es", false); + books = ["3JN.txt"]; + var corpusId = await _helperClient.AddTextCorpusToEngineAsync(engineId, books, "es", "es", true); + await _helperClient.BuildEngineAsync(engineId); + var pretranslations = await _helperClient.TranslationEnginesClient.GetAllPretranslationsAsync( engineId, corpusId ); - Assert.IsTrue(pretranslations.Count > 1); + Assert.That(pretranslations.Count, Is.GreaterThan(1)); } [Test] public async Task GetSmtTranslation() { - await _helperClient!.ClearEngines(); - string engineId = await _helperClient.CreateNewEngine("SmtTransfer", "es", "en", "SMT1"); - var books = new string[] { "1JN.txt", "2JN.txt", "3JN.txt" }; - await _helperClient.AddTextCorpusToEngine(engineId, books, "es", "en", false); - await _helperClient.BuildEngine(engineId); - TranslationResult tResult = await _helperClient.translationEnginesClient.TranslateAsync(engineId, "Espíritu"); - Assert.AreEqual(tResult.Translation, "spirit"); + string engineId = await _helperClient.CreateNewEngineAsync("SmtTransfer", "es", "en", "SMT1"); + string[] books = ["1JN.txt", "2JN.txt", "3JN.txt"]; + await _helperClient.AddTextCorpusToEngineAsync(engineId, books, "es", "en", false); + await _helperClient.BuildEngineAsync(engineId); + TranslationResult tResult = await _helperClient.TranslationEnginesClient.TranslateAsync(engineId, "Espíritu"); + Assert.That(tResult.Translation, Is.EqualTo("spirit")); } [Test] public async Task GetSmtAddSegment() { - await _helperClient!.ClearEngines(); - string engineId = await _helperClient.CreateNewEngine("smt-transfer", "es", "en", "SMT3"); - var books = new string[] { "1JN.txt", "2JN.txt", "3JN.txt" }; - await _helperClient.AddTextCorpusToEngine(engineId, books, "es", "en", false); - await _helperClient.BuildEngine(engineId); - TranslationResult tResult = await _helperClient.translationEnginesClient.TranslateAsync( + string engineId = await _helperClient.CreateNewEngineAsync("smt-transfer", "es", "en", "SMT3"); + string[] books = ["1JN.txt", "2JN.txt", "3JN.txt"]; + await _helperClient.AddTextCorpusToEngineAsync(engineId, books, "es", "en", false); + await _helperClient.BuildEngineAsync(engineId); + TranslationResult tResult = await _helperClient.TranslationEnginesClient.TranslateAsync( engineId, "ungidos espíritu" ); - Assert.AreEqual(tResult.Translation, "ungidos spirit"); - await _helperClient.translationEnginesClient.TrainSegmentAsync( + Assert.That(tResult.Translation, Is.EqualTo("ungidos spirit")); + await _helperClient.TranslationEnginesClient.TrainSegmentAsync( engineId, new SegmentPair { @@ -76,59 +72,56 @@ await _helperClient.translationEnginesClient.TrainSegmentAsync( SentenceStart = true } ); - TranslationResult tResult2 = await _helperClient.translationEnginesClient.TranslateAsync( + TranslationResult tResult2 = await _helperClient.TranslationEnginesClient.TranslateAsync( engineId, "ungidos espíritu" ); - Assert.AreEqual(tResult2.Translation, "unction spirit"); + Assert.That(tResult2.Translation, Is.EqualTo("unction spirit")); } [Test] public async Task GetSmtMoreCorpus() { - await _helperClient!.ClearEngines(); - string engineId = await _helperClient.CreateNewEngine("SmtTransfer", "es", "en", "SMT4"); - await _helperClient.AddTextCorpusToEngine(engineId, new string[] { "3JN.txt" }, "es", "en", false); - await _helperClient.BuildEngine(engineId); - TranslationResult tResult = await _helperClient.translationEnginesClient.TranslateAsync( + string engineId = await _helperClient.CreateNewEngineAsync("SmtTransfer", "es", "en", "SMT4"); + await _helperClient.AddTextCorpusToEngineAsync(engineId, ["3JN.txt"], "es", "en", false); + await _helperClient.BuildEngineAsync(engineId); + TranslationResult tResult = await _helperClient.TranslationEnginesClient.TranslateAsync( engineId, "verdad mundo" ); - Assert.AreEqual(tResult.Translation, "truth mundo"); - await _helperClient.AddTextCorpusToEngine(engineId, new string[] { "1JN.txt", "2JN.txt" }, "es", "en", false); - await _helperClient.BuildEngine(engineId); - TranslationResult tResult2 = await _helperClient.translationEnginesClient.TranslateAsync( + Assert.That(tResult.Translation, Is.EqualTo("truth mundo")); + await _helperClient.AddTextCorpusToEngineAsync(engineId, ["1JN.txt", "2JN.txt"], "es", "en", false); + await _helperClient.BuildEngineAsync(engineId); + TranslationResult tResult2 = await _helperClient.TranslationEnginesClient.TranslateAsync( engineId, "verdad mundo" ); - Assert.AreEqual(tResult2.Translation, "truth world"); + Assert.That(tResult2.Translation, Is.EqualTo("truth world")); } [Test] public async Task NmtBatch() { - await _helperClient!.ClearEngines(); - string engineId = await _helperClient.CreateNewEngine("Nmt", "es", "en", "NMT1"); - var books = new string[] { "MAT.txt", "1JN.txt", "2JN.txt" }; - var cId1 = await _helperClient.AddTextCorpusToEngine(engineId, books, "es", "en", false); + string engineId = await _helperClient.CreateNewEngineAsync("Nmt", "es", "en", "NMT1"); + string[] books = ["MAT.txt", "1JN.txt", "2JN.txt"]; + var cId1 = await _helperClient.AddTextCorpusToEngineAsync(engineId, books, "es", "en", false); _helperClient.TranslationBuildConfig.TrainOn = new List { - new TrainingCorpusConfig { CorpusId = cId1, TextIds = new string[] { "1JN.txt" } } + new() { CorpusId = cId1, TextIds = ["1JN.txt"] } }; - var cId2 = await _helperClient.AddTextCorpusToEngine(engineId, new string[] { "3JN.txt" }, "es", "en", true); - await _helperClient.BuildEngine(engineId); + var cId2 = await _helperClient.AddTextCorpusToEngineAsync(engineId, ["3JN.txt"], "es", "en", true); + await _helperClient.BuildEngineAsync(engineId); await Task.Delay(1000); - IList lTrans = await _helperClient.translationEnginesClient.GetAllPretranslationsAsync( + IList lTrans = await _helperClient.TranslationEnginesClient.GetAllPretranslationsAsync( engineId, cId2 ); - Assert.IsTrue(lTrans.Count == 14); + Assert.That(lTrans.Count, Is.EqualTo(14)); } [Test] public async Task NmtQueueMultiple() { - await _helperClient!.ClearEngines(); const int NUM_ENGINES = 9; const int NUM_WORKERS = 1; string[] engineIds = new string[NUM_ENGINES]; @@ -139,11 +132,11 @@ public async Task NmtQueueMultiple() Pretranslate = new List(), Options = "{\"max_steps\":10}" }; - engineIds[i] = await _helperClient.CreateNewEngine("Nmt", "es", "en", $"NMT1_{i}"); + engineIds[i] = await _helperClient.CreateNewEngineAsync("Nmt", "es", "en", $"NMT1_{i}"); string engineId = engineIds[i]; - var books = new string[] { "MAT.txt", "1JN.txt", "2JN.txt" }; - await _helperClient.AddTextCorpusToEngine(engineId, books, "es", "en", false); - await _helperClient.AddTextCorpusToEngine(engineId, new string[] { "3JN.txt" }, "es", "en", true); + string[] books = ["MAT.txt", "1JN.txt", "2JN.txt"]; + await _helperClient.AddTextCorpusToEngineAsync(engineId, books, "es", "en", false); + await _helperClient.AddTextCorpusToEngineAsync(engineId, ["3JN.txt"], "es", "en", true); await _helperClient.StartBuildAsync(engineId); //Ensure that tasks are enqueued roughly in order await Task.Delay(1_000); @@ -153,28 +146,32 @@ public async Task NmtQueueMultiple() string builds = ""; for (int i = 0; i < NUM_ENGINES; i++) { - TranslationBuild build = await _helperClient.translationEnginesClient.GetCurrentBuildAsync(engineIds[i]); + TranslationBuild build = await _helperClient.TranslationEnginesClient.GetCurrentBuildAsync(engineIds[i]); builds += $"{JsonSerializer.Serialize(build)}\n"; } - builds += "Depth = " + (await _helperClient.translationClient.GetQueueAsync("Nmt")).Size.ToString(); + builds += "Depth = " + (await _helperClient.TranslationEngineTypesClient.GetQueueAsync("Nmt")).Size.ToString(); //Status message of last started build says that there is at least one job ahead of it in the queue // (this variable due to how many jobs may already exist in the production queue from other Serval instances) - TranslationBuild newestEngineCurrentBuild = await _helperClient.translationEnginesClient.GetCurrentBuildAsync( + TranslationBuild newestEngineCurrentBuild = await _helperClient.TranslationEnginesClient.GetCurrentBuildAsync( engineIds[NUM_ENGINES - 1] ); int? queueDepth = newestEngineCurrentBuild.QueueDepth; - Queue queue = await _helperClient.translationClient.GetQueueAsync("Nmt"); + Queue queue = await _helperClient.TranslationEngineTypesClient.GetQueueAsync("Nmt"); for (int i = 0; i < NUM_ENGINES; i++) { try { - await _helperClient.translationEnginesClient.CancelBuildAsync(engineIds[i]); + await _helperClient.TranslationEnginesClient.CancelBuildAsync(engineIds[i]); } catch { } } - Assert.NotNull(queueDepth, JsonSerializer.Serialize(newestEngineCurrentBuild) + "|||" + builds); + Assert.That( + queueDepth, + Is.Not.Null, + message: JsonSerializer.Serialize(newestEngineCurrentBuild) + "|||" + builds + ); Assert.Multiple(() => { Assert.That(queueDepth, Is.GreaterThan(0), message: builds); @@ -185,14 +182,13 @@ public async Task NmtQueueMultiple() [Test] public async Task NmtLargeBatch() { - await _helperClient!.ClearEngines(); - string engineId = await _helperClient.CreateNewEngine("Nmt", "es", "en", "NMT3"); - var books = new string[] { "bible_LARGEFILE.txt" }; - await _helperClient.AddTextCorpusToEngine(engineId, books, "es", "en", false); - var cId = await _helperClient.AddTextCorpusToEngine(engineId, ["3JN.txt"], "es", "en", true); - await _helperClient.BuildEngine(engineId); + string engineId = await _helperClient.CreateNewEngineAsync("Nmt", "es", "en", "NMT3"); + string[] books = ["bible_LARGEFILE.txt"]; + await _helperClient.AddTextCorpusToEngineAsync(engineId, books, "es", "en", false); + var cId = await _helperClient.AddTextCorpusToEngineAsync(engineId, ["3JN.txt"], "es", "en", true); + await _helperClient.BuildEngineAsync(engineId); await Task.Delay(1000); - IList lTrans = await _helperClient.translationEnginesClient.GetAllPretranslationsAsync( + IList lTrans = await _helperClient.TranslationEnginesClient.GetAllPretranslationsAsync( engineId, cId ); @@ -202,65 +198,59 @@ public async Task NmtLargeBatch() [Test] public async Task GetNmtCancelAndRestartBuild() { - await _helperClient!.ClearEngines(); - string engineId = await _helperClient.CreateNewEngine("Nmt", "es", "en", "NMT2"); - var books = new string[] { "1JN.txt", "2JN.txt", "3JN.txt" }; - await _helperClient.AddTextCorpusToEngine(engineId, books, "es", "en", false); + string engineId = await _helperClient.CreateNewEngineAsync("Nmt", "es", "en", "NMT2"); + string[] books = ["1JN.txt", "2JN.txt", "3JN.txt"]; + await _helperClient.AddTextCorpusToEngineAsync(engineId, books, "es", "en", false); await StartAndCancelTwice(engineId); } [Test] public async Task CircuitousRouteGetWordGraphAsync() { - await _helperClient!.ClearEngines(); - //Create smt engine - string smtEngineId = await _helperClient.CreateNewEngine("SmtTransfer", "es", "en", "SMT5"); + string smtEngineId = await _helperClient.CreateNewEngineAsync("SmtTransfer", "es", "en", "SMT5"); //Try to get word graph - should fail: unbuilt ServalApiException? ex = Assert.ThrowsAsync(async () => { - await _helperClient.translationEnginesClient.GetWordGraphAsync(smtEngineId, "verdad"); + await _helperClient.TranslationEnginesClient.GetWordGraphAsync(smtEngineId, "verdad"); }); - Assert.NotNull(ex); - Assert.That(ex!.StatusCode, Is.EqualTo(409)); + Assert.That(ex, Is.Not.Null); + Assert.That(ex.StatusCode, Is.EqualTo(409)); //Add corpus - var cId = await _helperClient.AddTextCorpusToEngine( + var cId = await _helperClient.AddTextCorpusToEngineAsync( smtEngineId, - new string[] { "2JN.txt", "3JN.txt" }, + ["2JN.txt", "3JN.txt"], "es", "en", false ); //Build the new engine - await _helperClient.BuildEngine(smtEngineId); + await _helperClient.BuildEngineAsync(smtEngineId); //Remove added corpus (shouldn't affect translation) - await _helperClient.translationEnginesClient.DeleteCorpusAsync(smtEngineId, cId); + await _helperClient.TranslationEnginesClient.DeleteCorpusAsync(smtEngineId, cId); //Add corpus - await _helperClient.AddTextCorpusToEngine( + await _helperClient.AddTextCorpusToEngineAsync( smtEngineId, - new string[] { "1JN.txt", "2JN.txt", "3JN.txt" }, + ["1JN.txt", "2JN.txt", "3JN.txt"], "es", "en", false ); //Build the new engine - await _helperClient.BuildEngine(smtEngineId); + await _helperClient.BuildEngineAsync(smtEngineId); - WordGraph result = await _helperClient.translationEnginesClient.GetWordGraphAsync(smtEngineId, "verdad"); + WordGraph result = await _helperClient.TranslationEnginesClient.GetWordGraphAsync(smtEngineId, "verdad"); Assert.That(result.SourceTokens, Has.Count.EqualTo(1)); Assert.That( - result - .Arcs.Where(arc => arc != null && arc.Confidences != null)! - .MaxBy(arc => arc.Confidences.Average())! - .TargetTokens.All(tk => tk == "truth"), - "Best translation should have been 'truth'but returned word graph: \n{0}", - JsonSerializer.Serialize(result) + result.Arcs.MaxBy(arc => arc.Confidences.Average())?.TargetTokens.All(tk => tk == "truth"), + Is.True, + message: $"Best translation should have been 'truth'but returned word graph: \n{JsonSerializer.Serialize(result)}" ); } @@ -270,90 +260,86 @@ public async Task CircuitousRouteTranslateTopNAsync() const int N = 3; //Create engine - string engineId = await _helperClient!.CreateNewEngine("smt-transfer", "en", "fa", "SMT6"); + string engineId = await _helperClient.CreateNewEngineAsync("smt-transfer", "en", "fa", "SMT6"); //Retrieve engine - TranslationEngine? engine = await _helperClient.translationEnginesClient.GetAsync(engineId); - Assert.NotNull(engine); + TranslationEngine engine = await _helperClient.TranslationEnginesClient.GetAsync(engineId); Assert.That(engine.Type, Is.EqualTo("smt-transfer")); //Add corpus - string cId = await _helperClient.AddTextCorpusToEngine( + string cId = await _helperClient.AddTextCorpusToEngineAsync( engineId, - new string[] { "1JN.txt", "2JN.txt", "3JN.txt" }, + ["1JN.txt", "2JN.txt", "3JN.txt"], "en", "fa", false ); //Retrieve corpus - TranslationCorpus? corpus = await _helperClient.translationEnginesClient.GetCorpusAsync(engineId, cId); - Assert.NotNull(corpus); + TranslationCorpus corpus = await _helperClient.TranslationEnginesClient.GetCorpusAsync(engineId, cId); Assert.That(corpus.SourceLanguage, Is.EqualTo("en")); Assert.That(corpus.TargetFiles, Has.Count.EqualTo(3)); //Build engine - await _helperClient.BuildEngine(engineId); + await _helperClient.BuildEngineAsync(engineId); //Get top `N` translations - ICollection results = await _helperClient.translationEnginesClient.TranslateNAsync( + ICollection results = await _helperClient.TranslationEnginesClient.TranslateNAsync( engineId, N, "love" ); - Assert.NotNull(results); Assert.That( - results.MaxBy(t => t.Confidences.Average())!.Translation, + results.MaxBy(t => t.Confidences.Average())?.Translation, Is.EqualTo("amour"), - "Expected best translation to be 'amour' but results were this:\n" + JsonSerializer.Serialize(results) + message: "Expected best translation to be 'amour' but results were this:\n" + + JsonSerializer.Serialize(results) ); } [Test] public async Task GetSmtCancelAndRestartBuild() { - await _helperClient!.ClearEngines(); - string engineId = await _helperClient.CreateNewEngine("smt-transfer", "es", "en", "SMT7"); - var books = new string[] { "1JN.txt", "2JN.txt", "3JN.txt" }; - await _helperClient.AddTextCorpusToEngine(engineId, books, "es", "en", false); + string engineId = await _helperClient.CreateNewEngineAsync("smt-transfer", "es", "en", "SMT7"); + string[] books = ["1JN.txt", "2JN.txt", "3JN.txt"]; + await _helperClient.AddTextCorpusToEngineAsync(engineId, books, "es", "en", false); await StartAndCancelTwice(engineId); // do a job normally and make sure it works. - await _helperClient.BuildEngine(engineId); - TranslationResult tResult = await _helperClient.translationEnginesClient.TranslateAsync(engineId, "Espíritu"); - Assert.AreEqual(tResult.Translation, "spirit"); + await _helperClient.BuildEngineAsync(engineId); + TranslationResult tResult = await _helperClient.TranslationEnginesClient.TranslateAsync(engineId, "Espíritu"); + Assert.That(tResult.Translation, Is.EqualTo("spirit")); } async Task StartAndCancelTwice(string engineId) { // start and first job - var build = await _helperClient!.StartBuildAsync(engineId); + var build = await _helperClient.StartBuildAsync(engineId); await Task.Delay(1000); - build = await _helperClient.translationEnginesClient.GetBuildAsync(engineId, build.Id); + build = await _helperClient.TranslationEnginesClient.GetBuildAsync(engineId, build.Id); Assert.That(build.State == JobState.Active || build.State == JobState.Pending); // and then cancel it - await _helperClient.CancelBuild(engineId, build.Id); - build = await _helperClient.translationEnginesClient.GetBuildAsync(engineId, build.Id); + await _helperClient.CancelBuildAsync(engineId, build.Id); + build = await _helperClient.TranslationEnginesClient.GetBuildAsync(engineId, build.Id); Assert.That(build.State == JobState.Canceled); // do a second job normally and make sure it works. build = await _helperClient.StartBuildAsync(engineId); await Task.Delay(1000); - build = await _helperClient.translationEnginesClient.GetBuildAsync(engineId, build.Id); + build = await _helperClient.TranslationEnginesClient.GetBuildAsync(engineId, build.Id); Assert.That(build.State == JobState.Active || build.State == JobState.Pending); // and cancel again - let's not wait forever - await _helperClient.CancelBuild(engineId, build.Id); - build = await _helperClient.translationEnginesClient.GetBuildAsync(engineId, build.Id); + await _helperClient.CancelBuildAsync(engineId, build.Id); + build = await _helperClient.TranslationEnginesClient.GetBuildAsync(engineId, build.Id); Assert.That(build.State == JobState.Canceled); } [Test] public async Task ParatextProjectNmtJobAsync() { - await _helperClient!.ClearEngines(); string tempDirectory = Path.GetTempPath(); DataFile file1, file2; @@ -368,11 +354,11 @@ public async Task ParatextProjectNmtJobAsync() Path.Combine(tempDirectory, "TestProjectTarget.zip") ); - file1 = await _helperClient.dataFilesClient.CreateAsync( + file1 = await _helperClient.DataFilesClient.CreateAsync( new FileParameter(data: File.OpenRead(Path.Combine(tempDirectory, "TestProject.zip"))), FileFormat.Paratext ); - file2 = await _helperClient.dataFilesClient.CreateAsync( + file2 = await _helperClient.DataFilesClient.CreateAsync( new FileParameter(data: File.OpenRead(Path.Combine(tempDirectory, "TestProjectTarget.zip"))), FileFormat.Paratext ); @@ -383,38 +369,38 @@ public async Task ParatextProjectNmtJobAsync() File.Delete(Path.Combine(tempDirectory, "TestProjectTarget.zip")); } - string engineId = await _helperClient.CreateNewEngine("Nmt", "en", "sbp", "NMT4"); + string engineId = await _helperClient.CreateNewEngineAsync("Nmt", "en", "sbp", "NMT4"); - TranslationCorpus corpus = await _helperClient.translationEnginesClient.AddCorpusAsync( + TranslationCorpus corpus = await _helperClient.TranslationEnginesClient.AddCorpusAsync( engineId, new TranslationCorpusConfig { SourceLanguage = "en", TargetLanguage = "sbp", - SourceFiles = new TranslationCorpusFileConfig[] - { - new TranslationCorpusFileConfig { FileId = file1.Id } - }, - TargetFiles = new TranslationCorpusFileConfig[] - { - new TranslationCorpusFileConfig { FileId = file2.Id } - } + SourceFiles = [new() { FileId = file1.Id }], + TargetFiles = [new() { FileId = file2.Id }] } ); _helperClient.TranslationBuildConfig.Pretranslate!.Add( - new PretranslateCorpusConfig { CorpusId = corpus.Id, TextIds = new string[] { "JHN", "REV" } } + new PretranslateCorpusConfig { CorpusId = corpus.Id, TextIds = ["JHN", "REV"] } ); _helperClient.TranslationBuildConfig.Options = "{\"max_steps\":10, \"use_key_terms\":true}"; - await _helperClient.BuildEngine(engineId); + await _helperClient.BuildEngineAsync(engineId); Assert.That( - (await _helperClient.translationEnginesClient.GetAllBuildsAsync(engineId)).First().State - == JobState.Completed + (await _helperClient.TranslationEnginesClient.GetAllBuildsAsync(engineId)).First().State, + Is.EqualTo(JobState.Completed) ); - IList lTrans = await _helperClient.translationEnginesClient.GetAllPretranslationsAsync( + IList lTrans = await _helperClient.TranslationEnginesClient.GetAllPretranslationsAsync( engineId, corpus.Id ); Assert.That(lTrans, Is.Not.Empty); } + + [TearDown] + public async Task TearDown() + { + await _helperClient.DisposeAsync(); + } } diff --git a/tests/Serval.E2ETests/ServalClientHelper.cs b/tests/Serval.E2ETests/ServalClientHelper.cs index fd1c3c7e..96b60145 100644 --- a/tests/Serval.E2ETests/ServalClientHelper.cs +++ b/tests/Serval.E2ETests/ServalClientHelper.cs @@ -1,18 +1,15 @@ -using Microsoft.AspNetCore.Mvc; -using Newtonsoft.Json; +namespace Serval.E2ETests; -public class ServalClientHelper +public class ServalClientHelper : IAsyncDisposable { - public readonly DataFilesClient dataFilesClient; - public readonly TranslationEnginesClient translationEnginesClient; - public readonly TranslationClient translationClient; private readonly HttpClient _httpClient; - readonly Dictionary EnginePerUser = []; - private string _prefix; + private readonly Dictionary _enginePerUser = []; + private readonly string _prefix; + private readonly string _audience; public ServalClientHelper(string audience, string prefix = "SCE_", bool ignoreSSLErrors = false) { - Dictionary env = GetEnvironment(); + _audience = audience; //setup http client if (ignoreSSLErrors) { @@ -21,15 +18,14 @@ public ServalClientHelper(string audience, string prefix = "SCE_", bool ignoreSS } else _httpClient = new HttpClient(); - _httpClient.BaseAddress = new Uri(env["hostUrl"]); + string? hostUrl = Environment.GetEnvironmentVariable("SERVAL_HOST_URL"); + if (hostUrl is null) + throw new InvalidOperationException("The environment variable SERVAL_HOST_URL is not set."); + _httpClient.BaseAddress = new Uri(hostUrl); _httpClient.Timeout = TimeSpan.FromSeconds(60); - dataFilesClient = new DataFilesClient(_httpClient); - translationEnginesClient = new TranslationEnginesClient(_httpClient); - translationClient = new TranslationClient(_httpClient); - _httpClient.DefaultRequestHeaders.Add( - "authorization", - $"Bearer {GetAuth0Authentication(env["authUrl"], audience, env["clientId"], env["clientSecret"]).Result}" - ); + DataFilesClient = new DataFilesClient(_httpClient); + TranslationEnginesClient = new TranslationEnginesClient(_httpClient); + TranslationEngineTypesClient = new TranslationEngineTypesClient(_httpClient); _prefix = prefix; TranslationBuildConfig = new TranslationBuildConfig { @@ -38,67 +34,53 @@ public ServalClientHelper(string audience, string prefix = "SCE_", bool ignoreSS }; } - public TranslationBuildConfig TranslationBuildConfig { get; set; } - - public static Dictionary GetEnvironment() + public async Task InitAsync() { - Dictionary env = - new() - { - { "hostUrl", Environment.GetEnvironmentVariable("SERVAL_HOST_URL") ?? "" }, - { "clientId", Environment.GetEnvironmentVariable("SERVAL_CLIENT_ID") ?? "" }, - { "clientSecret", Environment.GetEnvironmentVariable("SERVAL_CLIENT_SECRET") ?? "" }, - { "authUrl", Environment.GetEnvironmentVariable("SERVAL_AUTH_URL") ?? "" } - }; - if (env["hostUrl"] == null) - { - Console.WriteLine( - "You need a serval host url in the environment variable SERVAL_HOST_URL! Look at README for instructions on getting one." - ); - } - else if (env["clientId"] == null) - { - Console.WriteLine( - "You need an auth0 client_id in the environment variable SERVAL_CLIENT_ID! Look at README for instructions on getting one." - ); - } - else if (env["clientSecret"] == null) - { - Console.WriteLine( - "You need an auth0 client_secret in the environment variable SERVAL_CLIENT_SECRET! Look at README for instructions on getting one." - ); - } - else if (env["authUrl"] == null) - { - Console.WriteLine( - "You need an auth0 authorization url in the environment variable SERVAL_AUTH_URL! Look at README for instructions on getting one." - ); - } - return env; + string? authUrl = Environment.GetEnvironmentVariable("SERVAL_AUTH_URL"); + if (authUrl is null) + throw new InvalidOperationException("The environment variable SERVAL_HOST_URL is not set."); + string? clientId = Environment.GetEnvironmentVariable("SERVAL_CLIENT_ID"); + if (clientId is null) + throw new InvalidOperationException("The environment variable SERVAL_CLIENT_ID is not set."); + string? clientSecret = Environment.GetEnvironmentVariable("SERVAL_CLIENT_SECRET"); + if (clientSecret is null) + throw new InvalidOperationException("The environment variable SERVAL_CLIENT_SECRET is not set."); + + string authToken = await GetAuth0AuthenticationAsync(authUrl, _audience, clientId, clientSecret); + + _httpClient.DefaultRequestHeaders.Add("authorization", $"Bearer {authToken}"); + + await ClearEnginesAsync(); } - public async Task ClearEngines(string name = "") + public DataFilesClient DataFilesClient { get; } + public TranslationEnginesClient TranslationEnginesClient { get; } + public TranslationEngineTypesClient TranslationEngineTypesClient { get; } + + public TranslationBuildConfig TranslationBuildConfig { get; set; } + + public async Task ClearEnginesAsync(string name = "") { - IList existingTranslationEngines = await translationEnginesClient.GetAllAsync(); + IList existingTranslationEngines = await TranslationEnginesClient.GetAllAsync(); foreach (var translationEngine in existingTranslationEngines) { if (translationEngine.Name?.Contains(_prefix + name) ?? false) { - await translationEnginesClient.DeleteAsync(translationEngine.Id); + await TranslationEnginesClient.DeleteAsync(translationEngine.Id); } } TranslationBuildConfig.Pretranslate = new List(); - EnginePerUser.Clear(); + _enginePerUser.Clear(); } - public async Task CreateNewEngine( + public async Task CreateNewEngineAsync( string engineTypeString, string source_language, string target_language, string name = "" ) { - var engine = await translationEnginesClient.CreateAsync( + var engine = await TranslationEnginesClient.CreateAsync( new TranslationEngineConfig { Name = _prefix + name, @@ -107,25 +89,25 @@ public async Task CreateNewEngine( Type = engineTypeString } ); - EnginePerUser.Add(name, engine.Id); + _enginePerUser.Add(name, engine.Id); return engine.Id; } public async Task StartBuildAsync(string engineId) { - return await translationEnginesClient.StartBuildAsync(engineId, TranslationBuildConfig); + return await TranslationEnginesClient.StartBuildAsync(engineId, TranslationBuildConfig); } - public async Task BuildEngine(string engineId) + public async Task BuildEngineAsync(string engineId) { var newJob = await StartBuildAsync(engineId); int revision = newJob.Revision; - await translationEnginesClient.GetBuildAsync(engineId, newJob.Id, newJob.Revision); + await TranslationEnginesClient.GetBuildAsync(engineId, newJob.Id, newJob.Revision); while (true) { try { - var result = await translationEnginesClient.GetBuildAsync(engineId, newJob.Id, revision + 1); + var result = await TranslationEnginesClient.GetBuildAsync(engineId, newJob.Id, revision + 1); if (!(result.State == JobState.Active || result.State == JobState.Pending)) { // build completed @@ -144,14 +126,14 @@ public async Task BuildEngine(string engineId) } } - public async Task CancelBuild(string engineId, string buildId, int timeoutSeconds = 20) + public async Task CancelBuildAsync(string engineId, string buildId, int timeoutSeconds = 20) { - await translationEnginesClient.CancelBuildAsync(engineId); + await TranslationEnginesClient.CancelBuildAsync(engineId); int pollIntervalMs = 1000; int tries = 1; while (true) { - var build = await translationEnginesClient.GetBuildAsync(engineId, buildId); + var build = await TranslationEnginesClient.GetBuildAsync(engineId, buildId); if (build.State != JobState.Pending && build.State != JobState.Active) { break; @@ -164,7 +146,7 @@ public async Task CancelBuild(string engineId, string buildId, int timeoutSecond } } - public async Task AddTextCorpusToEngine( + public async Task AddTextCorpusToEngineAsync( string engineId, string[] filesToAdd, string sourceLanguage, @@ -172,12 +154,12 @@ public async Task AddTextCorpusToEngine( bool pretranslate ) { - List sourceFiles = await UploadFiles(filesToAdd, FileFormat.Text, sourceLanguage); + List sourceFiles = await UploadFilesAsync(filesToAdd, FileFormat.Text, sourceLanguage); var targetFileConfig = new List(); if (!pretranslate) { - var targetFiles = await UploadFiles(filesToAdd, FileFormat.Text, targetLanguage); + var targetFiles = await UploadFilesAsync(filesToAdd, FileFormat.Text, targetLanguage); foreach (var item in targetFiles.Select((file, i) => new { i, file })) { targetFileConfig.Add( @@ -204,7 +186,7 @@ bool pretranslate } } - var response = await translationEnginesClient.AddCorpusAsync( + var response = await TranslationEnginesClient.AddCorpusAsync( id: engineId, new TranslationCorpusConfig { @@ -226,7 +208,7 @@ bool pretranslate return response.Id; } - public async Task> UploadFiles( + public async Task> UploadFilesAsync( IEnumerable filesToAdd, FileFormat fileFormat, string language @@ -242,9 +224,9 @@ string language if (files.Length == 0) throw new ArgumentException($"The language data directory {languageFolder} contains no files!"); var fileList = new List(); - var allFiles = await dataFilesClient.GetAllAsync(); - ILookup filenameToId = allFiles - .Where(f => f.Name is object) // not null + var allFiles = await DataFilesClient.GetAllAsync(); + ILookup filenameToId = allFiles + .Where(f => f.Name is not null) .ToLookup(file => file.Name!, file => file.Id); foreach (var fileName in filesToAdd) @@ -257,7 +239,7 @@ string language var matchedFiles = filenameToId[fullName]; foreach (var fileId in matchedFiles) { - await dataFilesClient.DeleteAsync(fileId); + await DataFilesClient.DeleteAsync(fileId); } } @@ -265,7 +247,7 @@ string language string filePath = Path.GetFullPath(Path.Combine(languageFolder, fileName)); if (!File.Exists(filePath)) throw new FileNotFoundException($"The corpus file {filePath} does not exist!"); - var response = await dataFilesClient.CreateAsync( + var response = await DataFilesClient.CreateAsync( file: new FileParameter(data: File.OpenRead(filePath), fileName: fileName), format: fileFormat, name: fullName @@ -275,43 +257,53 @@ string language return fileList; } - private async Task GetAuth0Authentication( + private static async Task GetAuth0AuthenticationAsync( string authUrl, string audience, string clientId, string clientSecret ) { - var authHttpClient = new HttpClient(); - authHttpClient.Timeout = TimeSpan.FromSeconds(3); - var request = new HttpRequestMessage(HttpMethod.Post, authUrl + "/oauth/token"); - request.Content = new FormUrlEncodedContent( - new Dictionary - { - { "grant_type", "client_credentials" }, - { "client_id", clientId }, - { "client_secret", clientSecret }, - { "audience", audience }, - } - ); + var authHttpClient = new HttpClient { Timeout = TimeSpan.FromSeconds(3) }; + var request = new HttpRequestMessage(HttpMethod.Post, authUrl + "/oauth/token") + { + Content = new FormUrlEncodedContent( + new Dictionary + { + { "grant_type", "client_credentials" }, + { "client_id", clientId }, + { "client_secret", clientSecret }, + { "audience", audience }, + } + ) + }; var response = authHttpClient.SendAsync(request).Result; if (response.Content is null) throw new HttpRequestException("Error getting auth0 Authentication."); - var dict = JsonConvert.DeserializeObject>( - await response.Content.ReadAsStringAsync() - ); + var dict = JsonSerializer.Deserialize>(await response.Content.ReadAsStringAsync()); return dict?["access_token"] ?? ""; } - private HttpClientHandler GetHttHandlerToIgnoreSslErrors() + private static HttpClientHandler GetHttHandlerToIgnoreSslErrors() { //ignore ssl errors - HttpClientHandler handler = new HttpClientHandler(); - handler.ClientCertificateOptions = ClientCertificateOption.Manual; - handler.ServerCertificateCustomValidationCallback = (httpRequestMessage, cert, cetChain, policyErrors) => - { - return true; - }; + HttpClientHandler handler = + new() + { + ClientCertificateOptions = ClientCertificateOption.Manual, + ServerCertificateCustomValidationCallback = (httpRequestMessage, cert, cetChain, policyErrors) => + { + return true; + } + }; return handler; } + + public async ValueTask DisposeAsync() + { + await ClearEnginesAsync(); + + _httpClient.Dispose(); + GC.SuppressFinalize(this); + } } diff --git a/tests/Serval.Shared.Tests/Serval.Shared.Tests.csproj b/tests/Serval.Shared.Tests/Serval.Shared.Tests.csproj new file mode 100644 index 00000000..bc72afdd --- /dev/null +++ b/tests/Serval.Shared.Tests/Serval.Shared.Tests.csproj @@ -0,0 +1,36 @@ + + + + net8.0 + enable + enable + + false + true + Serval.Shared + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + diff --git a/tests/Serval.Shared.Tests/Services/ScriptureDataFileServiceTests.cs b/tests/Serval.Shared.Tests/Services/ScriptureDataFileServiceTests.cs new file mode 100644 index 00000000..db0219ff --- /dev/null +++ b/tests/Serval.Shared.Tests/Services/ScriptureDataFileServiceTests.cs @@ -0,0 +1,114 @@ +namespace Serval.Shared.Services; + +[TestFixture] +public class ScriptureDataFileServiceTests +{ + [Test] + public void GetParatextProjectSettings() + { + TestEnvironment env = new(); + ParatextProjectSettings settings = env.Service.GetParatextProjectSettings("file1.zip"); + Assert.That(settings.Name, Is.EqualTo("PROJ")); + } + + [Test] + public async Task ReadParatextProjectBookAsync_Exists() + { + TestEnvironment env = new(); + string? usfm = await env.Service.ReadParatextProjectBookAsync("file1.zip", "MAT"); + Assert.That(usfm, Is.Not.Null); + Assert.That( + usfm.Replace("\r\n", "\n"), + Is.EqualTo( + @"\id MAT - PROJ +\h Matthew +\c 1 +\p +\v 1 Chapter one, verse one. +\v 2 Chapter one, verse two. +\c 2 +\p +\v 1 Chapter two, verse one. +\v 2 Chapter two, verse two. +".Replace("\r\n", "\n") + ) + ); + } + + [Test] + public async Task ReadParatextProjectBookAsync_DoesNotExist() + { + TestEnvironment env = new(); + string? usfm = await env.Service.ReadParatextProjectBookAsync("file1.zip", "MRK"); + Assert.That(usfm, Is.Null); + } + + private class TestEnvironment + { + public TestEnvironment() + { + var fileSystem = Substitute.For(); + fileSystem + .OpenZipFile("file1.zip") + .Returns(ci => + { + IZipContainer container = CreateZipContainer(); + AddBook(container, "MAT"); + return container; + }); + var dataFileOptions = Substitute.For>(); + dataFileOptions.CurrentValue.Returns(new DataFileOptions()); + + Service = new ScriptureDataFileService(fileSystem, dataFileOptions); + } + + public ScriptureDataFileService Service { get; } + + private static IZipContainer CreateZipContainer() + { + var container = Substitute.For(); + container.EntryExists("Settings.xml").Returns(true); + XElement settingsXml = + new( + "ScriptureText", + new XElement("StyleSheet", "usfm.sty"), + new XElement("Name", "PROJ"), + new XElement("FullName", "PROJ"), + new XElement("Encoding", "65001"), + new XElement( + "Naming", + new XAttribute("PrePart", ""), + new XAttribute("PostPart", "PROJ.SFM"), + new XAttribute("BookNameForm", "MAT") + ), + new XElement("BiblicalTermsListSetting", "Major::BiblicalTerms.xml") + ); + container + .OpenEntry("Settings.xml") + .Returns(new MemoryStream(Encoding.UTF8.GetBytes(settingsXml.ToString()))); + container.EntryExists("custom.vrs").Returns(false); + container.EntryExists("usfm.sty").Returns(false); + container.EntryExists("custom.sty").Returns(false); + return container; + } + + private static void AddBook(IZipContainer container, string book) + { + string bookFileName = $"{book}PROJ.SFM"; + container.EntryExists(bookFileName).Returns(true); + string usfm = + $@"\id {book} - PROJ +\h {Canon.BookIdToEnglishName(book)} +\c 1 +\p +\v 1 Chapter one, verse one. +\v 2 Chapter one, verse two. +\c 2 +\p +\v 1 Chapter two, verse one. +\v 2 Chapter two, verse two. +"; + container.OpenEntry(bookFileName).Returns(new MemoryStream(Encoding.UTF8.GetBytes(usfm))); + } + } +} diff --git a/tests/Serval.Shared.Tests/Usings.cs b/tests/Serval.Shared.Tests/Usings.cs new file mode 100644 index 00000000..1c985f06 --- /dev/null +++ b/tests/Serval.Shared.Tests/Usings.cs @@ -0,0 +1,8 @@ +global using System.Text; +global using System.Xml.Linq; +global using Microsoft.Extensions.Options; +global using NSubstitute; +global using NUnit.Framework; +global using Serval.Shared.Configuration; +global using SIL.Machine.Corpora; +global using SIL.Scripture; diff --git a/tests/Serval.Translation.Tests/Serval.Translation.Tests.csproj b/tests/Serval.Translation.Tests/Serval.Translation.Tests.csproj index b04e0780..8a82e2b9 100644 --- a/tests/Serval.Translation.Tests/Serval.Translation.Tests.csproj +++ b/tests/Serval.Translation.Tests/Serval.Translation.Tests.csproj @@ -13,15 +13,15 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/Serval.Translation.Tests/Services/EngineServiceTests.cs b/tests/Serval.Translation.Tests/Services/EngineServiceTests.cs index 311cfa2b..3c0b6a2f 100644 --- a/tests/Serval.Translation.Tests/Services/EngineServiceTests.cs +++ b/tests/Serval.Translation.Tests/Services/EngineServiceTests.cs @@ -1,5 +1,6 @@ using Google.Protobuf.WellKnownTypes; using Microsoft.Extensions.Logging; +using Serval.Shared.Utils; using Serval.Translation.V1; namespace Serval.Translation.Services; @@ -10,11 +11,10 @@ public class EngineServiceTests const string BUILD1_ID = "b00000000000000000000001"; [Test] - public async Task TranslateAsync_EngineDoesNotExist() + public void TranslateAsync_EngineDoesNotExist() { var env = new TestEnvironment(); - Models.TranslationResult? result = await env.Service.TranslateAsync("engine1", "esto es una prueba."); - Assert.That(result, Is.Null); + Assert.ThrowsAsync(() => env.Service.TranslateAsync("engine1", "esto es una prueba.")); } [Test] @@ -28,11 +28,12 @@ public async Task TranslateAsync_EngineExists() } [Test] - public async Task GetWordGraphAsync_EngineDoesNotExist() + public void GetWordGraphAsync_EngineDoesNotExist() { var env = new TestEnvironment(); - Models.WordGraph? result = await env.Service.GetWordGraphAsync("engine1", "esto es una prueba."); - Assert.That(result, Is.Null); + Assert.ThrowsAsync( + () => env.Service.GetWordGraphAsync("engine1", "esto es una prueba.") + ); } [Test] @@ -46,16 +47,12 @@ public async Task GetWordGraphAsync_EngineExists() } [Test] - public async Task TrainSegmentAsync_EngineDoesNotExist() + public void TrainSegmentAsync_EngineDoesNotExist() { var env = new TestEnvironment(); - bool result = await env.Service.TrainSegmentPairAsync( - "engine1", - "esto es una prueba.", - "this is a test.", - true + Assert.ThrowsAsync( + () => env.Service.TrainSegmentPairAsync("engine1", "esto es una prueba.", "this is a test.", true) ); - Assert.That(result, Is.False); } [Test] @@ -63,8 +60,9 @@ public async Task TrainSegmentAsync_EngineExists() { var env = new TestEnvironment(); string engineId = (await env.CreateEngineAsync()).Id; - bool result = await env.Service.TrainSegmentPairAsync(engineId, "esto es una prueba.", "this is a test.", true); - Assert.That(result, Is.True); + Assert.DoesNotThrowAsync( + () => env.Service.TrainSegmentPairAsync(engineId, "esto es una prueba.", "this is a test.", true) + ); } [Test] @@ -90,8 +88,7 @@ public async Task DeleteAsync_EngineExists() { var env = new TestEnvironment(); string engineId = (await env.CreateEngineAsync()).Id; - bool result = await env.Service.DeleteAsync("engine1"); - Assert.That(result, Is.True); + await env.Service.DeleteAsync("engine1"); Engine? engine = await env.Engines.GetAsync(engineId); Assert.That(engine, Is.Null); } @@ -101,8 +98,7 @@ public async Task DeleteAsync_ProjectDoesNotExist() { var env = new TestEnvironment(); await env.CreateEngineAsync(); - bool result = await env.Service.DeleteAsync("engine3"); - Assert.That(result, Is.False); + Assert.ThrowsAsync(() => env.Service.DeleteAsync("engine3")); } [Test] @@ -110,8 +106,7 @@ public async Task StartBuildAsync_EngineExists() { var env = new TestEnvironment(); string engineId = (await env.CreateEngineAsync()).Id; - bool result = await env.Service.StartBuildAsync(new Build { Id = BUILD1_ID, EngineRef = engineId }); - Assert.That(result, Is.True); + Assert.DoesNotThrowAsync(() => env.Service.StartBuildAsync(new Build { Id = BUILD1_ID, EngineRef = engineId })); } [Test] diff --git a/tests/Serval.Translation.Tests/Services/PretranslationServiceTests.cs b/tests/Serval.Translation.Tests/Services/PretranslationServiceTests.cs new file mode 100644 index 00000000..07a3572b --- /dev/null +++ b/tests/Serval.Translation.Tests/Services/PretranslationServiceTests.cs @@ -0,0 +1,189 @@ +namespace Serval.Translation.Services; + +[TestFixture] +public class PretranslationServiceTests +{ + [Test] + public async Task GetUsfmAsync_SourceBook() + { + TestEnvironment env = new(); + string usfm = await env.Service.GetUsfmAsync("engine1", 1, "corpus1", "MAT"); + Assert.That( + usfm.Replace("\r\n", "\n"), + Is.EqualTo( + @"\id MAT - TRG +\h +\c 1 +\p +\v 1 Chapter 1, verse 1. +\v 2 Chapter 1, verse 2. +\c 2 +\p +\v 1 Chapter 2, verse 1. +\v 2 +".Replace("\r\n", "\n") + ) + ); + } + + [Test] + public async Task GetUsfmAsync_TargetBook() + { + TestEnvironment env = new(); + env.AddMatthewToTarget(); + string usfm = await env.Service.GetUsfmAsync("engine1", 1, "corpus1", "MAT"); + Assert.That( + usfm.Replace("\r\n", "\n"), + Is.EqualTo( + @"\id MAT - TRG +\h Matthew +\c 1 +\p +\v 1 Chapter 1, verse 1. +\v 2 Chapter 1, verse 2. +\c 2 +\p +\v 1 Chapter 2, verse 1. +\v 2 Chapter two, verse two. +".Replace("\r\n", "\n") + ) + ); + } + + private class TestEnvironment + { + public TestEnvironment() + { + Engines = new MemoryRepository( + [ + new() + { + Id = "engine1", + SourceLanguage = "en", + TargetLanguage = "en", + Type = "nmt", + ModelRevision = 1, + Corpora = + [ + new() + { + Id = "corpus1", + SourceLanguage = "en", + TargetLanguage = "en", + SourceFiles = + [ + new() + { + Id = "file1", + Filename = "file1.zip", + Format = Shared.Contracts.FileFormat.Paratext, + TextId = "project1" + } + ], + TargetFiles = + [ + new() + { + Id = "file2", + Filename = "file2.zip", + Format = Shared.Contracts.FileFormat.Paratext, + TextId = "project1" + } + ], + } + ] + } + ] + ); + + Pretranslations = new MemoryRepository( + [ + new() + { + Id = "pt1", + EngineRef = "engine1", + ModelRevision = 1, + CorpusRef = "corpus1", + TextId = "MAT", + Refs = ["MAT 1:1"], + Translation = "Chapter 1, verse 1." + }, + new() + { + Id = "pt2", + EngineRef = "engine1", + ModelRevision = 1, + CorpusRef = "corpus1", + TextId = "MAT", + Refs = ["MAT 1:2"], + Translation = "Chapter 1, verse 2." + }, + new() + { + Id = "pt3", + EngineRef = "engine1", + ModelRevision = 1, + CorpusRef = "corpus1", + TextId = "MAT", + Refs = ["MAT 2:1"], + Translation = "Chapter 2, verse 1." + } + ] + ); + ScriptureDataFileService = Substitute.For(); + ScriptureDataFileService.GetParatextProjectSettings("file1.zip").Returns(CreateProjectSettings("SRC")); + ScriptureDataFileService.GetParatextProjectSettings("file2.zip").Returns(CreateProjectSettings("TRG")); + ScriptureDataFileService + .ReadParatextProjectBookAsync("file1.zip", "MAT") + .Returns(Task.FromResult(CreateUsfm("SRC", "MAT"))); + ScriptureDataFileService + .ReadParatextProjectBookAsync("file2.zip", "MAT") + .Returns(Task.FromResult(null)); + Service = new PretranslationService(Pretranslations, Engines, ScriptureDataFileService); + } + + public PretranslationService Service { get; } + public MemoryRepository Pretranslations { get; } + public MemoryRepository Engines { get; } + public IScriptureDataFileService ScriptureDataFileService { get; } + + public void AddMatthewToTarget() + { + ScriptureDataFileService + .ReadParatextProjectBookAsync("file2.zip", "MAT") + .Returns(Task.FromResult(CreateUsfm("TRG", "MAT"))); + } + + private static ParatextProjectSettings CreateProjectSettings(string name) + { + return new ParatextProjectSettings( + name: name, + fullName: name, + encoding: Encoding.UTF8, + versification: ScrVers.English, + stylesheet: new UsfmStylesheet("usfm.sty"), + fileNamePrefix: "", + fileNameForm: "MAT", + fileNameSuffix: $"{name}.SFM", + biblicalTermsListType: "Major", + biblicalTermsProjectName: "", + biblicalTermsFileName: "BiblicalTerms.xml" + ); + } + + private static string CreateUsfm(string name, string book) + { + return $@"\id {book} - {name} +\h {Canon.BookIdToEnglishName(book)} +\c 1 +\p +\v 1 Chapter one, verse one. +\v 2 Chapter one, verse two. +\c 2 +\p +\v 1 Chapter two, verse one. +\v 2 Chapter two, verse two. +"; + } + } +} diff --git a/tests/Serval.Translation.Tests/Usings.cs b/tests/Serval.Translation.Tests/Usings.cs index 54113655..fe6de340 100644 --- a/tests/Serval.Translation.Tests/Usings.cs +++ b/tests/Serval.Translation.Tests/Usings.cs @@ -1,9 +1,12 @@ -global using CaseExtensions; +global using System.Text; global using Grpc.Core; global using Grpc.Net.ClientFactory; global using Microsoft.Extensions.Options; global using NSubstitute; global using NUnit.Framework; global using Serval.Shared.Configuration; +global using Serval.Shared.Services; global using Serval.Translation.Models; global using SIL.DataAccess; +global using SIL.Machine.Corpora; +global using SIL.Scripture; diff --git a/tests/Serval.Webhooks.Tests/Serval.Webhooks.Tests.csproj b/tests/Serval.Webhooks.Tests/Serval.Webhooks.Tests.csproj index 53e2d8dc..10a5c330 100644 --- a/tests/Serval.Webhooks.Tests/Serval.Webhooks.Tests.csproj +++ b/tests/Serval.Webhooks.Tests/Serval.Webhooks.Tests.csproj @@ -13,19 +13,19 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/tests/Serval.Webhooks.Tests/Services/WebhookJobTests.cs b/tests/Serval.Webhooks.Tests/Services/WebhookJobTests.cs index fdb4041c..13d7c6ad 100644 --- a/tests/Serval.Webhooks.Tests/Services/WebhookJobTests.cs +++ b/tests/Serval.Webhooks.Tests/Services/WebhookJobTests.cs @@ -28,7 +28,7 @@ public async Task RunAsync_MatchingHook() Url = "https://test.client.com/hook", Secret = "this is a secret", Owner = "client", - Events = new List { WebhookEvent.TranslationBuildStarted } + Events = [WebhookEvent.TranslationBuildStarted] } ); env.MockHttp.Expect("https://test.client.com/hook") @@ -55,7 +55,7 @@ public async Task RunAsync_NoMatchingHook() Url = "https://test.client.com/hook", Secret = "this is a secret", Owner = "client", - Events = new List { WebhookEvent.TranslationBuildStarted } + Events = [WebhookEvent.TranslationBuildStarted] } ); MockedRequest req = env.MockHttp.When("*").Respond(HttpStatusCode.OK); @@ -77,7 +77,7 @@ public void RunAsync_RequestTimeout() Url = "https://test.client.com/hook", Secret = "this is a secret", Owner = "client", - Events = new List { WebhookEvent.TranslationBuildStarted } + Events = [WebhookEvent.TranslationBuildStarted] } ); env.MockHttp.Expect("https://test.client.com/hook")