From 79be5df1e7c3cf18fc6c8d69d7b1e7d6f4957121 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Tue, 5 Nov 2024 15:04:38 -0500 Subject: [PATCH] Add Word Alignment engine: * Add HTTP API * Add Grpc * Add Echo (as single engine) * Add Serval services * Add webhooks * Update docker-compose * Basic echo works All engines need to have different names, even if they are in different controllers. Calling Echo for word alignment, echo-word-alignment. --- .vscode/launch.json | 6 +- Serval.sln | 40 +- deploy/serval/templates/echo-deployment.yaml | 2 +- docker-compose.withatlas.yml | 11 +- docker-compose.yml | 7 +- dockerfile | 2 +- .../BackgroundTaskQueue.cs | 2 +- .../BackgroundTaskService.cs | 2 +- .../EchoEngine.csproj} | 0 .../HealthServiceV1.cs | 2 +- .../Program.cs | 24 +- .../Properties/launchSettings.json | 4 +- .../TranslationEngineServiceV1.cs | 4 +- .../Usings.cs | 3 +- .../WordAlignmentEngineServiceV1.cs | 221 + .../appsettings.Development.json | 5 +- .../appsettings.Production.json | 0 .../appsettings.Staging.json | 0 .../appsettings.json | 0 .../Serval.ApiServer/Serval.ApiServer.csproj | 1 + src/Serval/src/Serval.ApiServer/Startup.cs | 4 + .../appsettings.Development.json | 14 +- .../Serval.Assessment/Models/CorpusFile.cs | 9 - .../Services/EngineService.cs | 2 +- src/Serval/src/Serval.Client/Client.g.cs | 4127 ++++++++++++++--- .../Controllers/CorporaController.cs | 8 +- .../Services/CorpusService.cs | 2 +- .../Services/ICorpusService.cs | 6 +- .../serval/word_alignment/v1/engine.proto | 104 + .../serval/word_alignment/v1/platform.proto | 66 + .../Contracts/AlignedWordPairDto.cs | 2 +- .../Contracts}/ParallelCorpusFilter.cs | 2 +- .../Contracts/QueueDto.cs | 2 +- .../Contracts/WordAlignmentBuildFinished.cs | 11 + .../WordAlignmentBuildFinishedDto.cs | 10 + .../Contracts/WordAlignmentBuildStarted.cs | 8 + .../Contracts/WordAlignmentBuildStartedDto.cs | 7 + .../Serval.Shared/Controllers/Endpoints.cs | 3 + .../src/Serval.Shared/Controllers/Scopes.cs | 9 + .../Models/AlignedWordPair.cs | 2 +- .../Models/CorpusFile.cs | 2 +- .../Models/MonolingualCorpus.cs | 2 +- .../Models/Queue.cs | 2 +- src/Serval/src/Serval.Shared/Usings.cs | 1 + .../TranslationEnginesController.cs | 1 + .../Services/EngineService.cs | 16 +- ...iatorRegistrationConfiguratorExtensions.cs | 2 + .../WordAlignmentBuildFinishedConsumer.cs | 36 + .../WordAlignmentBuildStartedConsumer.cs | 33 + .../Serval.Webhooks/Contracts/WebhookEvent.cs | 4 +- .../IEndpointRouteBuilderExtensions.cs | 11 + ...iatorRegistrationConfiguratorExtensions.cs | 12 + ...IMemoryDataAccessConfiguratorExtensions.cs | 14 + .../IMongoDataAccessConfiguratorExtensions.cs | 62 + .../Configuration/IServalBuilderExtensions.cs | 48 + .../Configuration/WordAlignmentOptions.cs | 14 + .../Consumers/DataFileDeletedConsumer.cs | 11 + .../ParallelCorpusFilterConfigDto.cs | 8 + .../Contracts/ParallelCorpusFilterDto.cs | 8 + .../Contracts/TrainingCorpusConfigDto.cs | 8 + .../Contracts/TrainingCorpusDto.cs | 8 + .../Contracts/WordAlignmentBuildConfigDto.cs | 15 + .../Contracts/WordAlignmentBuildDto.cs | 30 + .../Contracts/WordAlignmentDto.cs | 11 + .../Contracts/WordAlignmentEngineConfigDto.cs | 24 + .../Contracts/WordAlignmentEngineDto.cs | 15 + .../WordAlignmentParallelCorpusConfigDto.cs | 12 + .../WordAlignmentParallelCorpusDto.cs | 10 + .../WordAlignmentParallelCorpusUpdateDto.cs | 23 + .../Contracts/WordAlignmentRequestDto.cs | 7 + .../Contracts/WordAlignmentResultDto.cs | 9 + .../WordAlignmentEngineTypesController.cs | 45 + .../WordAlignmentEnginesController.cs | 956 ++++ .../src/Serval.WordAlignment/Models/Build.cs | 18 + .../src/Serval.WordAlignment/Models/Engine.cs | 18 + .../Models/ParallelCorpus.cs | 8 + .../Models/TrainingCorpus.cs | 8 + .../Models/WordAlignment.cs | 16 + .../Models/WordAlignmentResult.cs | 9 + .../Serval.WordAlignment.csproj | 27 + .../Services/BuildService.cs | 63 + .../Services/EngineService.cs | 453 ++ .../Services/IBuildService.cs | 18 + .../Services/IEngineService.cs | 39 + .../Services/IWordAlignmentService.cs | 12 + .../WordAlignmentPlatformServiceV1.cs | 331 ++ .../Services/WordAlignmentService.cs | 24 + src/Serval/src/Serval.WordAlignment/using.cs | 30 + .../Services/EngineServiceTests.cs | 120 +- .../Services/PretranslationServiceTests.cs | 8 +- .../test/Serval.Translation.Tests/Usings.cs | 1 + 91 files changed, 6637 insertions(+), 760 deletions(-) rename src/Echo/src/{EchoTranslationEngine => EchoEngine}/BackgroundTaskQueue.cs (97%) rename src/Echo/src/{EchoTranslationEngine => EchoEngine}/BackgroundTaskService.cs (97%) rename src/Echo/src/{EchoTranslationEngine/EchoTranslationEngine.csproj => EchoEngine/EchoEngine.csproj} (100%) rename src/Echo/src/{EchoTranslationEngine => EchoEngine}/HealthServiceV1.cs (94%) rename src/Echo/src/{EchoTranslationEngine => EchoEngine}/Program.cs (56%) rename src/Echo/src/{EchoTranslationEngine => EchoEngine}/Properties/launchSettings.json (90%) rename src/Echo/src/{EchoTranslationEngine => EchoEngine}/TranslationEngineServiceV1.cs (99%) rename src/Echo/src/{EchoTranslationEngine => EchoEngine}/Usings.cs (74%) create mode 100644 src/Echo/src/EchoEngine/WordAlignmentEngineServiceV1.cs rename src/Echo/src/{EchoTranslationEngine => EchoEngine}/appsettings.Development.json (66%) rename src/Echo/src/{EchoTranslationEngine => EchoEngine}/appsettings.Production.json (100%) rename src/Echo/src/{EchoTranslationEngine => EchoEngine}/appsettings.Staging.json (100%) rename src/Echo/src/{EchoTranslationEngine => EchoEngine}/appsettings.json (100%) delete mode 100644 src/Serval/src/Serval.Assessment/Models/CorpusFile.cs create mode 100644 src/Serval/src/Serval.Grpc/Protos/serval/word_alignment/v1/engine.proto create mode 100644 src/Serval/src/Serval.Grpc/Protos/serval/word_alignment/v1/platform.proto rename src/Serval/src/{Serval.Translation => Serval.Shared}/Contracts/AlignedWordPairDto.cs (76%) rename src/Serval/src/{Serval.Translation/Models => Serval.Shared/Contracts}/ParallelCorpusFilter.cs (82%) rename src/Serval/src/{Serval.Translation => Serval.Shared}/Contracts/QueueDto.cs (75%) create mode 100644 src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildFinished.cs create mode 100644 src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildFinishedDto.cs create mode 100644 src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildStarted.cs create mode 100644 src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildStartedDto.cs rename src/Serval/src/{Serval.Translation => Serval.Shared}/Models/AlignedWordPair.cs (77%) rename src/Serval/src/{Serval.Translation => Serval.Shared}/Models/CorpusFile.cs (84%) rename src/Serval/src/{Serval.Translation => Serval.Shared}/Models/MonolingualCorpus.cs (86%) rename src/Serval/src/{Serval.Translation => Serval.Shared}/Models/Queue.cs (76%) create mode 100644 src/Serval/src/Serval.Webhooks/Consumers/WordAlignmentBuildFinishedConsumer.cs create mode 100644 src/Serval/src/Serval.Webhooks/Consumers/WordAlignmentBuildStartedConsumer.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Configuration/IEndpointRouteBuilderExtensions.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Configuration/IMediatorRegistrationConfiguratorExtensions.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Configuration/IMemoryDataAccessConfiguratorExtensions.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Configuration/IMongoDataAccessConfiguratorExtensions.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Configuration/IServalBuilderExtensions.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Configuration/WordAlignmentOptions.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Consumers/DataFileDeletedConsumer.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/ParallelCorpusFilterConfigDto.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/ParallelCorpusFilterDto.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/TrainingCorpusConfigDto.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/TrainingCorpusDto.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentBuildConfigDto.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentBuildDto.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentDto.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentEngineConfigDto.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentEngineDto.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusConfigDto.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusDto.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusUpdateDto.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentRequestDto.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentResultDto.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEngineTypesController.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEnginesController.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Models/Build.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Models/Engine.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Models/ParallelCorpus.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Models/TrainingCorpus.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Models/WordAlignment.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Models/WordAlignmentResult.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Serval.WordAlignment.csproj create mode 100644 src/Serval/src/Serval.WordAlignment/Services/BuildService.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Services/EngineService.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Services/IBuildService.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Services/IEngineService.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Services/IWordAlignmentService.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Services/WordAlignmentPlatformServiceV1.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Services/WordAlignmentService.cs create mode 100644 src/Serval/src/Serval.WordAlignment/using.cs diff --git a/.vscode/launch.json b/.vscode/launch.json index 24d96910..5038bdd1 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -25,7 +25,7 @@ "containerName": "echo_cntr", "request": "attach", "platform": "netCore", - "processName": "EchoTranslationEngine", + "processName": "EchoEngine", "sourceFileMap": { "/app": "${workspaceFolder}" }, @@ -77,9 +77,9 @@ "type": "coreclr", "request": "launch", "preLaunchTask": "build", - "program": "${workspaceFolder}/src/Echo/src/EchoTranslationEngine/bin/Debug/net8.0/EchoTranslationEngine.dll", + "program": "${workspaceFolder}/src/Echo/src/EchoEngine/bin/Debug/net8.0/EchoEngine.dll", "args": [], - "cwd": "${workspaceFolder}/src/Echo/src/EchoTranslationEngine", + "cwd": "${workspaceFolder}/src/Echo/src/EchoEngine", "stopAtEntry": false, "console": "internalConsole", "justMyCode": false, diff --git a/Serval.sln b/Serval.sln index edd3f075..0c7c7808 100644 --- a/Serval.sln +++ b/Serval.sln @@ -36,8 +36,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Translation.Tests", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.DataFiles", "src\Serval\src\Serval.DataFiles\Serval.DataFiles.csproj", "{4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EchoTranslationEngine", "src\Echo\src\EchoTranslationEngine\EchoTranslationEngine.csproj", "{A9D08CA2-3CF7-4BB7-A47F-5A567FFDB2CC}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.ApiServer.IntegrationTests", "src\Serval\test\Serval.ApiServer.IntegrationTests\Serval.ApiServer.IntegrationTests.csproj", "{0C3DF75B-B022-4EFC-882C-F276F1EC8435}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.E2ETests", "src\Serval\test\Serval.E2ETests\Serval.E2ETests.csproj", "{1F020042-D7B8-4541-9691-26ECFD1FFC73}" @@ -48,10 +46,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.DataAccess.Tests", "src EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Shared.Tests", "src\Serval\test\Serval.Shared.Tests\Serval.Shared.Tests.csproj", "{0E220C65-AA88-450E-AFB2-844E49060B3F}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Echo", "Echo", "{53B18D34-B7C7-4B91-BCB0-1021170DFC65}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{BE90915F-E16D-479C-9568-22A08F0FD8F9}" -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Serval", "Serval", "{6D20F76D-9A0E-44AC-8754-B4291C75D25B}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{25CDB05B-4E24-4A6E-933E-1E0BEC97D74D}" @@ -86,6 +80,20 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{C3A14577-A65 EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.ServiceToolkit", "src\ServiceToolkit\src\SIL.ServiceToolkit\SIL.ServiceToolkit.csproj", "{0E40F959-C641-40A2-9750-B17A4F9F9E55}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{9125C013-4F15-4761-BCD2-070524986737}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Serval", "Serval", "{A78D900F-AE52-436C-88CE-A22EAEDECD91}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{C1EF85B4-F2CD-407D-904D-BAC1BCE3A64C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serval.WordAlignment", "src\Serval\src\Serval.WordAlignment\Serval.WordAlignment.csproj", "{F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Echo", "Echo", "{D201886D-9299-4758-80E8-694DBCF8DF93}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{0904BA95-D5BF-4AC2-A919-20A785EF45F5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EchoEngine", "src\Echo\src\EchoEngine\EchoEngine.csproj", "{929FF600-8C7E-4498-A2A3-5534F3A3481E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -132,10 +140,6 @@ Global {4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Debug|Any CPU.Build.0 = Debug|Any CPU {4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Release|Any CPU.ActiveCfg = Release|Any CPU {4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Release|Any CPU.Build.0 = Release|Any CPU - {A9D08CA2-3CF7-4BB7-A47F-5A567FFDB2CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A9D08CA2-3CF7-4BB7-A47F-5A567FFDB2CC}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A9D08CA2-3CF7-4BB7-A47F-5A567FFDB2CC}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A9D08CA2-3CF7-4BB7-A47F-5A567FFDB2CC}.Release|Any CPU.Build.0 = Release|Any CPU {0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Debug|Any CPU.Build.0 = Debug|Any CPU {0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -180,6 +184,14 @@ Global {0E40F959-C641-40A2-9750-B17A4F9F9E55}.Debug|Any CPU.Build.0 = Debug|Any CPU {0E40F959-C641-40A2-9750-B17A4F9F9E55}.Release|Any CPU.ActiveCfg = Release|Any CPU {0E40F959-C641-40A2-9750-B17A4F9F9E55}.Release|Any CPU.Build.0 = Release|Any CPU + {F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Release|Any CPU.Build.0 = Release|Any CPU + {929FF600-8C7E-4498-A2A3-5534F3A3481E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {929FF600-8C7E-4498-A2A3-5534F3A3481E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {929FF600-8C7E-4498-A2A3-5534F3A3481E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {929FF600-8C7E-4498-A2A3-5534F3A3481E}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -195,13 +207,11 @@ Global {8FC30758-37FC-4819-805D-8EFF3DEDF05F} = {3E753B99-7C31-42AC-B02E-012B802F58DB} {539598C5-8634-4273-8714-A684622DDCFC} = {3E753B99-7C31-42AC-B02E-012B802F58DB} {4375A7BF-E3CE-4785-91E3-2ED6FCEB074F} = {25CDB05B-4E24-4A6E-933E-1E0BEC97D74D} - {A9D08CA2-3CF7-4BB7-A47F-5A567FFDB2CC} = {BE90915F-E16D-479C-9568-22A08F0FD8F9} {0C3DF75B-B022-4EFC-882C-F276F1EC8435} = {3E753B99-7C31-42AC-B02E-012B802F58DB} {1F020042-D7B8-4541-9691-26ECFD1FFC73} = {3E753B99-7C31-42AC-B02E-012B802F58DB} {63E4D71B-11BE-4D68-A876-5B1B5F0A4C88} = {3E753B99-7C31-42AC-B02E-012B802F58DB} {71151518-8774-44D0-8E69-D77FA447BEFA} = {BA044B98-3136-4FDE-B90F-B0975758C07F} {0E220C65-AA88-450E-AFB2-844E49060B3F} = {3E753B99-7C31-42AC-B02E-012B802F58DB} - {BE90915F-E16D-479C-9568-22A08F0FD8F9} = {53B18D34-B7C7-4B91-BCB0-1021170DFC65} {25CDB05B-4E24-4A6E-933E-1E0BEC97D74D} = {6D20F76D-9A0E-44AC-8754-B4291C75D25B} {3E753B99-7C31-42AC-B02E-012B802F58DB} = {6D20F76D-9A0E-44AC-8754-B4291C75D25B} {92805246-5285-4F0A-9BF8-6EE4A027A41B} = {33E6965E-5A58-4C6F-882E-F17C8E88A3FF} @@ -215,6 +225,12 @@ Global {10657805-48F1-4205-B8F5-79447F6EF620} = {25CDB05B-4E24-4A6E-933E-1E0BEC97D74D} {C3A14577-A654-4604-818C-4E683DD45A51} = {EA69B41C-49EF-4017-A687-44B9DF37FF98} {0E40F959-C641-40A2-9750-B17A4F9F9E55} = {C3A14577-A654-4604-818C-4E683DD45A51} + {A78D900F-AE52-436C-88CE-A22EAEDECD91} = {9125C013-4F15-4761-BCD2-070524986737} + {C1EF85B4-F2CD-407D-904D-BAC1BCE3A64C} = {A78D900F-AE52-436C-88CE-A22EAEDECD91} + {F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354} = {C1EF85B4-F2CD-407D-904D-BAC1BCE3A64C} + {D201886D-9299-4758-80E8-694DBCF8DF93} = {9125C013-4F15-4761-BCD2-070524986737} + {0904BA95-D5BF-4AC2-A919-20A785EF45F5} = {D201886D-9299-4758-80E8-694DBCF8DF93} + {929FF600-8C7E-4498-A2A3-5534F3A3481E} = {0904BA95-D5BF-4AC2-A919-20A785EF45F5} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9F18C25E-E140-43C3-B177-D562E1628370} diff --git a/deploy/serval/templates/echo-deployment.yaml b/deploy/serval/templates/echo-deployment.yaml index 72d961de..6f9f6b02 100644 --- a/deploy/serval/templates/echo-deployment.yaml +++ b/deploy/serval/templates/echo-deployment.yaml @@ -21,7 +21,7 @@ spec: containers: - command: - dotnet - - /app/echo_server/EchoTranslationEngine.dll + - /app/echo_server/EchoEngine.dll workingDir: /app/echo_server env: - name: ASPNETCORE_ENVIRONMENT diff --git a/docker-compose.withatlas.yml b/docker-compose.withatlas.yml index b23d0be0..0cd4869f 100644 --- a/docker-compose.withatlas.yml +++ b/docker-compose.withatlas.yml @@ -22,6 +22,11 @@ services: - ASPNETCORE_Translation__Engines__1__Address=http://machine-engine - ASPNETCORE_Translation__Engines__2__Type=Nmt - ASPNETCORE_Translation__Engines__2__Address=http://machine-engine + - ASPNETCORE_WordAlignment__Engines__0__Type=EchoWordAlignment + - ASPNETCORE_WordAlignment__Engines__0__Address=http://echo + - ASPNETCORE_WordAlignment__Engines__1__Type=Statistical + - ASPNETCORE_WordAlignment__Engines__1__Address=http://machine-engine + - "Bugsnag__ApiKey=${Bugsnag__ApiKey}" expose: - 80 - 81 @@ -51,6 +56,8 @@ services: - ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80 - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2 - ASPNETCORE_ConnectionStrings__TranslationPlatformApi=http://serval-api:81 + - ASPNETCORE_ConnectionStrings__WordAlignmentPlatformApi=http://serval-api:81 + - "Bugsnag__ApiKey=${Bugsnag__ApiKey}" expose: - 80 ports: @@ -61,7 +68,7 @@ services: - .:/app:ro - ~/.nuget/packages:/root/.nuget/packages:ro - /var/lib/serval:/var/lib/serval - working_dir: '/app/src/Echo/src/EchoTranslationEngine' + working_dir: '/app/src/Echo/src/EchoEngine' entrypoint: - dotnet - run @@ -95,6 +102,7 @@ services: - SharedFile__Uri=s3://silnlp/docker-compose/ - "SharedFile__S3AccessKeyId=${AWS_ACCESS_KEY_ID:?access key needed}" - "SharedFile__S3SecretAccessKey=${AWS_SECRET_ACCESS_KEY:?secret key needed}" + - "Bugsnag__ApiKey=${Bugsnag__ApiKey}" expose: - 80 ports: @@ -139,6 +147,7 @@ services: - SharedFile__Uri=s3://silnlp/docker-compose/ - "SharedFile__S3AccessKeyId=${AWS_ACCESS_KEY_ID:?access key needed}" - "SharedFile__S3SecretAccessKey=${AWS_SECRET_ACCESS_KEY:?secret key needed}" + - "Bugsnag__ApiKey=${Bugsnag__ApiKey}" expose: - 80 ports: diff --git a/docker-compose.yml b/docker-compose.yml index 6e568f99..93db1add 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,10 @@ services: - ASPNETCORE_Translation__Engines__1__Address=http://machine-engine - ASPNETCORE_Translation__Engines__2__Type=Nmt - ASPNETCORE_Translation__Engines__2__Address=http://machine-engine + - ASPNETCORE_WordAlignment__Engines__0__Type=EchoWordAlignment + - ASPNETCORE_WordAlignment__Engines__0__Address=http://echo + - ASPNETCORE_WordAlignment__Engines__1__Type=Statistical + - ASPNETCORE_WordAlignment__Engines__1__Address=http://machine-engine - "Bugsnag__ApiKey=${Bugsnag__ApiKey}" expose: - 80 @@ -53,6 +57,7 @@ services: - ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80 - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2 - ASPNETCORE_ConnectionStrings__TranslationPlatformApi=http://serval-api:81 + - ASPNETCORE_ConnectionStrings__WordAlignmentPlatformApi=http://serval-api:81 - "Bugsnag__ApiKey=${Bugsnag__ApiKey}" expose: - 80 @@ -64,7 +69,7 @@ services: - .:/app:ro - ~/.nuget/packages:/root/.nuget/packages:ro - /var/lib/serval:/var/lib/serval - working_dir: '/app/src/Echo/src/EchoTranslationEngine' + working_dir: '/app/src/Echo/src/EchoEngine' entrypoint: - dotnet - run diff --git a/dockerfile b/dockerfile index b58f3c21..4c8a78a3 100644 --- a/dockerfile +++ b/dockerfile @@ -9,7 +9,7 @@ COPY . ./ RUN dotnet restore # Build and publish a release RUN dotnet publish ./src/Serval/src/Serval.ApiServer/Serval.ApiServer.csproj -c Release -o out_api_server -RUN dotnet publish ./src/Echo/src/EchoTranslationEngine/EchoTranslationEngine.csproj -c Release -o out_echo_server +RUN dotnet publish ./src/Echo/src/EchoEngine/EchoEngine.csproj -c Release -o out_echo_server RUN dotnet publish ./src/Machine/src/Serval.Machine.EngineServer/Serval.Machine.EngineServer.csproj -c Release -o out_machine_engine_server RUN dotnet publish ./src/Machine/src/Serval.Machine.JobServer/Serval.Machine.JobServer.csproj -c Release -o out_machine_job_server diff --git a/src/Echo/src/EchoTranslationEngine/BackgroundTaskQueue.cs b/src/Echo/src/EchoEngine/BackgroundTaskQueue.cs similarity index 97% rename from src/Echo/src/EchoTranslationEngine/BackgroundTaskQueue.cs rename to src/Echo/src/EchoEngine/BackgroundTaskQueue.cs index 6ee3fd4e..03ef178f 100644 --- a/src/Echo/src/EchoTranslationEngine/BackgroundTaskQueue.cs +++ b/src/Echo/src/EchoEngine/BackgroundTaskQueue.cs @@ -1,4 +1,4 @@ -namespace EchoTranslationEngine; +namespace EchoEngine; public class BackgroundTaskQueue { diff --git a/src/Echo/src/EchoTranslationEngine/BackgroundTaskService.cs b/src/Echo/src/EchoEngine/BackgroundTaskService.cs similarity index 97% rename from src/Echo/src/EchoTranslationEngine/BackgroundTaskService.cs rename to src/Echo/src/EchoEngine/BackgroundTaskService.cs index cf901427..001895f0 100644 --- a/src/Echo/src/EchoTranslationEngine/BackgroundTaskService.cs +++ b/src/Echo/src/EchoEngine/BackgroundTaskService.cs @@ -1,4 +1,4 @@ -namespace EchoTranslationEngine; +namespace EchoEngine; public class BackgroundTaskService( BackgroundTaskQueue taskQueue, diff --git a/src/Echo/src/EchoTranslationEngine/EchoTranslationEngine.csproj b/src/Echo/src/EchoEngine/EchoEngine.csproj similarity index 100% rename from src/Echo/src/EchoTranslationEngine/EchoTranslationEngine.csproj rename to src/Echo/src/EchoEngine/EchoEngine.csproj diff --git a/src/Echo/src/EchoTranslationEngine/HealthServiceV1.cs b/src/Echo/src/EchoEngine/HealthServiceV1.cs similarity index 94% rename from src/Echo/src/EchoTranslationEngine/HealthServiceV1.cs rename to src/Echo/src/EchoEngine/HealthServiceV1.cs index 05bc98c1..025b2fee 100644 --- a/src/Echo/src/EchoTranslationEngine/HealthServiceV1.cs +++ b/src/Echo/src/EchoEngine/HealthServiceV1.cs @@ -1,6 +1,6 @@ using Serval.Health.V1; -namespace EchoTranslationEngine; +namespace EchoEngine; public class HealthServiceV1(HealthCheckService healthCheckService) : HealthApi.HealthApiBase { diff --git a/src/Echo/src/EchoTranslationEngine/Program.cs b/src/Echo/src/EchoEngine/Program.cs similarity index 56% rename from src/Echo/src/EchoTranslationEngine/Program.cs rename to src/Echo/src/EchoEngine/Program.cs index a679dfb5..a0ac947d 100644 --- a/src/Echo/src/EchoTranslationEngine/Program.cs +++ b/src/Echo/src/EchoEngine/Program.cs @@ -1,10 +1,24 @@ +using Serval.Translation.V1; +using Serval.WordAlignment.V1; + WebApplicationBuilder builder = WebApplication.CreateBuilder(args); // Add services to the container. -builder.Services.AddGrpcClient(o => -{ - o.Address = new Uri(builder.Configuration.GetConnectionString("TranslationPlatformApi")!); -}); +builder.Services.AddGrpcClient( + "Translation", + o => + { + o.Address = new Uri(builder.Configuration.GetConnectionString("TranslationPlatformApi")!); + } +); +builder.Services.AddGrpcClient( + "WordAlignment", + o => + { + o.Address = new Uri(builder.Configuration.GetConnectionString("WordAlignmentPlatformApi")!); + } +); + builder.Services.AddGrpc(); builder.Services.AddHostedService(); @@ -18,6 +32,8 @@ WebApplication app = builder.Build(); app.MapGrpcService(); +app.MapGrpcService(); + app.MapGrpcService(); app.Run(); diff --git a/src/Echo/src/EchoTranslationEngine/Properties/launchSettings.json b/src/Echo/src/EchoEngine/Properties/launchSettings.json similarity index 90% rename from src/Echo/src/EchoTranslationEngine/Properties/launchSettings.json rename to src/Echo/src/EchoEngine/Properties/launchSettings.json index 961e424a..3e0605d4 100644 --- a/src/Echo/src/EchoTranslationEngine/Properties/launchSettings.json +++ b/src/Echo/src/EchoEngine/Properties/launchSettings.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/launchsettings.json", "profiles": { - "EchoTranslationEngine": { + "EchoEngine": { "commandName": "Project", "dotnetRunMessages": true, "launchBrowser": false, @@ -11,4 +11,4 @@ } } } -} +} \ No newline at end of file diff --git a/src/Echo/src/EchoTranslationEngine/TranslationEngineServiceV1.cs b/src/Echo/src/EchoEngine/TranslationEngineServiceV1.cs similarity index 99% rename from src/Echo/src/EchoTranslationEngine/TranslationEngineServiceV1.cs rename to src/Echo/src/EchoEngine/TranslationEngineServiceV1.cs index 254fe0af..bd0f7771 100644 --- a/src/Echo/src/EchoTranslationEngine/TranslationEngineServiceV1.cs +++ b/src/Echo/src/EchoEngine/TranslationEngineServiceV1.cs @@ -1,4 +1,6 @@ -namespace EchoTranslationEngine; +using Serval.Translation.V1; + +namespace EchoEngine; public class TranslationEngineServiceV1(BackgroundTaskQueue taskQueue) : TranslationEngineApi.TranslationEngineApiBase { diff --git a/src/Echo/src/EchoTranslationEngine/Usings.cs b/src/Echo/src/EchoEngine/Usings.cs similarity index 74% rename from src/Echo/src/EchoTranslationEngine/Usings.cs rename to src/Echo/src/EchoEngine/Usings.cs index b7f3ba2d..059e98d4 100644 --- a/src/Echo/src/EchoTranslationEngine/Usings.cs +++ b/src/Echo/src/EchoEngine/Usings.cs @@ -1,7 +1,6 @@ global using System.Threading.Channels; global using Bugsnag.AspNet.Core; -global using EchoTranslationEngine; +global using EchoEngine; global using Google.Protobuf.WellKnownTypes; global using Grpc.Core; global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Serval.Translation.V1; diff --git a/src/Echo/src/EchoEngine/WordAlignmentEngineServiceV1.cs b/src/Echo/src/EchoEngine/WordAlignmentEngineServiceV1.cs new file mode 100644 index 00000000..461938ff --- /dev/null +++ b/src/Echo/src/EchoEngine/WordAlignmentEngineServiceV1.cs @@ -0,0 +1,221 @@ +using Serval.WordAlignment.V1; + +namespace EchoEngine; + +public class WordAlignmentEngineServiceV1(BackgroundTaskQueue taskQueue) + : WordAlignmentEngineApi.WordAlignmentEngineApiBase +{ + private static readonly Empty Empty = new(); + private readonly BackgroundTaskQueue _taskQueue = taskQueue; + + public override Task Create(CreateRequest request, ServerCallContext context) + { + if (request.SourceLanguage != request.TargetLanguage) + { + Status status = new Status(StatusCode.InvalidArgument, "Source and target languages must be the same"); + throw new RpcException(status); + } + return Task.FromResult(Empty); + } + + public override Task Delete(DeleteRequest request, ServerCallContext context) + { + return Task.FromResult(Empty); + } + + public static IEnumerable GenerateAlignedWordPairs(int number) + { + if (number < 0) + { + throw new ArgumentOutOfRangeException(nameof(number), "Number must be non-negative"); + } + return Enumerable.Range(0, number).Select(i => new AlignedWordPair { SourceIndex = i, TargetIndex = i }); + } + + public override Task GetWordAlignment( + GetWordAlignmentRequest request, + ServerCallContext context + ) + { + string[] sourceTokens = request.SourceSegment.Split(); + string[] targetTokens = request.TargetSegment.Split(); + int minLength = Math.Min(sourceTokens.Length, targetTokens.Length); + + var response = new GetWordAlignmentResponse + { + Result = new WordAlignmentResult + { + SourceTokens = { sourceTokens }, + TargetTokens = { targetTokens }, + Confidences = { Enumerable.Repeat(1.0, minLength) }, + Alignment = { GenerateAlignedWordPairs(minLength) } + } + }; + return Task.FromResult(response); + } + + public override async Task StartBuild(StartBuildRequest request, ServerCallContext context) + { + await _taskQueue.QueueBackgroundWorkItemAsync( + async (services, cancellationToken) => + { + WordAlignmentPlatformApi.WordAlignmentPlatformApiClient client = + services.GetRequiredService(); + await client.BuildStartedAsync( + new BuildStartedRequest { BuildId = request.BuildId }, + cancellationToken: cancellationToken + ); + + try + { + using ( + AsyncClientStreamingCall call = client.InsertWordAlignments( + cancellationToken: cancellationToken + ) + ) + { + foreach (ParallelCorpus corpus in request.Corpora) + { + var sourceFiles = corpus + .SourceCorpora.SelectMany(sc => + sc.Files.Where(f => + ( + sc.WordAlignOnAll + || sc.WordAlignOnTextIds is null + || sc.WordAlignOnTextIds.Contains(f.TextId) + ) + && f.Format == FileFormat.Text + ) + ) + .ToDictionary(f => f.TextId, f => f.Location); + var targetFiles = corpus + .TargetCorpora.SelectMany(tc => + tc.Files.Where(f => + ( + tc.WordAlignOnAll + || tc.WordAlignOnTextIds is null + || tc.WordAlignOnTextIds.Contains(f.TextId) + ) + && f.Format == FileFormat.Text + ) + ) + .ToDictionary(f => f.TextId, f => f.Location); + + foreach (KeyValuePair sourceFile in sourceFiles) + { + string[] sourceLines = await File.ReadAllLinesAsync( + sourceFile.Value, + cancellationToken + ); + + if (targetFiles.TryGetValue(sourceFile.Key, out string? targetPath)) + { + string[] targetLines = await File.ReadAllLinesAsync(targetPath, cancellationToken); + bool isTabSeparated = (sourceLines.Length > 0) && sourceLines[0].Contains('/'); + if (!isTabSeparated) + { + int lineNum = 1; + foreach ( + (string sourceLine, string targetLine) in sourceLines + .Select(l => l.Trim()) + .Zip(targetLines.Select(l => l.Trim())) + ) + { + if (sourceLine.Length > 0 && targetLine.Length == 0) + { + int minLength = Math.Min( + sourceLine.Split().Length, + targetLine.Split().Length + ); + await call.RequestStream.WriteAsync( + new InsertWordAlignmentsRequest + { + EngineId = request.EngineId, + CorpusId = corpus.Id, + TextId = sourceFile.Key, + Refs = { $"{sourceFile.Key}:{lineNum}" }, + SourceTokens = { sourceLine.Split() }, + TargetTokens = { targetLine.Split() }, + Confidences = { Enumerable.Repeat(1.0, minLength) }, + Alignment = { GenerateAlignedWordPairs(minLength) } + }, + cancellationToken + ); + } + lineNum++; + } + } + else + { + var sourceLinesDict = sourceLines.ToDictionary( + l => l.Split('\t')[0].Trim(), + l => l.Split('\t')[1].Trim() + ); + var targetLinesDict = targetLines.ToDictionary( + l => l.Split('\t')[0].Trim(), + l => l.Contains('\t') ? l.Split('\t')[1].Trim() : string.Empty + ); + foreach (KeyValuePair targetLineKVPair in targetLinesDict) + { + string? sourceLine = null; + sourceLinesDict.TryGetValue(targetLineKVPair.Key, out sourceLine); + sourceLine ??= string.Empty; + string? targetLine = targetLineKVPair.Value; + if (sourceLine.Length > 0 && targetLine.Length == 0) + { + int minLength = Math.Min( + sourceLine.Split().Length, + targetLine.Split().Length + ); + await call.RequestStream.WriteAsync( + new InsertWordAlignmentsRequest + { + EngineId = request.EngineId, + CorpusId = corpus.Id, + TextId = sourceFile.Key, + Refs = { $"{sourceFile.Key}:{targetLineKVPair.Key}" }, + SourceTokens = { sourceLine.Split() }, + TargetTokens = { targetLine.Split() }, + Confidences = + { + Enumerable.Repeat(1.0, sourceLine.Split().Length) + }, + Alignment = { GenerateAlignedWordPairs(minLength) } + }, + cancellationToken + ); + } + } + } + } + } + } + await call.RequestStream.CompleteAsync(); + await call; + } + + await client.BuildCompletedAsync( + new BuildCompletedRequest { BuildId = request.BuildId, Confidence = 1.0 }, + cancellationToken: CancellationToken.None + ); + } + catch (OperationCanceledException) + { + await client.BuildCanceledAsync( + new BuildCanceledRequest { BuildId = request.BuildId }, + cancellationToken: CancellationToken.None + ); + } + catch (Exception e) + { + await client.BuildFaultedAsync( + new BuildFaultedRequest { BuildId = request.BuildId, Message = e.Message }, + cancellationToken: CancellationToken.None + ); + } + } + ); + + return Empty; + } +} diff --git a/src/Echo/src/EchoTranslationEngine/appsettings.Development.json b/src/Echo/src/EchoEngine/appsettings.Development.json similarity index 66% rename from src/Echo/src/EchoTranslationEngine/appsettings.Development.json rename to src/Echo/src/EchoEngine/appsettings.Development.json index 838bc6f4..25895bf0 100644 --- a/src/Echo/src/EchoTranslationEngine/appsettings.Development.json +++ b/src/Echo/src/EchoEngine/appsettings.Development.json @@ -5,7 +5,8 @@ } }, "ConnectionStrings": { - "TranslationPlatformApi": "http://localhost:8080" + "TranslationPlatformApi": "http://localhost:8080", + "WordAlignmentPlatformApi": "http://localhost:8081" }, "Logging": { "LogLevel": { @@ -13,4 +14,4 @@ "Microsoft.AspNetCore": "Warning" } } -} +} \ No newline at end of file diff --git a/src/Echo/src/EchoTranslationEngine/appsettings.Production.json b/src/Echo/src/EchoEngine/appsettings.Production.json similarity index 100% rename from src/Echo/src/EchoTranslationEngine/appsettings.Production.json rename to src/Echo/src/EchoEngine/appsettings.Production.json diff --git a/src/Echo/src/EchoTranslationEngine/appsettings.Staging.json b/src/Echo/src/EchoEngine/appsettings.Staging.json similarity index 100% rename from src/Echo/src/EchoTranslationEngine/appsettings.Staging.json rename to src/Echo/src/EchoEngine/appsettings.Staging.json diff --git a/src/Echo/src/EchoTranslationEngine/appsettings.json b/src/Echo/src/EchoEngine/appsettings.json similarity index 100% rename from src/Echo/src/EchoTranslationEngine/appsettings.json rename to src/Echo/src/EchoEngine/appsettings.json diff --git a/src/Serval/src/Serval.ApiServer/Serval.ApiServer.csproj b/src/Serval/src/Serval.ApiServer/Serval.ApiServer.csproj index 3ea7d09f..d2268351 100644 --- a/src/Serval/src/Serval.ApiServer/Serval.ApiServer.csproj +++ b/src/Serval/src/Serval.ApiServer/Serval.ApiServer.csproj @@ -47,6 +47,7 @@ + diff --git a/src/Serval/src/Serval.ApiServer/Startup.cs b/src/Serval/src/Serval.ApiServer/Startup.cs index d4c5a3cd..d3850926 100644 --- a/src/Serval/src/Serval.ApiServer/Startup.cs +++ b/src/Serval/src/Serval.ApiServer/Startup.cs @@ -80,11 +80,13 @@ public void ConfigureServices(IServiceCollection services) { cfg.AddTranslationRepositories(); cfg.AddAssessmentRepositories(); + cfg.AddWordAlignmentRepositories(); cfg.AddDataFilesRepositories(); cfg.AddWebhooksRepositories(); }) .AddTranslation() .AddAssessment() + .AddWordAlignment() .AddDataFiles() .AddWebhooks(); services.AddTransient(); @@ -113,6 +115,7 @@ public void ConfigureServices(IServiceCollection services) { cfg.AddTranslationConsumers(); cfg.AddAssessmentConsumers(); + cfg.AddWordAlignmentConsumers(); cfg.AddDataFilesConsumers(); cfg.AddWebhooksConsumers(); }); @@ -232,6 +235,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) x.MapControllers(); x.MapServalTranslationServices(); x.MapServalAssessmentServices(); + x.MapServalWordAlignmentServices(); x.MapHangfireDashboard(); }); diff --git a/src/Serval/src/Serval.ApiServer/appsettings.Development.json b/src/Serval/src/Serval.ApiServer/appsettings.Development.json index 8a5d5cd6..7dd95b08 100644 --- a/src/Serval/src/Serval.ApiServer/appsettings.Development.json +++ b/src/Serval/src/Serval.ApiServer/appsettings.Development.json @@ -27,10 +27,22 @@ } ] }, + "WordAlignment": { + "Engines": [ + { + "Type": "EchoWordAlignment", + "Address": "http://localhost:8055" + }, + { + "Type": "Statistical", + "Address": "http://localhost:9000" + } + ] + }, "Logging": { "LogLevel": { "Default": "Information", "Microsoft.AspNetCore": "Warning" } } -} +} \ No newline at end of file diff --git a/src/Serval/src/Serval.Assessment/Models/CorpusFile.cs b/src/Serval/src/Serval.Assessment/Models/CorpusFile.cs deleted file mode 100644 index fa491558..00000000 --- a/src/Serval/src/Serval.Assessment/Models/CorpusFile.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Serval.Assessment.Models; - -public record CorpusFile -{ - public required string Id { get; init; } - public required string Filename { get; init; } - public required FileFormat Format { get; init; } - public required string TextId { get; init; } -} diff --git a/src/Serval/src/Serval.Assessment/Services/EngineService.cs b/src/Serval/src/Serval.Assessment/Services/EngineService.cs index 01ced93a..abe501c5 100644 --- a/src/Serval/src/Serval.Assessment/Services/EngineService.cs +++ b/src/Serval/src/Serval.Assessment/Services/EngineService.cs @@ -227,7 +227,7 @@ private V1.Corpus Map(Models.Corpus source) return new V1.Corpus { Language = source.Language, Files = { source.Files.Select(Map) } }; } - private V1.CorpusFile Map(Models.CorpusFile source) + private V1.CorpusFile Map(Shared.Models.CorpusFile source) { return new V1.CorpusFile { diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index b10b41c7..1635e9f8 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -4462,6 +4462,7 @@ public partial interface ITranslationEnginesClient ///
///
The `"options"` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. ///
See [nmt job settings documentation](https://github.com/sillsdev/serval/wiki/NMT-Build-Options) about configuring job parameters. + ///
See [smt-transfer job settings documentation](https://github.com/sillsdev/serval/wiki/SMT-Transfer-Build-Options) about configuring job parameters. ///
See [keyterms parsing documentation](https://github.com/sillsdev/serval/wiki/Paratext-Key-Terms-Parsing) on how to use keyterms for training. ///
///
When using a parallel corpus: @@ -7253,6 +7254,7 @@ public string BaseUrl ///
///
The `"options"` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. ///
See [nmt job settings documentation](https://github.com/sillsdev/serval/wiki/NMT-Build-Options) about configuring job parameters. + ///
See [smt-transfer job settings documentation](https://github.com/sillsdev/serval/wiki/SMT-Transfer-Build-Options) about configuring job parameters. ///
See [keyterms parsing documentation](https://github.com/sillsdev/serval/wiki/Paratext-Key-Terms-Parsing) on how to use keyterms for training. ///
///
When using a parallel corpus: @@ -8955,60 +8957,3174 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c } } + [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial interface IWordAlignmentEnginesClient + { + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get all word alignment engines + /// + /// The engines + /// A server side error occurred. + System.Threading.Tasks.Task> GetAllAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Create a new word alignment engine + /// + /// + /// ## Parameters + ///
* **name**: (optional) A name to help identify and distinguish the file. + ///
* Recommendation: Create a multi-part name to distinguish between projects, uses, etc. + ///
* The name does not have to be unique, as the engine is uniquely identified by the auto-generated id + ///
* **sourceLanguage**: The source language code (a valid [IETF language tag](https://en.wikipedia.org/wiki/IETF_language_tag) is recommended) + ///
* **targetLanguage**: The target language code (a valid IETF language tag is recommended) + ///
* **type**: **statistical** or **echo-word-alignment** + ///
### statistical + ///
The Statistical engine is based off of the [Thot library](https://github.com/sillsdev/thot) and contains IBM-1, IBM-2, IBM-3, IBM-4, FastAlign and HMM algorithms. + ///
### echo-word-alignment + ///
The echo-word-alignment engine has full coverage of all endpoints. Endpoints like create and build return empty responses. + ///
Endpoints like get-word-alignment echo the sent content back to the user in the proper format. This engine is useful for debugging and testing purposes. + ///
## Sample request: + ///
+ ///
{ + ///
"name": "myTeam:myProject:myEngine", + ///
"sourceLanguage": "el", + ///
"targetLanguage": "en", + ///
"type": "statistical" + ///
} + /// + /// The engine configuration (see above) + /// The new engine + /// A server side error occurred. + System.Threading.Tasks.Task CreateAsync(WordAlignmentEngineConfig engineConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get a word alignment engine by unique id + /// + /// The engine id + /// The engine + /// A server side error occurred. + System.Threading.Tasks.Task GetAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Delete a word alignment engine + /// + /// The engine id + /// The engine was successfully deleted. + /// A server side error occurred. + System.Threading.Tasks.Task DeleteAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Align words between a source and target segment + /// + /// The engine id + /// The source and target segment + /// The word alignment result + /// A server side error occurred. + System.Threading.Tasks.Task GetWordAlignmentAsync(string id, WordAlignmentRequest wordAlignmentRequest, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Add a parallel corpus to an engine + /// + /// + /// ## Parameters + ///
* **SourceCorpusIds**: The source corpora associated with the parallel corpus + ///
* **TargetCorpusIds**: The target corpora associated with the parallel corpus + /// + /// The engine id + /// The corpus configuration (see remarks) + /// The added corpus + /// A server side error occurred. + System.Threading.Tasks.Task AddParallelCorpusAsync(string id, WordAlignmentParallelCorpusConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get all parallel corpora for a engine + /// + /// The engine id + /// The parallel corpora + /// A server side error occurred. + System.Threading.Tasks.Task> GetAllParallelCorporaAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Update a parallel corpus with a new set of corpora + /// + /// + /// Will completely replace the parallel corpus' file associations. Will not affect jobs already queued or running. Will not affect existing word graphs until new build is complete. + /// + /// The engine id + /// The parallel corpus id + /// The corpus configuration + /// The corpus was updated successfully + /// A server side error occurred. + System.Threading.Tasks.Task UpdateParallelCorpusAsync(string id, string parallelCorpusId, WordAlignmentParallelCorpusUpdateConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get the configuration of a parallel corpus for a engine + /// + /// The engine id + /// The parallel corpus id + /// The parallel corpus configuration + /// A server side error occurred. + System.Threading.Tasks.Task GetParallelCorpusAsync(string id, string parallelCorpusId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Remove a parallel corpus from a engine + /// + /// + /// Removing a parallel corpus will remove all word alignments associated with that corpus. + /// + /// The engine id + /// The parallel corpus id + /// The parallel corpus was deleted successfully. + /// A server side error occurred. + System.Threading.Tasks.Task DeleteParallelCorpusAsync(string id, string parallelCorpusId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get all word alignments in a corpus of a engine + /// + /// + /// Word alignments are arranged in a list of dictionaries with the following fields per word alignment: + ///
* **TextId**: The TextId of the SourceFile defined when the corpus was created. + ///
* **Refs** (a list of strings): A list of references including: + ///
* The references defined in the SourceFile per line, if any. + ///
* An auto-generated reference of `[TextId]:[lineNumber]`, 1 indexed. + ///
* **SourceTokens**: the tokenized source segment + ///
* **TargetTokens**: the tokenized target segment + ///
* **Confidences**: the confidence of the alignment ona scale from 0 to 1 + ///
* **Alignment**: the word alignment, 0 indexed for source and target positions + ///
+ ///
Word alignments can be filtered by text id if provided. + ///
Only word alignments for the most recent successful build of the engine are returned. + /// + /// The engine id + /// The corpus id + /// The text id (optional) + /// The word alignments + /// A server side error occurred. + System.Threading.Tasks.Task> GetAllWordAlignmentsAsync(string id, string corpusId, string? textId = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get all build jobs for a engine + /// + /// The engine id + /// The build jobs + /// A server side error occurred. + System.Threading.Tasks.Task> GetAllBuildsAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Starts a build job for a engine. + /// + /// + /// Specify the corpora and textIds to train on. If no "trainOn" field is provided, all corpora will be used. + ///
Paratext Projects, you may flag a subset of books for training by including their [abbreviations] + ///
Paratext projects can be filtered by [book](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs) using the textId for training. + ///
Filters can also be supplied via scriptureRange parameter as ranges of biblical text. See [here](https://github.com/sillsdev/serval/wiki/Filtering-Paratext-Project-Data-with-a-Scripture-Range) + ///
All Paratext project filtering follows original versification. See [here](https://github.com/sillsdev/serval/wiki/Versification-in-Serval) for more information. + ///
+ ///
Specify the corpora or textIds to word align on. + ///
When a corpus or textId is selected for word align on, only text segments that are in both the source and the target will be aligned. + ///
+ ///
The `"options"` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. + ///
See [statistical alignment job settings documentation](https://github.com/sillsdev/serval/wiki/Statistical-Alignment-Build-Options) about configuring job parameters. + ///
See [keyterms parsing documentation](https://github.com/sillsdev/serval/wiki/Paratext-Key-Terms-Parsing) on how to use keyterms for training. + /// + /// The engine id + /// The build config (see remarks) + /// The new build job + /// A server side error occurred. + System.Threading.Tasks.Task StartBuildAsync(string id, WordAlignmentBuildConfig buildConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get a build job + /// + /// + /// If the `minRevision` is not defined, the current build, at whatever state it is, + ///
will be immediately returned. If `minRevision` is defined, Serval will wait for + ///
up to 40 seconds for the engine to build to the `minRevision` specified, else + ///
will timeout. + ///
A use case is to actively query the state of the current build, where the subsequent + ///
request sets the `minRevision` to the returned `revision` + 1 and timeouts are handled gracefully. + ///
This method should use request throttling. + ///
Note: Within the returned build, percentCompleted is a value between 0 and 1. + /// + /// The engine id + /// The build job id + /// The minimum revision + /// The build job + /// A server side error occurred. + System.Threading.Tasks.Task GetBuildAsync(string id, string buildId, long? minRevision = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get the currently running build job for a engine + /// + /// + /// See documentation on endpoint /word-alignment/engines/{id}/builds/{id} - "Get a Build Job" for details on using `minRevision`. + /// + /// The engine id + /// The minimum revision + /// The build job + /// A server side error occurred. + System.Threading.Tasks.Task GetCurrentBuildAsync(string id, long? minRevision = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Cancel the current build job (whether pending or active) for a engine + /// + /// The engine id + /// The build job was cancelled successfully. + /// A server side error occurred. + System.Threading.Tasks.Task CancelBuildAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + } + + [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class WordAlignmentEnginesClient : IWordAlignmentEnginesClient + { + #pragma warning disable 8618 + private string _baseUrl; + #pragma warning restore 8618 + + private System.Net.Http.HttpClient _httpClient; + private static System.Lazy _settings = new System.Lazy(CreateSerializerSettings, true); + private Newtonsoft.Json.JsonSerializerSettings _instanceSettings; + + #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. + public WordAlignmentEnginesClient(System.Net.Http.HttpClient httpClient) + #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. + { + BaseUrl = "/api/v1"; + _httpClient = httpClient; + Initialize(); + } + + private static Newtonsoft.Json.JsonSerializerSettings CreateSerializerSettings() + { + var settings = new Newtonsoft.Json.JsonSerializerSettings(); + UpdateJsonSerializerSettings(settings); + return settings; + } + + public string BaseUrl + { + get { return _baseUrl; } + set + { + _baseUrl = value; + if (!string.IsNullOrEmpty(_baseUrl) && !_baseUrl.EndsWith("/")) + _baseUrl += '/'; + } + } + + protected Newtonsoft.Json.JsonSerializerSettings JsonSerializerSettings { get { return _instanceSettings ?? _settings.Value; } } + + static partial void UpdateJsonSerializerSettings(Newtonsoft.Json.JsonSerializerSettings settings); + + partial void Initialize(); + + partial void PrepareRequest(System.Net.Http.HttpClient client, System.Net.Http.HttpRequestMessage request, string url); + partial void PrepareRequest(System.Net.Http.HttpClient client, System.Net.Http.HttpRequestMessage request, System.Text.StringBuilder urlBuilder); + partial void ProcessResponse(System.Net.Http.HttpClient client, System.Net.Http.HttpResponseMessage response); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get all word alignment engines + /// + /// The engines + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task> GetAllAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); + // Operation Path: "word-alignment/engines" + urlBuilder_.Append("word-alignment/engines"); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Create a new word alignment engine + /// + /// + /// ## Parameters + ///
* **name**: (optional) A name to help identify and distinguish the file. + ///
* Recommendation: Create a multi-part name to distinguish between projects, uses, etc. + ///
* The name does not have to be unique, as the engine is uniquely identified by the auto-generated id + ///
* **sourceLanguage**: The source language code (a valid [IETF language tag](https://en.wikipedia.org/wiki/IETF_language_tag) is recommended) + ///
* **targetLanguage**: The target language code (a valid IETF language tag is recommended) + ///
* **type**: **statistical** or **echo-word-alignment** + ///
### statistical + ///
The Statistical engine is based off of the [Thot library](https://github.com/sillsdev/thot) and contains IBM-1, IBM-2, IBM-3, IBM-4, FastAlign and HMM algorithms. + ///
### echo-word-alignment + ///
The echo-word-alignment engine has full coverage of all endpoints. Endpoints like create and build return empty responses. + ///
Endpoints like get-word-alignment echo the sent content back to the user in the proper format. This engine is useful for debugging and testing purposes. + ///
## Sample request: + ///
+ ///
{ + ///
"name": "myTeam:myProject:myEngine", + ///
"sourceLanguage": "el", + ///
"targetLanguage": "en", + ///
"type": "statistical" + ///
} + /// + /// The engine configuration (see above) + /// The new engine + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task CreateAsync(WordAlignmentEngineConfig engineConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (engineConfig == null) + throw new System.ArgumentNullException("engineConfig"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(engineConfig, JsonSerializerSettings); + var content_ = new System.Net.Http.StringContent(json_); + content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); + request_.Content = content_; + request_.Method = new System.Net.Http.HttpMethod("POST"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); + // Operation Path: "word-alignment/engines" + urlBuilder_.Append("word-alignment/engines"); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 201) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 400) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("Bad request. Is the engine type correct?", status_, responseText_, headers_, null); + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get a word alignment engine by unique id + /// + /// The engine id + /// The engine + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task GetAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); + // Operation Path: "word-alignment/engines/{id}" + urlBuilder_.Append("word-alignment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Delete a word alignment engine + /// + /// The engine id + /// The engine was successfully deleted. + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task DeleteAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("DELETE"); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); + // Operation Path: "word-alignment/engines/{id}" + urlBuilder_.Append("word-alignment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + return; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine does not exist and therefore cannot be deleted.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Align words between a source and target segment + /// + /// The engine id + /// The source and target segment + /// The word alignment result + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task GetWordAlignmentAsync(string id, WordAlignmentRequest wordAlignmentRequest, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + if (wordAlignmentRequest == null) + throw new System.ArgumentNullException("wordAlignmentRequest"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(wordAlignmentRequest, JsonSerializerSettings); + var content_ = new System.Net.Http.StringContent(json_); + content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); + request_.Content = content_; + request_.Method = new System.Net.Http.HttpMethod("POST"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); + // Operation Path: "word-alignment/engines/{id}/get-word-alignment" + urlBuilder_.Append("word-alignment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/get-word-alignment"); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 400) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("Bad request", status_, responseText_, headers_, null); + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 405) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The method is not supported.", status_, responseText_, headers_, null); + } + else + if (status_ == 409) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine needs to be built before it can alignment segments.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Add a parallel corpus to an engine + /// + /// + /// ## Parameters + ///
* **SourceCorpusIds**: The source corpora associated with the parallel corpus + ///
* **TargetCorpusIds**: The target corpora associated with the parallel corpus + /// + /// The engine id + /// The corpus configuration (see remarks) + /// The added corpus + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task AddParallelCorpusAsync(string id, WordAlignmentParallelCorpusConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + if (corpusConfig == null) + throw new System.ArgumentNullException("corpusConfig"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(corpusConfig, JsonSerializerSettings); + var content_ = new System.Net.Http.StringContent(json_); + content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); + request_.Content = content_; + request_.Method = new System.Net.Http.HttpMethod("POST"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); + // Operation Path: "word-alignment/engines/{id}/parallel-corpora" + urlBuilder_.Append("word-alignment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/parallel-corpora"); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 201) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 400) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("Bad request", status_, responseText_, headers_, null); + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get all parallel corpora for a engine + /// + /// The engine id + /// The parallel corpora + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task> GetAllParallelCorporaAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); + // Operation Path: "word-alignment/engines/{id}/parallel-corpora" + urlBuilder_.Append("word-alignment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/parallel-corpora"); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the engine", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine does not exist", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Update a parallel corpus with a new set of corpora + /// + /// + /// Will completely replace the parallel corpus' file associations. Will not affect jobs already queued or running. Will not affect existing word graphs until new build is complete. + /// + /// The engine id + /// The parallel corpus id + /// The corpus configuration + /// The corpus was updated successfully + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task UpdateParallelCorpusAsync(string id, string parallelCorpusId, WordAlignmentParallelCorpusUpdateConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + if (parallelCorpusId == null) + throw new System.ArgumentNullException("parallelCorpusId"); + + if (corpusConfig == null) + throw new System.ArgumentNullException("corpusConfig"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(corpusConfig, JsonSerializerSettings); + var content_ = new System.Net.Http.StringContent(json_); + content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); + request_.Content = content_; + request_.Method = new System.Net.Http.HttpMethod("PATCH"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); + // Operation Path: "word-alignment/engines/{id}/parallel-corpora/{parallelCorpusId}" + urlBuilder_.Append("word-alignment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/parallel-corpora/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(parallelCorpusId, System.Globalization.CultureInfo.InvariantCulture))); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 400) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("Bad request", status_, responseText_, headers_, null); + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get the configuration of a parallel corpus for a engine + /// + /// The engine id + /// The parallel corpus id + /// The parallel corpus configuration + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task GetParallelCorpusAsync(string id, string parallelCorpusId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + if (parallelCorpusId == null) + throw new System.ArgumentNullException("parallelCorpusId"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); + // Operation Path: "word-alignment/engines/{id}/parallel-corpora/{parallelCorpusId}" + urlBuilder_.Append("word-alignment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/parallel-corpora/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(parallelCorpusId, System.Globalization.CultureInfo.InvariantCulture))); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine or parallel corpus does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Remove a parallel corpus from a engine + /// + /// + /// Removing a parallel corpus will remove all word alignments associated with that corpus. + /// + /// The engine id + /// The parallel corpus id + /// The parallel corpus was deleted successfully. + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task DeleteParallelCorpusAsync(string id, string parallelCorpusId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + if (parallelCorpusId == null) + throw new System.ArgumentNullException("parallelCorpusId"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("DELETE"); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); + // Operation Path: "word-alignment/engines/{id}/parallel-corpora/{parallelCorpusId}" + urlBuilder_.Append("word-alignment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/parallel-corpora/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(parallelCorpusId, System.Globalization.CultureInfo.InvariantCulture))); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + return; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine or parallel corpus does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get all word alignments in a corpus of a engine + /// + /// + /// Word alignments are arranged in a list of dictionaries with the following fields per word alignment: + ///
* **TextId**: The TextId of the SourceFile defined when the corpus was created. + ///
* **Refs** (a list of strings): A list of references including: + ///
* The references defined in the SourceFile per line, if any. + ///
* An auto-generated reference of `[TextId]:[lineNumber]`, 1 indexed. + ///
* **SourceTokens**: the tokenized source segment + ///
* **TargetTokens**: the tokenized target segment + ///
* **Confidences**: the confidence of the alignment ona scale from 0 to 1 + ///
* **Alignment**: the word alignment, 0 indexed for source and target positions + ///
+ ///
Word alignments can be filtered by text id if provided. + ///
Only word alignments for the most recent successful build of the engine are returned. + /// + /// The engine id + /// The corpus id + /// The text id (optional) + /// The word alignments + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task> GetAllWordAlignmentsAsync(string id, string corpusId, string? textId = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + if (corpusId == null) + throw new System.ArgumentNullException("corpusId"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); + // Operation Path: "word-alignment/engines/{id}/corpora/{corpusId}/word-alignments" + urlBuilder_.Append("word-alignment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/corpora/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(corpusId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/word-alignments"); + urlBuilder_.Append('?'); + if (textId != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("textId")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(textId, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + urlBuilder_.Length--; + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 409) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine needs to be built first.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get all build jobs for a engine + /// + /// The engine id + /// The build jobs + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task> GetAllBuildsAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); + // Operation Path: "word-alignment/engines/{id}/builds" + urlBuilder_.Append("word-alignment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/builds"); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation or does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Starts a build job for a engine. + /// + /// + /// Specify the corpora and textIds to train on. If no "trainOn" field is provided, all corpora will be used. + ///
Paratext Projects, you may flag a subset of books for training by including their [abbreviations] + ///
Paratext projects can be filtered by [book](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs) using the textId for training. + ///
Filters can also be supplied via scriptureRange parameter as ranges of biblical text. See [here](https://github.com/sillsdev/serval/wiki/Filtering-Paratext-Project-Data-with-a-Scripture-Range) + ///
All Paratext project filtering follows original versification. See [here](https://github.com/sillsdev/serval/wiki/Versification-in-Serval) for more information. + ///
+ ///
Specify the corpora or textIds to word align on. + ///
When a corpus or textId is selected for word align on, only text segments that are in both the source and the target will be aligned. + ///
+ ///
The `"options"` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. + ///
See [statistical alignment job settings documentation](https://github.com/sillsdev/serval/wiki/Statistical-Alignment-Build-Options) about configuring job parameters. + ///
See [keyterms parsing documentation](https://github.com/sillsdev/serval/wiki/Paratext-Key-Terms-Parsing) on how to use keyterms for training. + /// + /// The engine id + /// The build config (see remarks) + /// The new build job + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task StartBuildAsync(string id, WordAlignmentBuildConfig buildConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + if (buildConfig == null) + throw new System.ArgumentNullException("buildConfig"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(buildConfig, JsonSerializerSettings); + var content_ = new System.Net.Http.StringContent(json_); + content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); + request_.Content = content_; + request_.Method = new System.Net.Http.HttpMethod("POST"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); + // Operation Path: "word-alignment/engines/{id}/builds" + urlBuilder_.Append("word-alignment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/builds"); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 201) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 400) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The build configuration was invalid.", status_, responseText_, headers_, null); + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 409) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("There is already an active or pending build or a build in the process of being canceled.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get a build job + /// + /// + /// If the `minRevision` is not defined, the current build, at whatever state it is, + ///
will be immediately returned. If `minRevision` is defined, Serval will wait for + ///
up to 40 seconds for the engine to build to the `minRevision` specified, else + ///
will timeout. + ///
A use case is to actively query the state of the current build, where the subsequent + ///
request sets the `minRevision` to the returned `revision` + 1 and timeouts are handled gracefully. + ///
This method should use request throttling. + ///
Note: Within the returned build, percentCompleted is a value between 0 and 1. + /// + /// The engine id + /// The build job id + /// The minimum revision + /// The build job + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task GetBuildAsync(string id, string buildId, long? minRevision = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + if (buildId == null) + throw new System.ArgumentNullException("buildId"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); + // Operation Path: "word-alignment/engines/{id}/builds/{buildId}" + urlBuilder_.Append("word-alignment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/builds/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(buildId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append('?'); + if (minRevision != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("minRevision")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(minRevision, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + urlBuilder_.Length--; + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine or build does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 408) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The long polling request timed out. This is expected behavior if you\'re using long-polling with the minRevision strategy specified in the docs.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get the currently running build job for a engine + /// + /// + /// See documentation on endpoint /word-alignment/engines/{id}/builds/{id} - "Get a Build Job" for details on using `minRevision`. + /// + /// The engine id + /// The minimum revision + /// The build job + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task GetCurrentBuildAsync(string id, long? minRevision = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); + // Operation Path: "word-alignment/engines/{id}/current-build" + urlBuilder_.Append("word-alignment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/current-build"); + urlBuilder_.Append('?'); + if (minRevision != null) + { + urlBuilder_.Append(System.Uri.EscapeDataString("minRevision")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(minRevision, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); + } + urlBuilder_.Length--; + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 204) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("There is no build currently running.", status_, responseText_, headers_, null); + } + else + if (status_ == 400) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("Bad request", status_, responseText_, headers_, null); + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 408) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The long polling request timed out. This is expected behavior if you\'re using long-polling with the minRevision strategy specified in the docs.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Cancel the current build job (whether pending or active) for a engine + /// + /// The engine id + /// The build job was cancelled successfully. + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task CancelBuildAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (id == null) + throw new System.ArgumentNullException("id"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Content = new System.Net.Http.StringContent(string.Empty, System.Text.Encoding.UTF8, "application/json"); + request_.Method = new System.Net.Http.HttpMethod("POST"); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); + // Operation Path: "word-alignment/engines/{id}/current-build/cancel" + urlBuilder_.Append("word-alignment/engines/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/current-build/cancel"); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + return; + } + else + if (status_ == 204) + { + return; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client does not own the engine.", status_, responseText_, headers_, null); + } + else + if (status_ == 404) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine does not exist.", status_, responseText_, headers_, null); + } + else + if (status_ == 405) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The engine does not support cancelling builds.", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + protected struct ObjectResponseResult + { + public ObjectResponseResult(T responseObject, string responseText) + { + this.Object = responseObject; + this.Text = responseText; + } + + public T Object { get; } + + public string Text { get; } + } + + public bool ReadResponseAsString { get; set; } + + protected virtual async System.Threading.Tasks.Task> ReadObjectResponseAsync(System.Net.Http.HttpResponseMessage response, System.Collections.Generic.IReadOnlyDictionary> headers, System.Threading.CancellationToken cancellationToken) + { + if (response == null || response.Content == null) + { + return new ObjectResponseResult(default(T)!, string.Empty); + } + + if (ReadResponseAsString) + { + var responseText = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + try + { + var typedBody = Newtonsoft.Json.JsonConvert.DeserializeObject(responseText, JsonSerializerSettings); + return new ObjectResponseResult(typedBody!, responseText); + } + catch (Newtonsoft.Json.JsonException exception) + { + var message = "Could not deserialize the response body string as " + typeof(T).FullName + "."; + throw new ServalApiException(message, (int)response.StatusCode, responseText, headers, exception); + } + } + else + { + try + { + using (var responseStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) + using (var streamReader = new System.IO.StreamReader(responseStream)) + using (var jsonTextReader = new Newtonsoft.Json.JsonTextReader(streamReader)) + { + var serializer = Newtonsoft.Json.JsonSerializer.Create(JsonSerializerSettings); + var typedBody = serializer.Deserialize(jsonTextReader); + return new ObjectResponseResult(typedBody!, string.Empty); + } + } + catch (Newtonsoft.Json.JsonException exception) + { + var message = "Could not deserialize the response body stream as " + typeof(T).FullName + "."; + throw new ServalApiException(message, (int)response.StatusCode, string.Empty, headers, exception); + } + } + } + + private string ConvertToString(object? value, System.Globalization.CultureInfo cultureInfo) + { + if (value == null) + { + return ""; + } + + if (value is System.Enum) + { + var name = System.Enum.GetName(value.GetType(), value); + if (name != null) + { + var field = System.Reflection.IntrospectionExtensions.GetTypeInfo(value.GetType()).GetDeclaredField(name); + if (field != null) + { + var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field, typeof(System.Runtime.Serialization.EnumMemberAttribute)) + as System.Runtime.Serialization.EnumMemberAttribute; + if (attribute != null) + { + return attribute.Value != null ? attribute.Value : name; + } + } + + var converted = System.Convert.ToString(System.Convert.ChangeType(value, System.Enum.GetUnderlyingType(value.GetType()), cultureInfo)); + return converted == null ? string.Empty : converted; + } + } + else if (value is bool) + { + return System.Convert.ToString((bool)value, cultureInfo).ToLowerInvariant(); + } + else if (value is byte[]) + { + return System.Convert.ToBase64String((byte[]) value); + } + else if (value is string[]) + { + return string.Join(",", (string[])value); + } + else if (value.GetType().IsArray) + { + var valueArray = (System.Array)value; + var valueTextArray = new string[valueArray.Length]; + for (var i = 0; i < valueArray.Length; i++) + { + valueTextArray[i] = ConvertToString(valueArray.GetValue(i), cultureInfo); + } + return string.Join(",", valueTextArray); + } + + var result = System.Convert.ToString(value, cultureInfo); + return result == null ? "" : result; + } + } + + [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial interface IWordAlignmentEngineTypesClient + { + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get queue information for a given engine type + /// + /// A valid engine type: statistical or echo-word-alignment + /// Queue information for the specified engine type + /// A server side error occurred. + System.Threading.Tasks.Task GetQueueAsync(string engineType, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + } + + [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class WordAlignmentEngineTypesClient : IWordAlignmentEngineTypesClient + { + #pragma warning disable 8618 + private string _baseUrl; + #pragma warning restore 8618 + + private System.Net.Http.HttpClient _httpClient; + private static System.Lazy _settings = new System.Lazy(CreateSerializerSettings, true); + private Newtonsoft.Json.JsonSerializerSettings _instanceSettings; + + #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. + public WordAlignmentEngineTypesClient(System.Net.Http.HttpClient httpClient) + #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. + { + BaseUrl = "/api/v1"; + _httpClient = httpClient; + Initialize(); + } + + private static Newtonsoft.Json.JsonSerializerSettings CreateSerializerSettings() + { + var settings = new Newtonsoft.Json.JsonSerializerSettings(); + UpdateJsonSerializerSettings(settings); + return settings; + } + + public string BaseUrl + { + get { return _baseUrl; } + set + { + _baseUrl = value; + if (!string.IsNullOrEmpty(_baseUrl) && !_baseUrl.EndsWith("/")) + _baseUrl += '/'; + } + } + + protected Newtonsoft.Json.JsonSerializerSettings JsonSerializerSettings { get { return _instanceSettings ?? _settings.Value; } } + + static partial void UpdateJsonSerializerSettings(Newtonsoft.Json.JsonSerializerSettings settings); + + partial void Initialize(); + + partial void PrepareRequest(System.Net.Http.HttpClient client, System.Net.Http.HttpRequestMessage request, string url); + partial void PrepareRequest(System.Net.Http.HttpClient client, System.Net.Http.HttpRequestMessage request, System.Text.StringBuilder urlBuilder); + partial void ProcessResponse(System.Net.Http.HttpClient client, System.Net.Http.HttpResponseMessage response); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get queue information for a given engine type + /// + /// A valid engine type: statistical or echo-word-alignment + /// Queue information for the specified engine type + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task GetQueueAsync(string engineType, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (engineType == null) + throw new System.ArgumentNullException("engineType"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); + // Operation Path: "word-alignment/engine-types/{engineType}/queues" + urlBuilder_.Append("word-alignment/engine-types/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(engineType, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/queues"); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + protected struct ObjectResponseResult + { + public ObjectResponseResult(T responseObject, string responseText) + { + this.Object = responseObject; + this.Text = responseText; + } + + public T Object { get; } + + public string Text { get; } + } + + public bool ReadResponseAsString { get; set; } + + protected virtual async System.Threading.Tasks.Task> ReadObjectResponseAsync(System.Net.Http.HttpResponseMessage response, System.Collections.Generic.IReadOnlyDictionary> headers, System.Threading.CancellationToken cancellationToken) + { + if (response == null || response.Content == null) + { + return new ObjectResponseResult(default(T)!, string.Empty); + } + + if (ReadResponseAsString) + { + var responseText = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + try + { + var typedBody = Newtonsoft.Json.JsonConvert.DeserializeObject(responseText, JsonSerializerSettings); + return new ObjectResponseResult(typedBody!, responseText); + } + catch (Newtonsoft.Json.JsonException exception) + { + var message = "Could not deserialize the response body string as " + typeof(T).FullName + "."; + throw new ServalApiException(message, (int)response.StatusCode, responseText, headers, exception); + } + } + else + { + try + { + using (var responseStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) + using (var streamReader = new System.IO.StreamReader(responseStream)) + using (var jsonTextReader = new Newtonsoft.Json.JsonTextReader(streamReader)) + { + var serializer = Newtonsoft.Json.JsonSerializer.Create(JsonSerializerSettings); + var typedBody = serializer.Deserialize(jsonTextReader); + return new ObjectResponseResult(typedBody!, string.Empty); + } + } + catch (Newtonsoft.Json.JsonException exception) + { + var message = "Could not deserialize the response body stream as " + typeof(T).FullName + "."; + throw new ServalApiException(message, (int)response.StatusCode, string.Empty, headers, exception); + } + } + } + + private string ConvertToString(object? value, System.Globalization.CultureInfo cultureInfo) + { + if (value == null) + { + return ""; + } + + if (value is System.Enum) + { + var name = System.Enum.GetName(value.GetType(), value); + if (name != null) + { + var field = System.Reflection.IntrospectionExtensions.GetTypeInfo(value.GetType()).GetDeclaredField(name); + if (field != null) + { + var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field, typeof(System.Runtime.Serialization.EnumMemberAttribute)) + as System.Runtime.Serialization.EnumMemberAttribute; + if (attribute != null) + { + return attribute.Value != null ? attribute.Value : name; + } + } + + var converted = System.Convert.ToString(System.Convert.ChangeType(value, System.Enum.GetUnderlyingType(value.GetType()), cultureInfo)); + return converted == null ? string.Empty : converted; + } + } + else if (value is bool) + { + return System.Convert.ToString((bool)value, cultureInfo).ToLowerInvariant(); + } + else if (value is byte[]) + { + return System.Convert.ToBase64String((byte[]) value); + } + else if (value is string[]) + { + return string.Join(",", (string[])value); + } + else if (value.GetType().IsArray) + { + var valueArray = (System.Array)value; + var valueTextArray = new string[valueArray.Length]; + for (var i = 0; i < valueArray.Length; i++) + { + valueTextArray[i] = ConvertToString(valueArray.GetValue(i), cultureInfo); + } + return string.Join(",", valueTextArray); + } + + var result = System.Convert.ToString(value, cultureInfo); + return result == null ? "" : result; + } + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class HealthReport + { + [Newtonsoft.Json.JsonProperty("status", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Status { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("totalDuration", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string TotalDuration { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("results", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IDictionary Results { get; set; } = new System.Collections.Generic.Dictionary(); + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class HealthReportEntry + { + [Newtonsoft.Json.JsonProperty("status", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Status { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("duration", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Duration { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("description", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Description { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("exception", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Exception { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("data", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IDictionary? Data { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class DeploymentInfo + { + [Newtonsoft.Json.JsonProperty("deploymentVersion", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string DeploymentVersion { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("aspNetCoreEnvironment", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string AspNetCoreEnvironment { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AssessmentEngine + { + [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Id { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Url { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Name { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("type", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Type { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("corpus", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public AssessmentCorpus Corpus { get; set; } = new AssessmentCorpus(); + + [Newtonsoft.Json.JsonProperty("referenceCorpus", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public AssessmentCorpus? ReferenceCorpus { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AssessmentCorpus + { + [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Url { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Name { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("language", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Language { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("files", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Files { get; set; } = new System.Collections.ObjectModel.Collection(); + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AssessmentCorpusFile + { + [Newtonsoft.Json.JsonProperty("file", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public ResourceLink File { get; set; } = new ResourceLink(); + + [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? TextId { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class ResourceLink + { + [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Id { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Url { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AssessmentEngineConfig + { + /// + /// The assessment engine name. + /// + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Name { get; set; } = default!; + + /// + /// The assessment engine type. + /// + [Newtonsoft.Json.JsonProperty("type", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Type { get; set; } = default!; + + /// + /// The corpus. + /// + [Newtonsoft.Json.JsonProperty("corpus", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public AssessmentCorpusConfig Corpus { get; set; } = new AssessmentCorpusConfig(); + + /// + /// The reference corpus. + /// + [Newtonsoft.Json.JsonProperty("referenceCorpus", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public AssessmentCorpusConfig? ReferenceCorpus { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AssessmentCorpusConfig + { + /// + /// The corpus name. + /// + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Name { get; set; } = default!; + + /// + /// The language tag. + /// + [Newtonsoft.Json.JsonProperty("language", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Language { get; set; } = default!; + + /// + /// The corpus files. + /// + [Newtonsoft.Json.JsonProperty("files", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Files { get; set; } = new System.Collections.ObjectModel.Collection(); + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AssessmentCorpusFileConfig + { + [Newtonsoft.Json.JsonProperty("fileId", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string FileId { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? TextId { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AssessmentJob + { + [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Id { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Url { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("revision", Required = Newtonsoft.Json.Required.Always)] + public int Revision { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Name { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("engine", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public ResourceLink Engine { get; set; } = new ResourceLink(); + + [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TextIds { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? ScriptureRange { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("percentCompleted", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public double? PercentCompleted { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("message", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Message { get; set; } = default!; + + /// + /// The current job state. + /// + [Newtonsoft.Json.JsonProperty("state", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + [Newtonsoft.Json.JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] + public JobState State { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("dateFinished", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.DateTimeOffset? DateFinished { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("options", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public object? Options { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public enum JobState + { + + [System.Runtime.Serialization.EnumMember(Value = @"Pending")] + Pending = 0, + + [System.Runtime.Serialization.EnumMember(Value = @"Active")] + Active = 1, + + [System.Runtime.Serialization.EnumMember(Value = @"Completed")] + Completed = 2, + + [System.Runtime.Serialization.EnumMember(Value = @"Faulted")] + Faulted = 3, + + [System.Runtime.Serialization.EnumMember(Value = @"Canceled")] + Canceled = 4, + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AssessmentJobConfig + { + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Name { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TextIds { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? ScriptureRange { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("options", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public object? Options { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AssessmentResult + { + [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string TextId { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("ref", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Ref { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("score", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public double? Score { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("description", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Description { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class Corpus + { + [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Id { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("revision", Required = Newtonsoft.Json.Required.Always)] + public int Revision { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("language", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Language { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Name { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Url { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("files", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Files { get; set; } = new System.Collections.ObjectModel.Collection(); + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class CorpusFile + { + [Newtonsoft.Json.JsonProperty("file", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public DataFile File { get; set; } = new DataFile(); + + [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? TextId { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class DataFile + { + [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Id { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Url { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Name { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("format", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + [Newtonsoft.Json.JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] + public FileFormat Format { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("revision", Required = Newtonsoft.Json.Required.Always)] + public int Revision { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public enum FileFormat + { + + [System.Runtime.Serialization.EnumMember(Value = @"Text")] + Text = 0, + + [System.Runtime.Serialization.EnumMember(Value = @"Paratext")] + Paratext = 1, + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class CorpusConfig + { + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Name { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("language", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Language { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("files", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Files { get; set; } = new System.Collections.ObjectModel.Collection(); + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class CorpusFileConfig + { + [Newtonsoft.Json.JsonProperty("fileId", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string FileId { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? TextId { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class TranslationEngine + { + [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Id { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Url { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Name { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("sourceLanguage", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string SourceLanguage { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("targetLanguage", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string TargetLanguage { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("type", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Type { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("isModelPersisted", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public bool? IsModelPersisted { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("isBuilding", Required = Newtonsoft.Json.Required.Always)] + public bool IsBuilding { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("modelRevision", Required = Newtonsoft.Json.Required.Always)] + public int ModelRevision { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("confidence", Required = Newtonsoft.Json.Required.Always)] + public double Confidence { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("corpusSize", Required = Newtonsoft.Json.Required.Always)] + public int CorpusSize { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class TranslationEngineConfig + { + /// + /// The translation engine name. + /// + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Name { get; set; } = default!; + + /// + /// The source language tag. + /// + [Newtonsoft.Json.JsonProperty("sourceLanguage", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string SourceLanguage { get; set; } = default!; + + /// + /// The target language tag. + /// + [Newtonsoft.Json.JsonProperty("targetLanguage", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string TargetLanguage { get; set; } = default!; + + /// + /// The translation engine type. + /// + [Newtonsoft.Json.JsonProperty("type", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Type { get; set; } = default!; + + /// + /// The model is saved when built and can be retrieved. + /// + [Newtonsoft.Json.JsonProperty("isModelPersisted", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public bool? IsModelPersisted { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class TranslationResult + { + [Newtonsoft.Json.JsonProperty("translation", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Translation { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("sourceTokens", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList SourceTokens { get; set; } = new System.Collections.ObjectModel.Collection(); + + [Newtonsoft.Json.JsonProperty("targetTokens", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList TargetTokens { get; set; } = new System.Collections.ObjectModel.Collection(); + + [Newtonsoft.Json.JsonProperty("confidences", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Confidences { get; set; } = new System.Collections.ObjectModel.Collection(); + + [Newtonsoft.Json.JsonProperty("sources", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList> Sources { get; set; } = new System.Collections.ObjectModel.Collection>(); + + [Newtonsoft.Json.JsonProperty("alignment", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Alignment { get; set; } = new System.Collections.ObjectModel.Collection(); + + [Newtonsoft.Json.JsonProperty("phrases", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Phrases { get; set; } = new System.Collections.ObjectModel.Collection(); + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public enum TranslationSource + { + + [System.Runtime.Serialization.EnumMember(Value = @"Primary")] + Primary = 0, + + [System.Runtime.Serialization.EnumMember(Value = @"Secondary")] + Secondary = 1, + + [System.Runtime.Serialization.EnumMember(Value = @"Human")] + Human = 2, + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class AlignedWordPair + { + [Newtonsoft.Json.JsonProperty("sourceIndex", Required = Newtonsoft.Json.Required.Always)] + public int SourceIndex { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("targetIndex", Required = Newtonsoft.Json.Required.Always)] + public int TargetIndex { get; set; } = default!; + + } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class HealthReport + public partial class Phrase { - [Newtonsoft.Json.JsonProperty("status", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Status { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("sourceSegmentStart", Required = Newtonsoft.Json.Required.Always)] + public int SourceSegmentStart { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("totalDuration", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string TotalDuration { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("sourceSegmentEnd", Required = Newtonsoft.Json.Required.Always)] + public int SourceSegmentEnd { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("results", Required = Newtonsoft.Json.Required.Always)] + [Newtonsoft.Json.JsonProperty("targetSegmentCut", Required = Newtonsoft.Json.Required.Always)] + public int TargetSegmentCut { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class WordGraph + { + [Newtonsoft.Json.JsonProperty("sourceTokens", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IDictionary Results { get; set; } = new System.Collections.Generic.Dictionary(); + public System.Collections.Generic.IList SourceTokens { get; set; } = new System.Collections.ObjectModel.Collection(); + + [Newtonsoft.Json.JsonProperty("initialStateScore", Required = Newtonsoft.Json.Required.Always)] + public float InitialStateScore { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("finalStates", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList FinalStates { get; set; } = new System.Collections.ObjectModel.Collection(); + + [Newtonsoft.Json.JsonProperty("arcs", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Arcs { get; set; } = new System.Collections.ObjectModel.Collection(); } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class HealthReportEntry + public partial class WordGraphArc { - [Newtonsoft.Json.JsonProperty("status", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Status { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("prevState", Required = Newtonsoft.Json.Required.Always)] + public int PrevState { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("duration", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Duration { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("nextState", Required = Newtonsoft.Json.Required.Always)] + public int NextState { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("description", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? Description { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("score", Required = Newtonsoft.Json.Required.Always)] + public double Score { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("exception", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? Exception { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("targetTokens", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList TargetTokens { get; set; } = new System.Collections.ObjectModel.Collection(); - [Newtonsoft.Json.JsonProperty("data", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IDictionary? Data { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("confidences", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Confidences { get; set; } = new System.Collections.ObjectModel.Collection(); + + [Newtonsoft.Json.JsonProperty("sourceSegmentStart", Required = Newtonsoft.Json.Required.Always)] + public int SourceSegmentStart { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("sourceSegmentEnd", Required = Newtonsoft.Json.Required.Always)] + public int SourceSegmentEnd { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("alignment", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Alignment { get; set; } = new System.Collections.ObjectModel.Collection(); + + [Newtonsoft.Json.JsonProperty("sources", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList> Sources { get; set; } = new System.Collections.ObjectModel.Collection>(); } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class DeploymentInfo + public partial class SegmentPair { - [Newtonsoft.Json.JsonProperty("deploymentVersion", Required = Newtonsoft.Json.Required.Always)] + [Newtonsoft.Json.JsonProperty("sourceSegment", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string DeploymentVersion { get; set; } = default!; + public string SourceSegment { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("aspNetCoreEnvironment", Required = Newtonsoft.Json.Required.Always)] + [Newtonsoft.Json.JsonProperty("targetSegment", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string AspNetCoreEnvironment { get; set; } = default!; + public string TargetSegment { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("sentenceStart", Required = Newtonsoft.Json.Required.Always)] + public bool SentenceStart { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class AssessmentEngine + public partial class TranslationCorpus { [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] @@ -9018,48 +12134,76 @@ public partial class AssessmentEngine [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] public string Url { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("engine", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public ResourceLink Engine { get; set; } = new ResourceLink(); + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? Name { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("type", Required = Newtonsoft.Json.Required.Always)] + [Newtonsoft.Json.JsonProperty("sourceLanguage", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Type { get; set; } = default!; + public string SourceLanguage { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("corpus", Required = Newtonsoft.Json.Required.Always)] + [Newtonsoft.Json.JsonProperty("targetLanguage", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string TargetLanguage { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("sourceFiles", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] - public AssessmentCorpus Corpus { get; set; } = new AssessmentCorpus(); + public System.Collections.Generic.IList SourceFiles { get; set; } = new System.Collections.ObjectModel.Collection(); - [Newtonsoft.Json.JsonProperty("referenceCorpus", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public AssessmentCorpus? ReferenceCorpus { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("targetFiles", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList TargetFiles { get; set; } = new System.Collections.ObjectModel.Collection(); } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class AssessmentCorpus + public partial class TranslationCorpusFile { - [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Url { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("file", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public ResourceLink File { get; set; } = new ResourceLink(); + + [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? TextId { get; set; } = default!; + + } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class TranslationCorpusConfig + { + /// + /// The corpus name. + /// [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? Name { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("language", Required = Newtonsoft.Json.Required.Always)] + [Newtonsoft.Json.JsonProperty("sourceLanguage", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Language { get; set; } = default!; + public string SourceLanguage { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("files", Required = Newtonsoft.Json.Required.Always)] + [Newtonsoft.Json.JsonProperty("targetLanguage", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string TargetLanguage { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("sourceFiles", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList Files { get; set; } = new System.Collections.ObjectModel.Collection(); + public System.Collections.Generic.IList SourceFiles { get; set; } = new System.Collections.ObjectModel.Collection(); + + [Newtonsoft.Json.JsonProperty("targetFiles", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList TargetFiles { get; set; } = new System.Collections.ObjectModel.Collection(); } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class AssessmentCorpusFile + public partial class TranslationCorpusFileConfig { - [Newtonsoft.Json.JsonProperty("file", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public ResourceLink File { get; set; } = new ResourceLink(); + [Newtonsoft.Json.JsonProperty("fileId", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string FileId { get; set; } = default!; [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? TextId { get; set; } = default!; @@ -9067,7 +12211,18 @@ public partial class AssessmentCorpusFile } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class ResourceLink + public partial class TranslationCorpusUpdateConfig + { + [Newtonsoft.Json.JsonProperty("sourceFiles", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? SourceFiles { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("targetFiles", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TargetFiles { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class TranslationParallelCorpus { [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] @@ -9077,78 +12232,102 @@ public partial class ResourceLink [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] public string Url { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("engine", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public ResourceLink Engine { get; set; } = new ResourceLink(); + + [Newtonsoft.Json.JsonProperty("sourceCorpora", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList SourceCorpora { get; set; } = new System.Collections.ObjectModel.Collection(); + + [Newtonsoft.Json.JsonProperty("targetCorpora", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList TargetCorpora { get; set; } = new System.Collections.ObjectModel.Collection(); + } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class AssessmentEngineConfig + public partial class TranslationParallelCorpusConfig { /// - /// The assessment engine name. + /// The corpus name. /// [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? Name { get; set; } = default!; - /// - /// The assessment engine type. - /// - [Newtonsoft.Json.JsonProperty("type", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Type { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("sourceCorpusIds", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList SourceCorpusIds { get; set; } = new System.Collections.ObjectModel.Collection(); - /// - /// The corpus. - /// - [Newtonsoft.Json.JsonProperty("corpus", Required = Newtonsoft.Json.Required.Always)] + [Newtonsoft.Json.JsonProperty("targetCorpusIds", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] - public AssessmentCorpusConfig Corpus { get; set; } = new AssessmentCorpusConfig(); + public System.Collections.Generic.IList TargetCorpusIds { get; set; } = new System.Collections.ObjectModel.Collection(); - /// - /// The reference corpus. - /// - [Newtonsoft.Json.JsonProperty("referenceCorpus", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public AssessmentCorpusConfig? ReferenceCorpus { get; set; } = default!; + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class TranslationParallelCorpusUpdateConfig + { + [Newtonsoft.Json.JsonProperty("sourceCorpusIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? SourceCorpusIds { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("targetCorpusIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TargetCorpusIds { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class AssessmentCorpusConfig + public partial class Pretranslation { - /// - /// The corpus name. - /// - [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? Name { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string TextId { get; set; } = default!; - /// - /// The language tag. - /// - [Newtonsoft.Json.JsonProperty("language", Required = Newtonsoft.Json.Required.Always)] + [Newtonsoft.Json.JsonProperty("refs", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Refs { get; set; } = new System.Collections.ObjectModel.Collection(); + + [Newtonsoft.Json.JsonProperty("translation", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Language { get; set; } = default!; + public string Translation { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public enum PretranslationUsfmTextOrigin + { + + [System.Runtime.Serialization.EnumMember(Value = @"PreferExisting")] + PreferExisting = 0, + + [System.Runtime.Serialization.EnumMember(Value = @"PreferPretranslated")] + PreferPretranslated = 1, + + [System.Runtime.Serialization.EnumMember(Value = @"OnlyExisting")] + OnlyExisting = 2, - /// - /// The corpus files. - /// - [Newtonsoft.Json.JsonProperty("files", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList Files { get; set; } = new System.Collections.ObjectModel.Collection(); + [System.Runtime.Serialization.EnumMember(Value = @"OnlyPretranslated")] + OnlyPretranslated = 3, } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class AssessmentCorpusFileConfig + public enum PretranslationUsfmTemplate { - [Newtonsoft.Json.JsonProperty("fileId", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string FileId { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? TextId { get; set; } = default!; + [System.Runtime.Serialization.EnumMember(Value = @"Auto")] + Auto = 0, + + [System.Runtime.Serialization.EnumMember(Value = @"Source")] + Source = 1, + + [System.Runtime.Serialization.EnumMember(Value = @"Target")] + Target = 2, } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class AssessmentJob + public partial class TranslationBuild { [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] @@ -9168,11 +12347,14 @@ public partial class AssessmentJob [System.ComponentModel.DataAnnotations.Required] public ResourceLink Engine { get; set; } = new ResourceLink(); - [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? TextIds { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("trainOn", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TrainOn { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? ScriptureRange { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("pretranslate", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? Pretranslate { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("step", Required = Newtonsoft.Json.Required.Always)] + public int Step { get; set; } = default!; [Newtonsoft.Json.JsonProperty("percentCompleted", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public double? PercentCompleted { get; set; } = default!; @@ -9180,8 +12362,11 @@ public partial class AssessmentJob [Newtonsoft.Json.JsonProperty("message", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? Message { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("queueDepth", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public int? QueueDepth { get; set; } = default!; + /// - /// The current job state. + /// The current build job state. /// [Newtonsoft.Json.JsonProperty("state", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] @@ -9194,34 +12379,16 @@ public partial class AssessmentJob [Newtonsoft.Json.JsonProperty("options", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public object? Options { get; set; } = default!; - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public enum JobState - { - - [System.Runtime.Serialization.EnumMember(Value = @"Pending")] - Pending = 0, - - [System.Runtime.Serialization.EnumMember(Value = @"Active")] - Active = 1, - - [System.Runtime.Serialization.EnumMember(Value = @"Completed")] - Completed = 2, - - [System.Runtime.Serialization.EnumMember(Value = @"Faulted")] - Faulted = 3, - - [System.Runtime.Serialization.EnumMember(Value = @"Canceled")] - Canceled = 4, + [Newtonsoft.Json.JsonProperty("deploymentVersion", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? DeploymentVersion { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class AssessmentJobConfig + public partial class TrainingCorpus { - [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? Name { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("corpus", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public ResourceLink? Corpus { get; set; } = default!; [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public System.Collections.Generic.IList? TextIds { get; set; } = default!; @@ -9229,361 +12396,243 @@ public partial class AssessmentJobConfig [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? ScriptureRange { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("options", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public object? Options { get; set; } = default!; - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class AssessmentResult - { - [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string TextId { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("ref", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Ref { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("score", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public double? Score { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("description", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? Description { get; set; } = default!; - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class Corpus - { - [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Id { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("revision", Required = Newtonsoft.Json.Required.Always)] - public int Revision { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("language", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Language { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? Name { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("parallelCorpus", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public ResourceLink? ParallelCorpus { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Url { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("sourceFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? SourceFilters { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("files", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList Files { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("targetFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TargetFilters { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class CorpusFile + public partial class ParallelCorpusFilter { - [Newtonsoft.Json.JsonProperty("file", Required = Newtonsoft.Json.Required.Always)] + [Newtonsoft.Json.JsonProperty("corpus", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] - public DataFile File { get; set; } = new DataFile(); - - [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? TextId { get; set; } = default!; - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class DataFile - { - [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Id { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Url { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? Name { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("format", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - [Newtonsoft.Json.JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] - public FileFormat Format { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("revision", Required = Newtonsoft.Json.Required.Always)] - public int Revision { get; set; } = default!; - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public enum FileFormat - { + public ResourceLink Corpus { get; set; } = new ResourceLink(); - [System.Runtime.Serialization.EnumMember(Value = @"Text")] - Text = 0, + [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TextIds { get; set; } = default!; - [System.Runtime.Serialization.EnumMember(Value = @"Paratext")] - Paratext = 1, + [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? ScriptureRange { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class CorpusConfig + public partial class PretranslateCorpus { - [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? Name { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("language", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Language { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("corpus", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public ResourceLink? Corpus { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("files", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList Files { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TextIds { get; set; } = default!; - } + [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? ScriptureRange { get; set; } = default!; - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class CorpusFileConfig - { - [Newtonsoft.Json.JsonProperty("fileId", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string FileId { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("parallelCorpus", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public ResourceLink? ParallelCorpus { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? TextId { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("sourceFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? SourceFilters { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TranslationEngine + public partial class TranslationBuildConfig { - [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Id { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Url { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? Name { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("sourceLanguage", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string SourceLanguage { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("targetLanguage", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string TargetLanguage { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("type", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Type { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("isModelPersisted", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public bool? IsModelPersisted { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("isBuilding", Required = Newtonsoft.Json.Required.Always)] - public bool IsBuilding { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("modelRevision", Required = Newtonsoft.Json.Required.Always)] - public int ModelRevision { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("trainOn", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TrainOn { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("confidence", Required = Newtonsoft.Json.Required.Always)] - public double Confidence { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("pretranslate", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? Pretranslate { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("corpusSize", Required = Newtonsoft.Json.Required.Always)] - public int CorpusSize { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("options", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public object? Options { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TranslationEngineConfig + public partial class TrainingCorpusConfig { - /// - /// The translation engine name. - /// - [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? Name { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("corpusId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? CorpusId { get; set; } = default!; - /// - /// The source language tag. - /// - [Newtonsoft.Json.JsonProperty("sourceLanguage", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string SourceLanguage { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TextIds { get; set; } = default!; - /// - /// The target language tag. - /// - [Newtonsoft.Json.JsonProperty("targetLanguage", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string TargetLanguage { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? ScriptureRange { get; set; } = default!; - /// - /// The translation engine type. - /// - [Newtonsoft.Json.JsonProperty("type", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Type { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("parallelCorpusId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? ParallelCorpusId { get; set; } = default!; - /// - /// The model is saved when built and can be retrieved. - /// - [Newtonsoft.Json.JsonProperty("isModelPersisted", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public bool? IsModelPersisted { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("sourceFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? SourceFilters { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("targetFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TargetFilters { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TranslationResult + public partial class ParallelCorpusFilterConfig { - [Newtonsoft.Json.JsonProperty("translation", Required = Newtonsoft.Json.Required.Always)] + [Newtonsoft.Json.JsonProperty("corpusId", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Translation { get; set; } = default!; + public string CorpusId { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("sourceTokens", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList SourceTokens { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TextIds { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("targetTokens", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList TargetTokens { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? ScriptureRange { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("confidences", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList Confidences { get; set; } = new System.Collections.ObjectModel.Collection(); + } - [Newtonsoft.Json.JsonProperty("sources", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList> Sources { get; set; } = new System.Collections.ObjectModel.Collection>(); + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class PretranslateCorpusConfig + { + [Newtonsoft.Json.JsonProperty("corpusId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? CorpusId { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("alignment", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList Alignment { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TextIds { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("phrases", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList Phrases { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? ScriptureRange { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("parallelCorpusId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? ParallelCorpusId { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("sourceFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? SourceFilters { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public enum TranslationSource + public partial class ModelDownloadUrl { + [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Url { get; set; } = default!; - [System.Runtime.Serialization.EnumMember(Value = @"Primary")] - Primary = 0, - - [System.Runtime.Serialization.EnumMember(Value = @"Secondary")] - Secondary = 1, + [Newtonsoft.Json.JsonProperty("modelRevision", Required = Newtonsoft.Json.Required.Always)] + public int ModelRevision { get; set; } = default!; - [System.Runtime.Serialization.EnumMember(Value = @"Human")] - Human = 2, + [Newtonsoft.Json.JsonProperty("expiresAt", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public System.DateTimeOffset ExpiresAt { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class AlignedWordPair + public partial class Queue { - [Newtonsoft.Json.JsonProperty("sourceIndex", Required = Newtonsoft.Json.Required.Always)] - public int SourceIndex { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("size", Required = Newtonsoft.Json.Required.Always)] + public int Size { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("targetIndex", Required = Newtonsoft.Json.Required.Always)] - public int TargetIndex { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("engineType", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string EngineType { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class Phrase + public partial class LanguageInfo { - [Newtonsoft.Json.JsonProperty("sourceSegmentStart", Required = Newtonsoft.Json.Required.Always)] - public int SourceSegmentStart { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("engineType", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string EngineType { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("sourceSegmentEnd", Required = Newtonsoft.Json.Required.Always)] - public int SourceSegmentEnd { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("isNative", Required = Newtonsoft.Json.Required.Always)] + public bool IsNative { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("targetSegmentCut", Required = Newtonsoft.Json.Required.Always)] - public int TargetSegmentCut { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("internalCode", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? InternalCode { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class WordGraph + public partial class Webhook { - [Newtonsoft.Json.JsonProperty("sourceTokens", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList SourceTokens { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Id { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("initialStateScore", Required = Newtonsoft.Json.Required.Always)] - public float InitialStateScore { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Url { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("finalStates", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList FinalStates { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("payloadUrl", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string PayloadUrl { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("arcs", Required = Newtonsoft.Json.Required.Always)] + [Newtonsoft.Json.JsonProperty("events", Required = Newtonsoft.Json.Required.Always, ItemConverterType = typeof(Newtonsoft.Json.Converters.StringEnumConverter))] [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList Arcs { get; set; } = new System.Collections.ObjectModel.Collection(); + public System.Collections.Generic.IList Events { get; set; } = new System.Collections.ObjectModel.Collection(); } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class WordGraphArc + public enum WebhookEvent { - [Newtonsoft.Json.JsonProperty("prevState", Required = Newtonsoft.Json.Required.Always)] - public int PrevState { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("nextState", Required = Newtonsoft.Json.Required.Always)] - public int NextState { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("score", Required = Newtonsoft.Json.Required.Always)] - public double Score { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("targetTokens", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList TargetTokens { get; set; } = new System.Collections.ObjectModel.Collection(); + [System.Runtime.Serialization.EnumMember(Value = @"TranslationBuildStarted")] + TranslationBuildStarted = 0, - [Newtonsoft.Json.JsonProperty("confidences", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList Confidences { get; set; } = new System.Collections.ObjectModel.Collection(); + [System.Runtime.Serialization.EnumMember(Value = @"TranslationBuildFinished")] + TranslationBuildFinished = 1, - [Newtonsoft.Json.JsonProperty("sourceSegmentStart", Required = Newtonsoft.Json.Required.Always)] - public int SourceSegmentStart { get; set; } = default!; + [System.Runtime.Serialization.EnumMember(Value = @"AssessmentJobStarted")] + AssessmentJobStarted = 2, - [Newtonsoft.Json.JsonProperty("sourceSegmentEnd", Required = Newtonsoft.Json.Required.Always)] - public int SourceSegmentEnd { get; set; } = default!; + [System.Runtime.Serialization.EnumMember(Value = @"AssessmentJobFinished")] + AssessmentJobFinished = 3, - [Newtonsoft.Json.JsonProperty("alignment", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList Alignment { get; set; } = new System.Collections.ObjectModel.Collection(); + [System.Runtime.Serialization.EnumMember(Value = @"WordAlignmentBuildStarted")] + WordAlignmentBuildStarted = 4, - [Newtonsoft.Json.JsonProperty("sources", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList> Sources { get; set; } = new System.Collections.ObjectModel.Collection>(); + [System.Runtime.Serialization.EnumMember(Value = @"WordAlignmentBuildFinished")] + WordAlignmentBuildFinished = 5, } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class SegmentPair + public partial class WebhookConfig { - [Newtonsoft.Json.JsonProperty("sourceSegment", Required = Newtonsoft.Json.Required.Always)] + /// + /// The payload URL. + /// + [Newtonsoft.Json.JsonProperty("payloadUrl", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string SourceSegment { get; set; } = default!; + public string PayloadUrl { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("targetSegment", Required = Newtonsoft.Json.Required.Always)] + /// + /// The shared secret. + /// + [Newtonsoft.Json.JsonProperty("secret", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string TargetSegment { get; set; } = default!; + public string Secret { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("sentenceStart", Required = Newtonsoft.Json.Required.Always)] - public bool SentenceStart { get; set; } = default!; + /// + /// The webhook events. + /// + [Newtonsoft.Json.JsonProperty("events", Required = Newtonsoft.Json.Required.Always, ItemConverterType = typeof(Newtonsoft.Json.Converters.StringEnumConverter))] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Events { get; set; } = new System.Collections.ObjectModel.Collection(); } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TranslationCorpus + public partial class WordAlignmentEngine { [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] @@ -9593,10 +12642,6 @@ public partial class TranslationCorpus [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] public string Url { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("engine", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public ResourceLink Engine { get; set; } = new ResourceLink(); - [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? Name { get; set; } = default!; @@ -9608,80 +12653,92 @@ public partial class TranslationCorpus [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] public string TargetLanguage { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("sourceFiles", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList SourceFiles { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("type", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Type { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("targetFiles", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList TargetFiles { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("isBuilding", Required = Newtonsoft.Json.Required.Always)] + public bool IsBuilding { get; set; } = default!; - } + [Newtonsoft.Json.JsonProperty("modelRevision", Required = Newtonsoft.Json.Required.Always)] + public int ModelRevision { get; set; } = default!; - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TranslationCorpusFile - { - [Newtonsoft.Json.JsonProperty("file", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public ResourceLink File { get; set; } = new ResourceLink(); + [Newtonsoft.Json.JsonProperty("confidence", Required = Newtonsoft.Json.Required.Always)] + public double Confidence { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? TextId { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("corpusSize", Required = Newtonsoft.Json.Required.Always)] + public int CorpusSize { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TranslationCorpusConfig + public partial class WordAlignmentEngineConfig { /// - /// The corpus name. + /// The word alignment engine name. /// [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? Name { get; set; } = default!; + /// + /// The source language tag. + /// [Newtonsoft.Json.JsonProperty("sourceLanguage", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] public string SourceLanguage { get; set; } = default!; + /// + /// The target language tag. + /// [Newtonsoft.Json.JsonProperty("targetLanguage", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] public string TargetLanguage { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("sourceFiles", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList SourceFiles { get; set; } = new System.Collections.ObjectModel.Collection(); - - [Newtonsoft.Json.JsonProperty("targetFiles", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList TargetFiles { get; set; } = new System.Collections.ObjectModel.Collection(); + /// + /// The translation engine type. + /// + [Newtonsoft.Json.JsonProperty("type", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Type { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TranslationCorpusFileConfig + public partial class WordAlignmentResult { - [Newtonsoft.Json.JsonProperty("fileId", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string FileId { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("sourceTokens", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList SourceTokens { get; set; } = new System.Collections.ObjectModel.Collection(); + + [Newtonsoft.Json.JsonProperty("targetTokens", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList TargetTokens { get; set; } = new System.Collections.ObjectModel.Collection(); - [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? TextId { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("confidences", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Confidences { get; set; } = new System.Collections.ObjectModel.Collection(); + + [Newtonsoft.Json.JsonProperty("alignment", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Alignment { get; set; } = new System.Collections.ObjectModel.Collection(); } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TranslationCorpusUpdateConfig + public partial class WordAlignmentRequest { - [Newtonsoft.Json.JsonProperty("sourceFiles", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? SourceFiles { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("sourceSegment", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string SourceSegment { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("targetFiles", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? TargetFiles { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("targetSegment", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string TargetSegment { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TranslationParallelCorpus + public partial class WordAlignmentParallelCorpus { [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] @@ -9706,7 +12763,7 @@ public partial class TranslationParallelCorpus } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TranslationParallelCorpusConfig + public partial class WordAlignmentParallelCorpusConfig { /// /// The corpus name. @@ -9725,7 +12782,7 @@ public partial class TranslationParallelCorpusConfig } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TranslationParallelCorpusUpdateConfig + public partial class WordAlignmentParallelCorpusUpdateConfig { [Newtonsoft.Json.JsonProperty("sourceCorpusIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public System.Collections.Generic.IList? SourceCorpusIds { get; set; } = default!; @@ -9736,7 +12793,7 @@ public partial class TranslationParallelCorpusUpdateConfig } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class Pretranslation + public partial class WordAlignment { [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] @@ -9746,47 +12803,26 @@ public partial class Pretranslation [System.ComponentModel.DataAnnotations.Required] public System.Collections.Generic.IList Refs { get; set; } = new System.Collections.ObjectModel.Collection(); - [Newtonsoft.Json.JsonProperty("translation", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Translation { get; set; } = default!; - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public enum PretranslationUsfmTextOrigin - { - - [System.Runtime.Serialization.EnumMember(Value = @"PreferExisting")] - PreferExisting = 0, - - [System.Runtime.Serialization.EnumMember(Value = @"PreferPretranslated")] - PreferPretranslated = 1, - - [System.Runtime.Serialization.EnumMember(Value = @"OnlyExisting")] - OnlyExisting = 2, - - [System.Runtime.Serialization.EnumMember(Value = @"OnlyPretranslated")] - OnlyPretranslated = 3, - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public enum PretranslationUsfmTemplate - { + [Newtonsoft.Json.JsonProperty("sourceTokens", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList SourceTokens { get; set; } = new System.Collections.ObjectModel.Collection(); - [System.Runtime.Serialization.EnumMember(Value = @"Auto")] - Auto = 0, + [Newtonsoft.Json.JsonProperty("targetTokens", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList TargetTokens { get; set; } = new System.Collections.ObjectModel.Collection(); - [System.Runtime.Serialization.EnumMember(Value = @"Source")] - Source = 1, + [Newtonsoft.Json.JsonProperty("confidences", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Confidences { get; set; } = new System.Collections.ObjectModel.Collection(); - [System.Runtime.Serialization.EnumMember(Value = @"Target")] - Target = 2, + [Newtonsoft.Json.JsonProperty("alignment", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Alignment { get; set; } = new System.Collections.ObjectModel.Collection(); } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TranslationBuild + public partial class WordAlignmentBuild { [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] @@ -9807,10 +12843,10 @@ public partial class TranslationBuild public ResourceLink Engine { get; set; } = new ResourceLink(); [Newtonsoft.Json.JsonProperty("trainOn", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? TrainOn { get; set; } = default!; + public System.Collections.Generic.IList? TrainOn { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("pretranslate", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? Pretranslate { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("wordAlignOn", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? WordAlignOn { get; set; } = default!; [Newtonsoft.Json.JsonProperty("step", Required = Newtonsoft.Json.Required.Always)] public int Step { get; set; } = default!; @@ -9838,36 +12874,24 @@ public partial class TranslationBuild [Newtonsoft.Json.JsonProperty("options", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public object? Options { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("deploymentVersion", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? DeploymentVersion { get; set; } = default!; - } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TrainingCorpus + public partial class TrainingCorpus2 { - [Newtonsoft.Json.JsonProperty("corpus", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public ResourceLink? Corpus { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? TextIds { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? ScriptureRange { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("parallelCorpus", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public ResourceLink? ParallelCorpus { get; set; } = default!; [Newtonsoft.Json.JsonProperty("sourceFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? SourceFilters { get; set; } = default!; + public System.Collections.Generic.IList? SourceFilters { get; set; } = default!; [Newtonsoft.Json.JsonProperty("targetFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? TargetFilters { get; set; } = default!; + public System.Collections.Generic.IList? TargetFilters { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class ParallelCorpusFilter + public partial class ParallelCorpusFilter2 { [Newtonsoft.Json.JsonProperty("corpus", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] @@ -9882,36 +12906,16 @@ public partial class ParallelCorpusFilter } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class PretranslateCorpus - { - [Newtonsoft.Json.JsonProperty("corpus", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public ResourceLink? Corpus { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? TextIds { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? ScriptureRange { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("parallelCorpus", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public ResourceLink? ParallelCorpus { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("sourceFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? SourceFilters { get; set; } = default!; - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TranslationBuildConfig + public partial class WordAlignmentBuildConfig { [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? Name { get; set; } = default!; [Newtonsoft.Json.JsonProperty("trainOn", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? TrainOn { get; set; } = default!; + public System.Collections.Generic.IList? TrainOn { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("pretranslate", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? Pretranslate { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("wordAlignOn", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? WordAlignOn { get; set; } = default!; [Newtonsoft.Json.JsonProperty("options", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public object? Options { get; set; } = default!; @@ -9919,30 +12923,21 @@ public partial class TranslationBuildConfig } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TrainingCorpusConfig + public partial class TrainingCorpusConfig2 { - [Newtonsoft.Json.JsonProperty("corpusId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? CorpusId { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? TextIds { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? ScriptureRange { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("parallelCorpusId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? ParallelCorpusId { get; set; } = default!; [Newtonsoft.Json.JsonProperty("sourceFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? SourceFilters { get; set; } = default!; + public System.Collections.Generic.IList? SourceFilters { get; set; } = default!; [Newtonsoft.Json.JsonProperty("targetFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? TargetFilters { get; set; } = default!; + public System.Collections.Generic.IList? TargetFilters { get; set; } = default!; } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class ParallelCorpusFilterConfig + public partial class ParallelCorpusFilterConfig2 { [Newtonsoft.Json.JsonProperty("corpusId", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] @@ -9956,134 +12951,6 @@ public partial class ParallelCorpusFilterConfig } - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class PretranslateCorpusConfig - { - [Newtonsoft.Json.JsonProperty("corpusId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? CorpusId { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? TextIds { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? ScriptureRange { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("parallelCorpusId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? ParallelCorpusId { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("sourceFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? SourceFilters { get; set; } = default!; - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class ModelDownloadUrl - { - [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Url { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("modelRevision", Required = Newtonsoft.Json.Required.Always)] - public int ModelRevision { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("expiresAt", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public System.DateTimeOffset ExpiresAt { get; set; } = default!; - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class Queue - { - [Newtonsoft.Json.JsonProperty("size", Required = Newtonsoft.Json.Required.Always)] - public int Size { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("engineType", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string EngineType { get; set; } = default!; - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class LanguageInfo - { - [Newtonsoft.Json.JsonProperty("engineType", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string EngineType { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("isNative", Required = Newtonsoft.Json.Required.Always)] - public bool IsNative { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("internalCode", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? InternalCode { get; set; } = default!; - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class Webhook - { - [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Id { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Url { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("payloadUrl", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string PayloadUrl { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("events", Required = Newtonsoft.Json.Required.Always, ItemConverterType = typeof(Newtonsoft.Json.Converters.StringEnumConverter))] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList Events { get; set; } = new System.Collections.ObjectModel.Collection(); - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public enum WebhookEvent - { - - [System.Runtime.Serialization.EnumMember(Value = @"TranslationBuildStarted")] - TranslationBuildStarted = 0, - - [System.Runtime.Serialization.EnumMember(Value = @"TranslationBuildFinished")] - TranslationBuildFinished = 1, - - [System.Runtime.Serialization.EnumMember(Value = @"AssessmentJobStarted")] - AssessmentJobStarted = 2, - - [System.Runtime.Serialization.EnumMember(Value = @"AssessmentJobFinished")] - AssessmentJobFinished = 3, - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class WebhookConfig - { - /// - /// The payload URL. - /// - [Newtonsoft.Json.JsonProperty("payloadUrl", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string PayloadUrl { get; set; } = default!; - - /// - /// The shared secret. - /// - [Newtonsoft.Json.JsonProperty("secret", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string Secret { get; set; } = default!; - - /// - /// The webhook events. - /// - [Newtonsoft.Json.JsonProperty("events", Required = Newtonsoft.Json.Required.Always, ItemConverterType = typeof(Newtonsoft.Json.Converters.StringEnumConverter))] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList Events { get; set; } = new System.Collections.ObjectModel.Collection(); - - } - [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] public partial class FileParameter { diff --git a/src/Serval/src/Serval.DataFiles/Controllers/CorporaController.cs b/src/Serval/src/Serval.DataFiles/Controllers/CorporaController.cs index 29bf041e..771ece32 100644 --- a/src/Serval/src/Serval.DataFiles/Controllers/CorporaController.cs +++ b/src/Serval/src/Serval.DataFiles/Controllers/CorporaController.cs @@ -166,18 +166,18 @@ private async Task MapAsync(CorpusConfigDto corpusConfig, string id, Can }; } - private async Task> MapAsync( + private async Task> MapAsync( IReadOnlyList files, CancellationToken cancellationToken ) { - var dataFiles = new List(); + var dataFiles = new List(); foreach (CorpusFileConfigDto file in files) { DataFile? dataFile = await _dataFileService.GetAsync(file.FileId, cancellationToken); if (dataFile == null) throw new InvalidOperationException($"DataFile with id {file.FileId} does not exist."); - dataFiles.Add(new CorpusFile { File = dataFile, TextId = file.TextId }); + dataFiles.Add(new Models.CorpusFile { File = dataFile, TextId = file.TextId }); } return dataFiles; } @@ -195,7 +195,7 @@ private CorpusDto Map(Corpus source) }; } - private CorpusFileDto Map(CorpusFile source) + private CorpusFileDto Map(Models.CorpusFile source) { return new CorpusFileDto { File = Map(source.File), TextId = source.TextId }; } diff --git a/src/Serval/src/Serval.DataFiles/Services/CorpusService.cs b/src/Serval/src/Serval.DataFiles/Services/CorpusService.cs index f5b8e4b6..a092f45b 100644 --- a/src/Serval/src/Serval.DataFiles/Services/CorpusService.cs +++ b/src/Serval/src/Serval.DataFiles/Services/CorpusService.cs @@ -12,7 +12,7 @@ public async Task GetAsync(string id, string owner, CancellationToken ca public async Task UpdateAsync( string id, - IReadOnlyList files, + IReadOnlyList files, CancellationToken cancellationToken = default ) { diff --git a/src/Serval/src/Serval.DataFiles/Services/ICorpusService.cs b/src/Serval/src/Serval.DataFiles/Services/ICorpusService.cs index a4f0e242..d5aa3645 100644 --- a/src/Serval/src/Serval.DataFiles/Services/ICorpusService.cs +++ b/src/Serval/src/Serval.DataFiles/Services/ICorpusService.cs @@ -6,6 +6,10 @@ public interface ICorpusService Task GetAsync(string id, CancellationToken cancellationToken = default); Task GetAsync(string id, string owner, CancellationToken cancellationToken = default); Task CreateAsync(Corpus corpus, CancellationToken cancellationToken = default); - Task UpdateAsync(string id, IReadOnlyList files, CancellationToken cancellationToken = default); + Task UpdateAsync( + string id, + IReadOnlyList files, + CancellationToken cancellationToken = default + ); Task DeleteAsync(string id, CancellationToken cancellationToken = default); } diff --git a/src/Serval/src/Serval.Grpc/Protos/serval/word_alignment/v1/engine.proto b/src/Serval/src/Serval.Grpc/Protos/serval/word_alignment/v1/engine.proto new file mode 100644 index 00000000..8fae85e1 --- /dev/null +++ b/src/Serval/src/Serval.Grpc/Protos/serval/word_alignment/v1/engine.proto @@ -0,0 +1,104 @@ +syntax = "proto3"; + +package serval.word_alignment.v1; + +import "google/protobuf/empty.proto"; + +service WordAlignmentEngineApi { + rpc Create(CreateRequest) returns (google.protobuf.Empty); + rpc Delete(DeleteRequest) returns (google.protobuf.Empty); + rpc GetWordAlignment(GetWordAlignmentRequest) returns (GetWordAlignmentResponse); + rpc StartBuild(StartBuildRequest) returns (google.protobuf.Empty); + rpc CancelBuild(CancelBuildRequest) returns (google.protobuf.Empty); + rpc GetQueueSize(GetQueueSizeRequest) returns (GetQueueSizeResponse); +} + +message CreateRequest { + string engine_type = 1; + string engine_id = 2; + optional string engine_name = 3; + string source_language = 4; + string target_language = 5; +} + +message DeleteRequest { + string engine_type = 1; + string engine_id = 2; +} + +message GetWordAlignmentRequest { + string engine_type = 1; + string engine_id = 2; + string source_segment = 3; + string target_segment = 4; +} + +message GetWordAlignmentResponse { + WordAlignmentResult result = 1; +} + +message StartBuildRequest { + string engine_type = 1; + string engine_id = 2; + string build_id = 3; + optional string options = 4; + repeated ParallelCorpus corpora = 5; +} + +message CancelBuildRequest { + string engine_type = 1; + string engine_id = 2; +} + +message GetQueueSizeRequest { + string engine_type = 1; +} + +message GetQueueSizeResponse { + int32 size = 1; +} + +message AlignedWordPair { + int32 source_index = 1; + int32 target_index = 2; +} + +message WordAlignmentResult { + repeated string source_tokens = 1; + repeated string target_tokens = 2; + repeated double confidences = 3; + repeated AlignedWordPair alignment = 4; +} + +message ParallelCorpus { + string id = 1; + repeated MonolingualCorpus source_corpora = 2; + repeated MonolingualCorpus target_corpora = 3; +} + +message MonolingualCorpus { + string id = 1; + string language = 2; + bool train_on_all = 3; + bool word_align_on_all = 4; + map train_on_chapters = 5; + map word_align_on_chapters = 6; + repeated string train_on_text_ids = 7; + repeated string word_align_on_text_ids = 8; + repeated CorpusFile files = 9; +} + +message ScriptureChapters { + repeated int32 chapters = 1; +} + +message CorpusFile { + string location = 1; + FileFormat format = 2; + string text_id = 3; +} + +enum FileFormat { + FILE_FORMAT_TEXT = 0; + FILE_FORMAT_PARATEXT = 1; +} diff --git a/src/Serval/src/Serval.Grpc/Protos/serval/word_alignment/v1/platform.proto b/src/Serval/src/Serval.Grpc/Protos/serval/word_alignment/v1/platform.proto new file mode 100644 index 00000000..ad5c1430 --- /dev/null +++ b/src/Serval/src/Serval.Grpc/Protos/serval/word_alignment/v1/platform.proto @@ -0,0 +1,66 @@ +syntax = "proto3"; + +package serval.word_alignment.v1; + +import "google/protobuf/empty.proto"; +import "Protos/serval/word_alignment/v1/engine.proto"; + + +service WordAlignmentPlatformApi { + rpc UpdateBuildStatus(UpdateBuildStatusRequest) returns (google.protobuf.Empty); + rpc BuildStarted(BuildStartedRequest) returns (google.protobuf.Empty); + rpc BuildCompleted(BuildCompletedRequest) returns (google.protobuf.Empty); + rpc BuildCanceled(BuildCanceledRequest) returns (google.protobuf.Empty); + rpc BuildFaulted(BuildFaultedRequest) returns (google.protobuf.Empty); + rpc BuildRestarting(BuildRestartingRequest) returns (google.protobuf.Empty); + + rpc IncrementWordAlignmentEngineCorpusSize(IncrementWordAlignmentEngineCorpusSizeRequest) returns (google.protobuf.Empty); + rpc InsertWordAlignments(stream InsertWordAlignmentsRequest) returns (google.protobuf.Empty); +} + +message UpdateBuildStatusRequest { + string build_id = 1; + int32 step = 2; + optional double percent_completed = 3; + optional string message = 4; + optional int32 queue_depth = 5; +} + +message BuildStartedRequest { + string build_id = 1; +} + +message BuildCompletedRequest { + string build_id = 1; + int32 corpus_size = 2; + double confidence = 3; +} + +message BuildCanceledRequest { + string build_id = 1; +} + +message BuildFaultedRequest { + string build_id = 1; + string message = 2; +} + +message BuildRestartingRequest { + string build_id = 1; +} + +message IncrementWordAlignmentEngineCorpusSizeRequest { + string engine_id = 1; + int32 count = 2; +} + +message InsertWordAlignmentsRequest { + string engine_id = 1; + string corpus_id = 2; + string text_id = 3; + repeated string refs = 4; + repeated string source_tokens = 5; + repeated string target_tokens = 6; + repeated double confidences = 7; + repeated AlignedWordPair alignment = 8; +} diff --git a/src/Serval/src/Serval.Translation/Contracts/AlignedWordPairDto.cs b/src/Serval/src/Serval.Shared/Contracts/AlignedWordPairDto.cs similarity index 76% rename from src/Serval/src/Serval.Translation/Contracts/AlignedWordPairDto.cs rename to src/Serval/src/Serval.Shared/Contracts/AlignedWordPairDto.cs index 4cd0dd66..5b9c63f6 100644 --- a/src/Serval/src/Serval.Translation/Contracts/AlignedWordPairDto.cs +++ b/src/Serval/src/Serval.Shared/Contracts/AlignedWordPairDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Shared.Contracts; public record AlignedWordPairDto { diff --git a/src/Serval/src/Serval.Translation/Models/ParallelCorpusFilter.cs b/src/Serval/src/Serval.Shared/Contracts/ParallelCorpusFilter.cs similarity index 82% rename from src/Serval/src/Serval.Translation/Models/ParallelCorpusFilter.cs rename to src/Serval/src/Serval.Shared/Contracts/ParallelCorpusFilter.cs index 1cb311e8..184aee25 100644 --- a/src/Serval/src/Serval.Translation/Models/ParallelCorpusFilter.cs +++ b/src/Serval/src/Serval.Shared/Contracts/ParallelCorpusFilter.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Shared.Contracts; public record ParallelCorpusFilter { diff --git a/src/Serval/src/Serval.Translation/Contracts/QueueDto.cs b/src/Serval/src/Serval.Shared/Contracts/QueueDto.cs similarity index 75% rename from src/Serval/src/Serval.Translation/Contracts/QueueDto.cs rename to src/Serval/src/Serval.Shared/Contracts/QueueDto.cs index 51c75357..ade49773 100644 --- a/src/Serval/src/Serval.Translation/Contracts/QueueDto.cs +++ b/src/Serval/src/Serval.Shared/Contracts/QueueDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Shared.Contracts; public record QueueDto { diff --git a/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildFinished.cs b/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildFinished.cs new file mode 100644 index 00000000..44886a44 --- /dev/null +++ b/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildFinished.cs @@ -0,0 +1,11 @@ +namespace Serval.Shared.Contracts; + +public record WordAlignmentBuildFinished +{ + public required string BuildId { get; init; } + public required string EngineId { get; init; } + public required string Owner { get; init; } + public required JobState BuildState { get; init; } + public required string Message { get; init; } + public required DateTime DateFinished { get; init; } +} diff --git a/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildFinishedDto.cs b/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildFinishedDto.cs new file mode 100644 index 00000000..f7ec9936 --- /dev/null +++ b/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildFinishedDto.cs @@ -0,0 +1,10 @@ +namespace Serval.Shared.Contracts; + +public record WordAlignmentBuildFinishedDto +{ + public required ResourceLinkDto Build { get; init; } + public required ResourceLinkDto Engine { get; init; } + public required JobState BuildState { get; init; } + public required string Message { get; init; } + public required DateTime DateFinished { get; init; } +} diff --git a/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildStarted.cs b/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildStarted.cs new file mode 100644 index 00000000..08a6375f --- /dev/null +++ b/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildStarted.cs @@ -0,0 +1,8 @@ +namespace Serval.Shared.Contracts; + +public record WordAlignmentBuildStarted +{ + public required string BuildId { get; init; } + public required string EngineId { get; init; } + public required string Owner { get; init; } +} diff --git a/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildStartedDto.cs b/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildStartedDto.cs new file mode 100644 index 00000000..00f1c178 --- /dev/null +++ b/src/Serval/src/Serval.Shared/Contracts/WordAlignmentBuildStartedDto.cs @@ -0,0 +1,7 @@ +namespace Serval.Shared.Contracts; + +public record WordAlignmentBuildStartedDto +{ + public required ResourceLinkDto Build { get; init; } + public required ResourceLinkDto Engine { get; init; } +} diff --git a/src/Serval/src/Serval.Shared/Controllers/Endpoints.cs b/src/Serval/src/Serval.Shared/Controllers/Endpoints.cs index e8e147a0..c58e4d44 100644 --- a/src/Serval/src/Serval.Shared/Controllers/Endpoints.cs +++ b/src/Serval/src/Serval.Shared/Controllers/Endpoints.cs @@ -13,6 +13,9 @@ public static class Endpoints public const string GetAssessmentCorpus = "GetAssessmentCorpus"; public const string GetAssessmentReferenceCorpus = "GetAssessmentReferenceCorpus"; public const string GetAssessmentJob = "GetAssessmentJob"; + public const string GetWordAlignmentEngine = "GetWordAlignmentEngine"; + public const string GetParallelWordAlignmentCorpus = "GetParallelWordAlignmentCorpus"; + public const string GetWordAlignmentBuild = "GetWordAlignmentBuild"; public const string GetWebhook = "GetWebhook"; diff --git a/src/Serval/src/Serval.Shared/Controllers/Scopes.cs b/src/Serval/src/Serval.Shared/Controllers/Scopes.cs index b94ccf87..b2435809 100644 --- a/src/Serval/src/Serval.Shared/Controllers/Scopes.cs +++ b/src/Serval/src/Serval.Shared/Controllers/Scopes.cs @@ -12,6 +12,11 @@ public static class Scopes public const string UpdateAssessmentEngines = "update:assessment_engines"; public const string DeleteAssessmentEngines = "delete:assessment_engines"; + public const string CreateWordAlignmentEngines = "create:word_alignment_engines"; + public const string ReadWordAlignmentEngines = "read:word_alignment_engines"; + public const string UpdateWordAlignmentEngines = "update:word_alignment_engines"; + public const string DeleteWordAlignmentEngines = "delete:word_alignment_engines"; + public const string CreateHooks = "create:hooks"; public const string ReadHooks = "read:hooks"; public const string DeleteHooks = "delete:hooks"; @@ -33,6 +38,10 @@ public static class Scopes ReadAssessmentEngines, UpdateAssessmentEngines, DeleteAssessmentEngines, + CreateWordAlignmentEngines, + ReadWordAlignmentEngines, + UpdateWordAlignmentEngines, + DeleteWordAlignmentEngines, CreateHooks, ReadHooks, DeleteHooks, diff --git a/src/Serval/src/Serval.Translation/Models/AlignedWordPair.cs b/src/Serval/src/Serval.Shared/Models/AlignedWordPair.cs similarity index 77% rename from src/Serval/src/Serval.Translation/Models/AlignedWordPair.cs rename to src/Serval/src/Serval.Shared/Models/AlignedWordPair.cs index 5e367495..550abeb0 100644 --- a/src/Serval/src/Serval.Translation/Models/AlignedWordPair.cs +++ b/src/Serval/src/Serval.Shared/Models/AlignedWordPair.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Models; +namespace Serval.Shared.Models; public record AlignedWordPair { diff --git a/src/Serval/src/Serval.Translation/Models/CorpusFile.cs b/src/Serval/src/Serval.Shared/Models/CorpusFile.cs similarity index 84% rename from src/Serval/src/Serval.Translation/Models/CorpusFile.cs rename to src/Serval/src/Serval.Shared/Models/CorpusFile.cs index 2672ba56..2739e605 100644 --- a/src/Serval/src/Serval.Translation/Models/CorpusFile.cs +++ b/src/Serval/src/Serval.Shared/Models/CorpusFile.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Models; +namespace Serval.Shared.Models; public record CorpusFile { diff --git a/src/Serval/src/Serval.Translation/Models/MonolingualCorpus.cs b/src/Serval/src/Serval.Shared/Models/MonolingualCorpus.cs similarity index 86% rename from src/Serval/src/Serval.Translation/Models/MonolingualCorpus.cs rename to src/Serval/src/Serval.Shared/Models/MonolingualCorpus.cs index 0762e878..f9c58fb4 100644 --- a/src/Serval/src/Serval.Translation/Models/MonolingualCorpus.cs +++ b/src/Serval/src/Serval.Shared/Models/MonolingualCorpus.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Models; +namespace Serval.Shared.Models; public record MonolingualCorpus { diff --git a/src/Serval/src/Serval.Translation/Models/Queue.cs b/src/Serval/src/Serval.Shared/Models/Queue.cs similarity index 76% rename from src/Serval/src/Serval.Translation/Models/Queue.cs rename to src/Serval/src/Serval.Shared/Models/Queue.cs index 3e269b2c..7a3272a8 100644 --- a/src/Serval/src/Serval.Translation/Models/Queue.cs +++ b/src/Serval/src/Serval.Shared/Models/Queue.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Models; +namespace Serval.Shared.Models; public record Queue { diff --git a/src/Serval/src/Serval.Shared/Usings.cs b/src/Serval/src/Serval.Shared/Usings.cs index 3e84144f..fb72d2d0 100644 --- a/src/Serval/src/Serval.Shared/Usings.cs +++ b/src/Serval/src/Serval.Shared/Usings.cs @@ -12,6 +12,7 @@ global using Microsoft.Extensions.Logging; global using Microsoft.Extensions.Options; global using Serval.Shared.Configuration; +global using Serval.Shared.Contracts; global using Serval.Shared.Models; global using Serval.Shared.Services; global using Serval.Shared.Utils; diff --git a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs b/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs index aeb87b96..b992c548 100644 --- a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs +++ b/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs @@ -1017,6 +1017,7 @@ CancellationToken cancellationToken /// /// The `"options"` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. /// See [nmt job settings documentation](https://github.com/sillsdev/serval/wiki/NMT-Build-Options) about configuring job parameters. + /// See [smt-transfer job settings documentation](https://github.com/sillsdev/serval/wiki/SMT-Transfer-Build-Options) about configuring job parameters. /// See [keyterms parsing documentation](https://github.com/sillsdev/serval/wiki/Paratext-Key-Terms-Parsing) on how to use keyterms for training. /// /// When using a parallel corpus: diff --git a/src/Serval/src/Serval.Translation/Services/EngineService.cs b/src/Serval/src/Serval.Translation/Services/EngineService.cs index 443b2d23..c81510ae 100644 --- a/src/Serval/src/Serval.Translation/Services/EngineService.cs +++ b/src/Serval/src/Serval.Translation/Services/EngineService.cs @@ -388,8 +388,8 @@ public Task AddCorpusAsync(string engineId, Models.Corpus corpus, CancellationTo public async Task UpdateCorpusAsync( string engineId, string corpusId, - IReadOnlyList? sourceFiles, - IReadOnlyList? targetFiles, + IReadOnlyList? sourceFiles, + IReadOnlyList? targetFiles, CancellationToken cancellationToken = default ) { @@ -465,8 +465,8 @@ public Task AddParallelCorpusAsync( public async Task UpdateParallelCorpusAsync( string engineId, string parallelCorpusId, - IReadOnlyList? sourceCorpora, - IReadOnlyList? targetCorpora, + IReadOnlyList? sourceCorpora, + IReadOnlyList? targetCorpora, CancellationToken cancellationToken = default ) { @@ -583,9 +583,9 @@ private Models.TranslationResult Map(V1.TranslationResult source) return source.Values.Cast().ToHashSet(); } - private Models.AlignedWordPair Map(V1.AlignedWordPair source) + private Shared.Models.AlignedWordPair Map(V1.AlignedWordPair source) { - return new Models.AlignedWordPair { SourceIndex = source.SourceIndex, TargetIndex = source.TargetIndex }; + return new Shared.Models.AlignedWordPair { SourceIndex = source.SourceIndex, TargetIndex = source.TargetIndex }; } private Models.Phrase Map(V1.Phrase source) @@ -790,7 +790,7 @@ bool pretranslateOnAllCorpora } private V1.MonolingualCorpus Map( - Models.MonolingualCorpus inputCorpus, + Shared.Models.MonolingualCorpus inputCorpus, ParallelCorpusFilter? trainingFilter, ParallelCorpusFilter? pretranslateFilter, string? referenceFileLocation, @@ -880,7 +880,7 @@ pretranslateFilter is not null return returnCorpus; } - private V1.CorpusFile Map(Models.CorpusFile source) + private V1.CorpusFile Map(Shared.Models.CorpusFile source) { return new V1.CorpusFile { diff --git a/src/Serval/src/Serval.Webhooks/Configuration/IMediatorRegistrationConfiguratorExtensions.cs b/src/Serval/src/Serval.Webhooks/Configuration/IMediatorRegistrationConfiguratorExtensions.cs index 37e15a45..3544dd17 100644 --- a/src/Serval/src/Serval.Webhooks/Configuration/IMediatorRegistrationConfiguratorExtensions.cs +++ b/src/Serval/src/Serval.Webhooks/Configuration/IMediatorRegistrationConfiguratorExtensions.cs @@ -10,6 +10,8 @@ this IMediatorRegistrationConfigurator configurator configurator.AddConsumer(); configurator.AddConsumer(); configurator.AddConsumer(); + configurator.AddConsumer(); + configurator.AddConsumer(); return configurator; } } diff --git a/src/Serval/src/Serval.Webhooks/Consumers/WordAlignmentBuildFinishedConsumer.cs b/src/Serval/src/Serval.Webhooks/Consumers/WordAlignmentBuildFinishedConsumer.cs new file mode 100644 index 00000000..de604125 --- /dev/null +++ b/src/Serval/src/Serval.Webhooks/Consumers/WordAlignmentBuildFinishedConsumer.cs @@ -0,0 +1,36 @@ +namespace Serval.Webhooks.Consumers; + +public class WordAlignmentBuildFinishedConsumer(IWebhookService webhookService, IUrlService urlService) + : IConsumer +{ + private readonly IWebhookService _webhookService = webhookService; + private readonly IUrlService _urlService = urlService; + + public async Task Consume(ConsumeContext context) + { + await _webhookService.SendEventAsync( + WebhookEvent.WordAlignmentBuildFinished, + context.Message.Owner, + new WordAlignmentBuildFinishedDto + { + Build = new ResourceLinkDto + { + Id = context.Message.BuildId, + Url = _urlService.GetUrl( + Endpoints.GetWordAlignmentBuild, + new { id = context.Message.EngineId, buildId = context.Message.BuildId } + ) + }, + Engine = new ResourceLinkDto + { + Id = context.Message.EngineId, + Url = _urlService.GetUrl(Endpoints.GetWordAlignmentEngine, new { id = context.Message.EngineId })! + }, + BuildState = context.Message.BuildState, + Message = context.Message.Message, + DateFinished = context.Message.DateFinished + }, + context.CancellationToken + ); + } +} diff --git a/src/Serval/src/Serval.Webhooks/Consumers/WordAlignmentBuildStartedConsumer.cs b/src/Serval/src/Serval.Webhooks/Consumers/WordAlignmentBuildStartedConsumer.cs new file mode 100644 index 00000000..5fb11b41 --- /dev/null +++ b/src/Serval/src/Serval.Webhooks/Consumers/WordAlignmentBuildStartedConsumer.cs @@ -0,0 +1,33 @@ +namespace Serval.Webhooks.Consumers; + +public class WordAlignmentBuildStartedConsumer(IWebhookService webhookService, IUrlService urlService) + : IConsumer +{ + private readonly IWebhookService _webhookService = webhookService; + private readonly IUrlService _urlService = urlService; + + public async Task Consume(ConsumeContext context) + { + await _webhookService.SendEventAsync( + WebhookEvent.WordAlignmentBuildStarted, + context.Message.Owner, + new WordAlignmentBuildStartedDto + { + Build = new ResourceLinkDto + { + Id = context.Message.BuildId, + Url = _urlService.GetUrl( + Endpoints.GetWordAlignmentBuild, + new { id = context.Message.EngineId, buildId = context.Message.BuildId } + ) + }, + Engine = new ResourceLinkDto + { + Id = context.Message.EngineId, + Url = _urlService.GetUrl(Endpoints.GetWordAlignmentEngine, new { id = context.Message.EngineId }) + } + }, + context.CancellationToken + ); + } +} diff --git a/src/Serval/src/Serval.Webhooks/Contracts/WebhookEvent.cs b/src/Serval/src/Serval.Webhooks/Contracts/WebhookEvent.cs index 1dc5d0a4..a031f4d6 100644 --- a/src/Serval/src/Serval.Webhooks/Contracts/WebhookEvent.cs +++ b/src/Serval/src/Serval.Webhooks/Contracts/WebhookEvent.cs @@ -6,5 +6,7 @@ public enum WebhookEvent TranslationBuildFinished, AssessmentJobStarted, - AssessmentJobFinished + AssessmentJobFinished, + WordAlignmentBuildStarted, + WordAlignmentBuildFinished } diff --git a/src/Serval/src/Serval.WordAlignment/Configuration/IEndpointRouteBuilderExtensions.cs b/src/Serval/src/Serval.WordAlignment/Configuration/IEndpointRouteBuilderExtensions.cs new file mode 100644 index 00000000..84d4d6a5 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Configuration/IEndpointRouteBuilderExtensions.cs @@ -0,0 +1,11 @@ +namespace Microsoft.AspNetCore.Builder; + +public static class IEndpointRouteBuilderExtensions +{ + public static IEndpointRouteBuilder MapServalWordAlignmentServices(this IEndpointRouteBuilder builder) + { + builder.MapGrpcService(); + + return builder; + } +} diff --git a/src/Serval/src/Serval.WordAlignment/Configuration/IMediatorRegistrationConfiguratorExtensions.cs b/src/Serval/src/Serval.WordAlignment/Configuration/IMediatorRegistrationConfiguratorExtensions.cs new file mode 100644 index 00000000..4583f4b7 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Configuration/IMediatorRegistrationConfiguratorExtensions.cs @@ -0,0 +1,12 @@ +namespace Microsoft.Extensions.DependencyInjection; + +public static class IMediatorRegistrationConfiguratorExtensions +{ + public static IMediatorRegistrationConfigurator AddWordAlignmentConsumers( + this IMediatorRegistrationConfigurator configurator + ) + { + configurator.AddConsumer(); + return configurator; + } +} diff --git a/src/Serval/src/Serval.WordAlignment/Configuration/IMemoryDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.WordAlignment/Configuration/IMemoryDataAccessConfiguratorExtensions.cs new file mode 100644 index 00000000..3edb081f --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Configuration/IMemoryDataAccessConfiguratorExtensions.cs @@ -0,0 +1,14 @@ +namespace Microsoft.Extensions.DependencyInjection; + +public static class IMemoryDataAccessConfiguratorExtensions +{ + public static IMemoryDataAccessConfigurator AddWordAlignmentRepositories( + this IMemoryDataAccessConfigurator configurator + ) + { + configurator.AddRepository(); + configurator.AddRepository(); + configurator.AddRepository(); + return configurator; + } +} diff --git a/src/Serval/src/Serval.WordAlignment/Configuration/IMongoDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.WordAlignment/Configuration/IMongoDataAccessConfiguratorExtensions.cs new file mode 100644 index 00000000..76e6a10f --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Configuration/IMongoDataAccessConfiguratorExtensions.cs @@ -0,0 +1,62 @@ +using MongoDB.Driver; + +namespace Microsoft.Extensions.DependencyInjection; + +public static class IMongoDataAccessConfiguratorExtensions +{ + public static IMongoDataAccessConfigurator AddWordAlignmentRepositories( + this IMongoDataAccessConfigurator configurator + ) + { + configurator.AddRepository( + "word_alignment.engines", + init: async c => + { + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(e => e.Owner)) + ); + } + ); + configurator.AddRepository( + "word_alignment.builds", + init: c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(b => b.EngineRef)) + ) + ); + configurator.AddRepository( + "word_alignment.pretranslations", + init: async c => + { + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(pt => pt.ModelRevision) + ) + ); + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(pt => pt.CorpusRef)) + ); + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(pt => pt.TextId)) + ); + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders + .IndexKeys.Ascending(pt => pt.EngineRef) + .Ascending(pt => pt.ModelRevision) + ) + ); + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders + .IndexKeys.Ascending(pt => pt.EngineRef) + .Ascending(pt => pt.CorpusRef) + .Ascending(pt => pt.ModelRevision) + .Ascending(pt => pt.TextId) + ) + ); + } + ); + return configurator; + } +} diff --git a/src/Serval/src/Serval.WordAlignment/Configuration/IServalBuilderExtensions.cs b/src/Serval/src/Serval.WordAlignment/Configuration/IServalBuilderExtensions.cs new file mode 100644 index 00000000..94ee2e99 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Configuration/IServalBuilderExtensions.cs @@ -0,0 +1,48 @@ +using Serval.Health.V1; +using Serval.WordAlignment.V1; + +namespace Microsoft.Extensions.DependencyInjection; + +public static class IServalBuilderExtensions +{ + public static IServalBuilder AddWordAlignment( + this IServalBuilder builder, + Action? configure = null + ) + { + if (builder.Configuration is null) + { + builder.AddApiOptions(o => { }); + builder.AddDataFileOptions(o => { }); + } + else + { + builder.AddApiOptions(builder.Configuration.GetSection(ApiOptions.Key)); + builder.AddDataFileOptions(builder.Configuration.GetSection(DataFileOptions.Key)); + } + + builder.Services.AddScoped(); + builder.Services.AddScoped(); + builder.Services.AddScoped(); + + var wordAlignmentOptions = new WordAlignmentOptions(); + builder.Configuration?.GetSection(WordAlignmentOptions.Key).Bind(wordAlignmentOptions); + if (configure is not null) + configure(wordAlignmentOptions); + + foreach (EngineInfo engine in wordAlignmentOptions.Engines) + { + builder.Services.AddGrpcClient( + engine.Type, + o => o.Address = new Uri(engine.Address) + ); + builder.Services.AddGrpcClient( + $"{engine.Type}-Health", + o => o.Address = new Uri(engine.Address) + ); + builder.Services.AddHealthChecks().AddCheck(engine.Type); + } + + return builder; + } +} diff --git a/src/Serval/src/Serval.WordAlignment/Configuration/WordAlignmentOptions.cs b/src/Serval/src/Serval.WordAlignment/Configuration/WordAlignmentOptions.cs new file mode 100644 index 00000000..ac6cf3e3 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Configuration/WordAlignmentOptions.cs @@ -0,0 +1,14 @@ +namespace Serval.WordAlignment.Configuration; + +public class WordAlignmentOptions +{ + public const string Key = "WordAlignment"; + + public List Engines { get; set; } = new List(); +} + +public class EngineInfo +{ + public string Type { get; set; } = ""; + public string Address { get; set; } = ""; +} diff --git a/src/Serval/src/Serval.WordAlignment/Consumers/DataFileDeletedConsumer.cs b/src/Serval/src/Serval.WordAlignment/Consumers/DataFileDeletedConsumer.cs new file mode 100644 index 00000000..c6ab6f3a --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Consumers/DataFileDeletedConsumer.cs @@ -0,0 +1,11 @@ +namespace Serval.WordAlignment.Consumers; + +public class DataFileDeletedConsumer(IEngineService engineService) : IConsumer +{ + private readonly IEngineService _engineService = engineService; + + public async Task Consume(ConsumeContext context) + { + await _engineService.DeleteAllCorpusFilesAsync(context.Message.DataFileId, context.CancellationToken); + } +} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/ParallelCorpusFilterConfigDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/ParallelCorpusFilterConfigDto.cs new file mode 100644 index 00000000..c1286abf --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Contracts/ParallelCorpusFilterConfigDto.cs @@ -0,0 +1,8 @@ +namespace Serval.WordAlignment.Contracts; + +public record ParallelCorpusFilterConfigDto +{ + public required string CorpusId { get; init; } + public IReadOnlyList? TextIds { get; init; } + public string? ScriptureRange { get; init; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/ParallelCorpusFilterDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/ParallelCorpusFilterDto.cs new file mode 100644 index 00000000..0824fe95 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Contracts/ParallelCorpusFilterDto.cs @@ -0,0 +1,8 @@ +namespace Serval.WordAlignment.Contracts; + +public record ParallelCorpusFilterDto +{ + public required ResourceLinkDto Corpus { get; init; } + public IReadOnlyList? TextIds { get; init; } + public string? ScriptureRange { get; init; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/TrainingCorpusConfigDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/TrainingCorpusConfigDto.cs new file mode 100644 index 00000000..bfe1c3e5 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Contracts/TrainingCorpusConfigDto.cs @@ -0,0 +1,8 @@ +namespace Serval.WordAlignment.Contracts; + +public record TrainingCorpusConfigDto +{ + public string? ParallelCorpusId { get; init; } + public IReadOnlyList? SourceFilters { get; init; } + public IReadOnlyList? TargetFilters { get; init; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/TrainingCorpusDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/TrainingCorpusDto.cs new file mode 100644 index 00000000..de110ceb --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Contracts/TrainingCorpusDto.cs @@ -0,0 +1,8 @@ +namespace Serval.WordAlignment.Contracts; + +public record TrainingCorpusDto +{ + public ResourceLinkDto? ParallelCorpus { get; init; } + public IReadOnlyList? SourceFilters { get; init; } + public IReadOnlyList? TargetFilters { get; init; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentBuildConfigDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentBuildConfigDto.cs new file mode 100644 index 00000000..3a79b0e7 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentBuildConfigDto.cs @@ -0,0 +1,15 @@ +namespace Serval.WordAlignment.Contracts; + +public record WordAlignmentBuildConfigDto +{ + public string? Name { get; init; } + public IReadOnlyList? TrainOn { get; init; } + public IReadOnlyList? WordAlignOn { get; init; } + + /// + /// { + /// "property" : "value" + /// } + /// + public object? Options { get; init; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentBuildDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentBuildDto.cs new file mode 100644 index 00000000..ce109806 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentBuildDto.cs @@ -0,0 +1,30 @@ +namespace Serval.WordAlignment.Contracts; + +public record WordAlignmentBuildDto +{ + public required string Id { get; init; } + public required string Url { get; init; } + public required int Revision { get; init; } + public string? Name { get; init; } + public required ResourceLinkDto Engine { get; init; } + public IReadOnlyList? TrainOn { get; init; } + public IReadOnlyList? WordAlignOn { get; init; } + public required int Step { get; init; } + public double? PercentCompleted { get; init; } + public string? Message { get; init; } + + public int? QueueDepth { get; init; } + + /// + /// The current build job state. + /// + public required JobState State { get; init; } + public DateTime? DateFinished { get; init; } + + /// + /// { + /// "property" : "value" + /// } + /// + public object? Options { get; init; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentDto.cs new file mode 100644 index 00000000..01da4cc7 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentDto.cs @@ -0,0 +1,11 @@ +namespace Serval.WordAlignment.Contracts; + +public record WordAlignmentDto +{ + public required string TextId { get; init; } + public required IReadOnlyList Refs { get; init; } + public required IReadOnlyList SourceTokens { get; init; } + public required IReadOnlyList TargetTokens { get; init; } + public required IReadOnlyList Confidences { get; init; } + public required IReadOnlyList Alignment { get; init; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentEngineConfigDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentEngineConfigDto.cs new file mode 100644 index 00000000..54a242b0 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentEngineConfigDto.cs @@ -0,0 +1,24 @@ +namespace Serval.WordAlignment.Contracts; + +public record WordAlignmentEngineConfigDto +{ + /// + /// The word alignment engine name. + /// + public string? Name { get; init; } + + /// + /// The source language tag. + /// + public required string SourceLanguage { get; init; } + + /// + /// The target language tag. + /// + public required string TargetLanguage { get; init; } + + /// + /// The translation engine type. + /// + public required string Type { get; init; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentEngineDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentEngineDto.cs new file mode 100644 index 00000000..ebccc8f4 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentEngineDto.cs @@ -0,0 +1,15 @@ +namespace Serval.WordAlignment.Contracts; + +public record WordAlignmentEngineDto +{ + public required string Id { get; init; } + public required string Url { get; init; } + public string? Name { get; init; } + public required string SourceLanguage { get; init; } + public required string TargetLanguage { get; init; } + public required string Type { get; init; } + public required bool IsBuilding { get; init; } + public required int ModelRevision { get; init; } + public required double Confidence { get; init; } + public required int CorpusSize { get; init; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusConfigDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusConfigDto.cs new file mode 100644 index 00000000..6aa341c4 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusConfigDto.cs @@ -0,0 +1,12 @@ +namespace Serval.WordAlignment.Contracts; + +public record WordAlignmentParallelCorpusConfigDto +{ + /// + /// The corpus name. + /// + public string? Name { get; init; } + + public required IReadOnlyList SourceCorpusIds { get; init; } = new List(); + public required IReadOnlyList TargetCorpusIds { get; init; } = new List(); +} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusDto.cs new file mode 100644 index 00000000..53f00302 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusDto.cs @@ -0,0 +1,10 @@ +namespace Serval.WordAlignment.Contracts; + +public record WordAlignmentParallelCorpusDto +{ + public required string Id { get; init; } + public required string Url { get; init; } + public required ResourceLinkDto Engine { get; init; } + public required IReadOnlyList SourceCorpora { get; init; } + public required IReadOnlyList TargetCorpora { get; init; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusUpdateDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusUpdateDto.cs new file mode 100644 index 00000000..5a966947 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusUpdateDto.cs @@ -0,0 +1,23 @@ +using System.ComponentModel.DataAnnotations; + +namespace Serval.WordAlignment.Contracts; + +public record WordAlignmentParallelCorpusUpdateConfigDto : IValidatableObject +{ + public IReadOnlyList? SourceCorpusIds { get; init; } + + public IReadOnlyList? TargetCorpusIds { get; init; } + + public IEnumerable Validate( + ValidationContext validationContext + ) + { + if (SourceCorpusIds is null && TargetCorpusIds is null) + { + yield return new System.ComponentModel.DataAnnotations.ValidationResult( + "At least one field must be specified.", + [nameof(SourceCorpusIds), nameof(TargetCorpusIds)] + ); + } + } +} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentRequestDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentRequestDto.cs new file mode 100644 index 00000000..0ad1fa0b --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentRequestDto.cs @@ -0,0 +1,7 @@ +namespace Serval.WordAlignment.Contracts; + +public record WordAlignmentRequestDto +{ + public required string SourceSegment { get; init; } + public required string TargetSegment { get; init; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentResultDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentResultDto.cs new file mode 100644 index 00000000..c3be0152 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentResultDto.cs @@ -0,0 +1,9 @@ +namespace Serval.WordAlignment.Contracts; + +public record WordAlignmentResultDto +{ + public required IReadOnlyList SourceTokens { get; init; } + public required IReadOnlyList TargetTokens { get; init; } + public required IReadOnlyList Confidences { get; init; } + public required IReadOnlyList Alignment { get; init; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEngineTypesController.cs b/src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEngineTypesController.cs new file mode 100644 index 00000000..87162909 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEngineTypesController.cs @@ -0,0 +1,45 @@ +namespace Serval.WordAlignment.Controllers; + +[ApiVersion(1.0)] +[Route("api/v{version:apiVersion}/word-alignment/engine-types")] +[OpenApiTag("Word Alignment Engines")] +public class WordAlignmentEngineTypesController(IAuthorizationService authService, IEngineService engineService) + : ServalControllerBase(authService) +{ + private readonly IEngineService _engineService = engineService; + + /// + /// Get queue information for a given engine type + /// + /// A valid engine type: statistical or echo-word-alignment + /// + /// Queue information for the specified engine type + /// The client is not authenticated + /// The authenticated client cannot perform the operation + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadWordAlignmentEngines)] + [HttpGet("{engineType}/queues")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> GetQueueAsync( + [NotNull] string engineType, + CancellationToken cancellationToken + ) + { + try + { + return Map( + await _engineService.GetQueueAsync(engineType.ToPascalCase(), cancellationToken: cancellationToken) + ); + } + catch (InvalidOperationException ioe) + { + return BadRequest(ioe.Message); + } + } + + private static QueueDto Map(Queue source) => + new() { Size = source.Size, EngineType = source.EngineType.ToKebabCase() }; +} diff --git a/src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEnginesController.cs b/src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEnginesController.cs new file mode 100644 index 00000000..750fca08 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEnginesController.cs @@ -0,0 +1,956 @@ +namespace Serval.WordAlignment.Controllers; + +[ApiVersion(1.0)] +[Route("api/v{version:apiVersion}/word-alignment/engines")] +[OpenApiTag("Word Alignment Engines")] +public class WordAlignmentEnginesController( + IAuthorizationService authService, + IEngineService engineService, + IBuildService buildService, + IWordAlignmentService wordAlignmentService, + IOptionsMonitor apiOptions, + IUrlService urlService, + ILogger logger +) : ServalControllerBase(authService) +{ + private static readonly JsonSerializerOptions ObjectJsonSerializerOptions = + new() { Converters = { new ObjectToInferredTypesConverter() } }; + + private readonly IEngineService _engineService = engineService; + private readonly IBuildService _buildService = buildService; + private readonly IWordAlignmentService _wordAlignmentService = wordAlignmentService; + private readonly IOptionsMonitor _apiOptions = apiOptions; + private readonly IUrlService _urlService = urlService; + private readonly ILogger _logger = logger; + + /// + /// Get all word alignment engines + /// + /// + /// The engines + /// The client is not authenticated. + /// The authenticated client cannot perform the operation. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadWordAlignmentEngines)] + [HttpGet] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> GetAllAsync(CancellationToken cancellationToken) + { + return (await _engineService.GetAllAsync(Owner, cancellationToken)).Select(Map); + } + + /// + /// Get a word alignment engine by unique id + /// + /// The engine id + /// + /// The engine + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the engine. + /// The engine does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. + + [Authorize(Scopes.ReadWordAlignmentEngines)] + [HttpGet("{id}", Name = Endpoints.GetWordAlignmentEngine)] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> GetAsync( + [NotNull] string id, + CancellationToken cancellationToken + ) + { + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); + return Ok(Map(engine)); + } + + /// + /// Create a new word alignment engine + /// + /// + /// ## Parameters + /// * **name**: (optional) A name to help identify and distinguish the file. + /// * Recommendation: Create a multi-part name to distinguish between projects, uses, etc. + /// * The name does not have to be unique, as the engine is uniquely identified by the auto-generated id + /// * **sourceLanguage**: The source language code (a valid [IETF language tag](https://en.wikipedia.org/wiki/IETF_language_tag) is recommended) + /// * **targetLanguage**: The target language code (a valid IETF language tag is recommended) + /// * **type**: **statistical** or **echo-word-alignment** + /// ### statistical + /// The Statistical engine is based off of the [Thot library](https://github.com/sillsdev/thot) and contains IBM-1, IBM-2, IBM-3, IBM-4, FastAlign and HMM algorithms. + /// ### echo-word-alignment + /// The echo-word-alignment engine has full coverage of all endpoints. Endpoints like create and build return empty responses. + /// Endpoints like get-word-alignment echo the sent content back to the user in the proper format. This engine is useful for debugging and testing purposes. + /// ## Sample request: + /// + /// { + /// "name": "myTeam:myProject:myEngine", + /// "sourceLanguage": "el", + /// "targetLanguage": "en", + /// "type": "statistical" + /// } + /// + /// + /// The engine configuration (see above) + /// + /// The new engine + /// Bad request. Is the engine type correct? + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the engine. + /// The engine does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.CreateWordAlignmentEngines)] + [HttpPost] + [ProducesResponseType(StatusCodes.Status201Created)] + [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> CreateAsync( + [FromBody] WordAlignmentEngineConfigDto engineConfig, + CancellationToken cancellationToken + ) + { + Engine engine = Map(engineConfig); + Engine updatedEngine = await _engineService.CreateAsync(engine, cancellationToken); + WordAlignmentEngineDto dto = Map(updatedEngine); + return Created(dto.Url, dto); + } + + /// + /// Delete a word alignment engine + /// + /// The engine id + /// + /// The engine was successfully deleted. + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the engine. + /// The engine does not exist and therefore cannot be deleted. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.DeleteWordAlignmentEngines)] + [HttpDelete("{id}")] + [ProducesResponseType(typeof(void), StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task DeleteAsync([NotNull] string id, CancellationToken cancellationToken) + { + await AuthorizeAsync(id, cancellationToken); + await _engineService.DeleteAsync(id, cancellationToken); + return Ok(); + } + + /// + /// Align words between a source and target segment + /// + /// The engine id + /// The source and target segment + /// + /// The word alignment result + /// Bad request + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the engine. + /// The engine does not exist. + /// The method is not supported. + /// The engine needs to be built before it can alignment segments. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadWordAlignmentEngines)] + [HttpPost("{id}/get-word-alignment")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status405MethodNotAllowed)] + [ProducesResponseType(typeof(void), StatusCodes.Status409Conflict)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> GetWordAlignmentAsync( + [NotNull] string id, + [FromBody] WordAlignmentRequestDto wordAlignmentRequest, + CancellationToken cancellationToken + ) + { + await AuthorizeAsync(id, cancellationToken); + WordAlignmentResult result = await _engineService.GetWordAlignmentAsync( + id, + wordAlignmentRequest.SourceSegment, + wordAlignmentRequest.TargetSegment, + cancellationToken + ); + _logger.LogInformation("Got word alignment for engine {EngineId}", id); + return Ok(Map(result)); + } + + /// + /// Add a parallel corpus to an engine + /// + /// + /// ## Parameters + /// * **SourceCorpusIds**: The source corpora associated with the parallel corpus + /// * **TargetCorpusIds**: The target corpora associated with the parallel corpus + /// + /// The engine id + /// The corpus configuration (see remarks) + /// + /// + /// + /// The added corpus + /// Bad request + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the engine. + /// The engine does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.UpdateWordAlignmentEngines)] + [HttpPost("{id}/parallel-corpora")] + [ProducesResponseType(StatusCodes.Status201Created)] + [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> AddParallelCorpusAsync( + [NotNull] string id, + [FromBody] WordAlignmentParallelCorpusConfigDto corpusConfig, + [FromServices] IRequestClient getCorpusClient, + [FromServices] IIdGenerator idGenerator, + CancellationToken cancellationToken + ) + { + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); + ParallelCorpus corpus = await MapAsync( + getCorpusClient, + idGenerator.GenerateId(), + corpusConfig, + cancellationToken + ); + await _engineService.AddParallelCorpusAsync(id, corpus, cancellationToken); + WordAlignmentParallelCorpusDto dto = Map(id, corpus); + return Created(dto.Url, dto); + } + + /// + /// Update a parallel corpus with a new set of corpora + /// + /// + /// Will completely replace the parallel corpus' file associations. Will not affect jobs already queued or running. Will not affect existing word graphs until new build is complete. + /// + /// The engine id + /// The parallel corpus id + /// The corpus configuration + /// The data file client + /// + /// The corpus was updated successfully + /// Bad request + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the engine. + /// The engine or corpus does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.UpdateWordAlignmentEngines)] + [HttpPatch("{id}/parallel-corpora/{parallelCorpusId}")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> UpdateParallelCorpusAsync( + [NotNull] string id, + [NotNull] string parallelCorpusId, + [FromBody] WordAlignmentParallelCorpusUpdateConfigDto corpusConfig, + [FromServices] IRequestClient getCorpusClient, + CancellationToken cancellationToken + ) + { + await AuthorizeAsync(id, cancellationToken); + ParallelCorpus parallelCorpus = await _engineService.UpdateParallelCorpusAsync( + id, + parallelCorpusId, + corpusConfig.SourceCorpusIds is null + ? null + : await MapAsync(getCorpusClient, corpusConfig.SourceCorpusIds, cancellationToken), + corpusConfig.TargetCorpusIds is null + ? null + : await MapAsync(getCorpusClient, corpusConfig.TargetCorpusIds, cancellationToken), + cancellationToken + ); + return Ok(Map(id, parallelCorpus)); + } + + /// + /// Get all parallel corpora for a engine + /// + /// The engine id + /// + /// The parallel corpora + /// The client is not authenticated + /// The authenticated client cannot perform the operation or does not own the engine + /// The engine does not exist + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadWordAlignmentEngines)] + [HttpGet("{id}/parallel-corpora")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task>> GetAllParallelCorporaAsync( + [NotNull] string id, + CancellationToken cancellationToken + ) + { + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); + return Ok(engine.ParallelCorpora.Select(c => Map(id, c))); + } + + /// + /// Get the configuration of a parallel corpus for a engine + /// + /// The engine id + /// The parallel corpus id + /// + /// The parallel corpus configuration + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the engine. + /// The engine or parallel corpus does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadTranslationEngines)] + [HttpGet("{id}/parallel-corpora/{parallelCorpusId}", Name = Endpoints.GetParallelWordAlignmentCorpus)] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> GetParallelCorpusAsync( + [NotNull] string id, + [NotNull] string parallelCorpusId, + CancellationToken cancellationToken + ) + { + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); + ParallelCorpus? corpus = engine.ParallelCorpora.FirstOrDefault(f => f.Id == parallelCorpusId); + if (corpus == null) + return NotFound(); + return Ok(Map(id, corpus)); + } + + /// + /// Remove a parallel corpus from a engine + /// + /// + /// Removing a parallel corpus will remove all word alignments associated with that corpus. + /// + /// The engine id + /// The parallel corpus id + /// + /// The parallel corpus was deleted successfully. + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the engine. + /// The engine or parallel corpus does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.UpdateWordAlignmentEngines)] + [HttpDelete("{id}/parallel-corpora/{parallelCorpusId}")] + [ProducesResponseType(typeof(void), StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task DeleteParallelCorpusAsync( + [NotNull] string id, + [NotNull] string parallelCorpusId, + CancellationToken cancellationToken + ) + { + await AuthorizeAsync(id, cancellationToken); + await _engineService.DeleteParallelCorpusAsync(id, parallelCorpusId, cancellationToken); + return Ok(); + } + + /// + /// Get all word alignments in a corpus of a engine + /// + /// + /// Word alignments are arranged in a list of dictionaries with the following fields per word alignment: + /// * **TextId**: The TextId of the SourceFile defined when the corpus was created. + /// * **Refs** (a list of strings): A list of references including: + /// * The references defined in the SourceFile per line, if any. + /// * An auto-generated reference of `[TextId]:[lineNumber]`, 1 indexed. + /// * **SourceTokens**: the tokenized source segment + /// * **TargetTokens**: the tokenized target segment + /// * **Confidences**: the confidence of the alignment ona scale from 0 to 1 + /// * **Alignment**: the word alignment, 0 indexed for source and target positions + /// + /// Word alignments can be filtered by text id if provided. + /// Only word alignments for the most recent successful build of the engine are returned. + /// + /// The engine id + /// The corpus id + /// The text id (optional) + /// + /// The word alignments + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the engine. + /// The engine or corpus does not exist. + /// The engine needs to be built first. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadWordAlignmentEngines)] + [HttpGet("{id}/corpora/{corpusId}/word-alignments")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status409Conflict)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task>> GetAllWordAlignmentsAsync( + [NotNull] string id, + [NotNull] string corpusId, + [FromQuery] string? textId, + CancellationToken cancellationToken + ) + { + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); + if (!engine.ParallelCorpora.Any(c => c.Id == corpusId)) + return NotFound(); + if (engine.ModelRevision == 0) + return Conflict(); + + IEnumerable wordAlignments = await _wordAlignmentService.GetAllAsync( + id, + engine.ModelRevision, + corpusId, + textId, + cancellationToken + ); + _logger.LogInformation( + "Returning {Count} word alignments for engine {EngineId}, corpus {CorpusId}, and query {TextId}", + wordAlignments.Count(), + id, + corpusId, + textId + ); + return Ok(wordAlignments.Select(Map)); + } + + /// + /// Get all build jobs for a engine + /// + /// The engine id + /// + /// The build jobs + /// The client is not authenticated. + /// The authenticated client cannot perform the operation or does not own the engine. + /// The engine does not exist. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadWordAlignmentEngines)] + [HttpGet("{id}/builds")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task>> GetAllBuildsAsync( + [NotNull] string id, + CancellationToken cancellationToken + ) + { + await AuthorizeAsync(id, cancellationToken); + return Ok((await _buildService.GetAllAsync(id, cancellationToken)).Select(Map)); + } + + /// + /// Get a build job + /// + /// + /// If the `minRevision` is not defined, the current build, at whatever state it is, + /// will be immediately returned. If `minRevision` is defined, Serval will wait for + /// up to 40 seconds for the engine to build to the `minRevision` specified, else + /// will timeout. + /// A use case is to actively query the state of the current build, where the subsequent + /// request sets the `minRevision` to the returned `revision` + 1 and timeouts are handled gracefully. + /// This method should use request throttling. + /// Note: Within the returned build, percentCompleted is a value between 0 and 1. + /// + /// The engine id + /// The build job id + /// The minimum revision + /// + /// The build job + /// The client is not authenticated. + /// The authenticated client does not own the engine. + /// The engine or build does not exist. + /// The long polling request timed out. This is expected behavior if you're using long-polling with the minRevision strategy specified in the docs. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadWordAlignmentEngines)] + [HttpGet("{id}/builds/{buildId}", Name = Endpoints.GetWordAlignmentBuild)] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status408RequestTimeout)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> GetBuildAsync( + [NotNull] string id, + [NotNull] string buildId, + [FromQuery] long? minRevision, + CancellationToken cancellationToken + ) + { + await AuthorizeAsync(id, cancellationToken); + if (minRevision != null) + { + (_, EntityChange change) = await TaskEx.Timeout( + ct => _buildService.GetNewerRevisionAsync(buildId, minRevision.Value, ct), + _apiOptions.CurrentValue.LongPollTimeout, + cancellationToken: cancellationToken + ); + return change.Type switch + { + EntityChangeType.None => StatusCode(StatusCodes.Status408RequestTimeout), + EntityChangeType.Delete => NotFound(), + _ => Ok(Map(change.Entity!)), + }; + } + else + { + Build build = await _buildService.GetAsync(buildId, cancellationToken); + return Ok(Map(build)); + } + } + + /// + /// Starts a build job for a engine. + /// + /// + /// Specify the corpora and textIds to train on. If no "trainOn" field is provided, all corpora will be used. + /// Paratext Projects, you may flag a subset of books for training by including their [abbreviations] + /// Paratext projects can be filtered by [book](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs) using the textId for training. + /// Filters can also be supplied via scriptureRange parameter as ranges of biblical text. See [here](https://github.com/sillsdev/serval/wiki/Filtering-Paratext-Project-Data-with-a-Scripture-Range) + /// All Paratext project filtering follows original versification. See [here](https://github.com/sillsdev/serval/wiki/Versification-in-Serval) for more information. + /// + /// Specify the corpora or textIds to word align on. + /// When a corpus or textId is selected for word align on, only text segments that are in both the source and the target will be aligned. + /// + /// The `"options"` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. + /// See [statistical alignment job settings documentation](https://github.com/sillsdev/serval/wiki/Statistical-Alignment-Build-Options) about configuring job parameters. + /// See [keyterms parsing documentation](https://github.com/sillsdev/serval/wiki/Paratext-Key-Terms-Parsing) on how to use keyterms for training. + /// + /// The engine id + /// The build config (see remarks) + /// + /// The new build job + /// The build configuration was invalid. + /// The client is not authenticated. + /// The authenticated client does not own the engine. + /// The engine does not exist. + /// There is already an active or pending build or a build in the process of being canceled. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.UpdateWordAlignmentEngines)] + [HttpPost("{id}/builds")] + [ProducesResponseType(StatusCodes.Status201Created)] + [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status409Conflict)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> StartBuildAsync( + [NotNull] string id, + [FromBody] WordAlignmentBuildConfigDto buildConfig, + CancellationToken cancellationToken + ) + { + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); + Build build = Map(engine, buildConfig); + await _engineService.StartBuildAsync(build, cancellationToken); + + WordAlignmentBuildDto dto = Map(build); + return Created(dto.Url, dto); + } + + /// + /// Get the currently running build job for a engine + /// + /// + /// See documentation on endpoint /word-alignment/engines/{id}/builds/{id} - "Get a Build Job" for details on using `minRevision`. + /// + /// The engine id + /// The minimum revision + /// + /// The build job + /// There is no build currently running. + /// Bad request + /// The client is not authenticated. + /// The authenticated client does not own the engine. + /// The engine does not exist. + /// The long polling request timed out. This is expected behavior if you're using long-polling with the minRevision strategy specified in the docs. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadWordAlignmentEngines)] + [HttpGet("{id}/current-build")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status204NoContent)] + [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status408RequestTimeout)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> GetCurrentBuildAsync( + [NotNull] string id, + [FromQuery] long? minRevision, + CancellationToken cancellationToken + ) + { + await AuthorizeAsync(id, cancellationToken); + if (minRevision != null) + { + (_, EntityChange change) = await TaskEx.Timeout( + ct => _buildService.GetActiveNewerRevisionAsync(id, minRevision.Value, ct), + _apiOptions.CurrentValue.LongPollTimeout, + cancellationToken: cancellationToken + ); + return change.Type switch + { + EntityChangeType.None => StatusCode(StatusCodes.Status408RequestTimeout), + EntityChangeType.Delete => NoContent(), + _ => Ok(Map(change.Entity!)), + }; + } + else + { + Build? build = await _buildService.GetActiveAsync(id, cancellationToken); + if (build == null) + return NoContent(); + + return Ok(Map(build)); + } + } + + /// + /// Cancel the current build job (whether pending or active) for a engine + /// + /// + /// + /// The engine id + /// + /// The build job was cancelled successfully. + /// There is no active build job. + /// The client is not authenticated. + /// The authenticated client does not own the engine. + /// The engine does not exist. + /// The engine does not support cancelling builds. + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.UpdateWordAlignmentEngines)] + [HttpPost("{id}/current-build/cancel")] + [ProducesResponseType(typeof(void), StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status204NoContent)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] + [ProducesResponseType(typeof(void), StatusCodes.Status405MethodNotAllowed)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task CancelBuildAsync([NotNull] string id, CancellationToken cancellationToken) + { + await AuthorizeAsync(id, cancellationToken); + if (!await _engineService.CancelBuildAsync(id, cancellationToken)) + return NoContent(); + return Ok(); + } + + private async Task AuthorizeAsync(string id, CancellationToken cancellationToken) + { + Engine engine = await _engineService.GetAsync(id, cancellationToken); + await AuthorizeAsync(engine); + } + + private async Task MapAsync( + IRequestClient getDataFileClient, + string corpusId, + WordAlignmentParallelCorpusConfigDto source, + CancellationToken cancellationToken + ) + { + return new ParallelCorpus + { + Id = corpusId, + SourceCorpora = await MapAsync(getDataFileClient, source.SourceCorpusIds, cancellationToken), + TargetCorpora = await MapAsync(getDataFileClient, source.TargetCorpusIds, cancellationToken) + }; + } + + private async Task> MapAsync( + IRequestClient getCorpusClient, + IEnumerable corpusIds, + CancellationToken cancellationToken + ) + { + var corpora = new List(); + foreach (string corpusId in corpusIds) + { + Response response = await getCorpusClient.GetResponse< + CorpusResult, + CorpusNotFound + >(new GetCorpus { CorpusId = corpusId, Owner = Owner }, cancellationToken); + if (response.Is(out Response? result)) + { + corpora.Add( + new MonolingualCorpus + { + Id = corpusId, + Name = result.Message.Name ?? "", + Language = result.Message.Language, + Files = result + .Message.Files.Select(f => new CorpusFile + { + Id = f.File.DataFileId, + Filename = f.File.Filename, + Format = f.File.Format, + TextId = f.TextId + }) + .ToList(), + } + ); + } + else if (response.Is(out Response? _)) + { + throw new InvalidOperationException($"The corpus {corpusId} cannot be found."); + } + } + return corpora; + } + + private WordAlignmentParallelCorpusDto Map(string engineId, ParallelCorpus source) + { + return new WordAlignmentParallelCorpusDto + { + Id = source.Id, + Url = _urlService.GetUrl(Endpoints.GetCorpus, new { id = engineId, corpusId = source.Id }), + Engine = new ResourceLinkDto + { + Id = engineId, + Url = _urlService.GetUrl(Endpoints.GetTranslationEngine, new { id = engineId }) + }, + SourceCorpora = source + .SourceCorpora.Select(c => new ResourceLinkDto + { + Id = c.Id, + Url = _urlService.GetUrl(Endpoints.GetCorpus, new { Id = c.Id }) + }) + .ToList(), + TargetCorpora = source + .TargetCorpora.Select(c => new ResourceLinkDto + { + Id = c.Id, + Url = _urlService.GetUrl(Endpoints.GetCorpus, new { Id = c.Id }) + }) + .ToList() + }; + } + + private static Build Map(Engine engine, WordAlignmentBuildConfigDto source) + { + return new Build + { + EngineRef = engine.Id, + Name = source.Name, + WordAlignOn = Map(engine, source.WordAlignOn), + TrainOn = Map(engine, source.TrainOn), + Options = Map(source.Options) + }; + } + + private static List? Map(Engine engine, IReadOnlyList? source) + { + if (source is null) + return null; + + var corpusIds = new HashSet(engine.ParallelCorpora.Select(c => c.Id)); + var trainingCorpora = new List(); + foreach (TrainingCorpusConfigDto cc in source) + { + if (cc.ParallelCorpusId == null) + { + throw new InvalidOperationException($"One of ParallelCorpusId and CorpusId must be set."); + } + if (!corpusIds.Contains(cc.ParallelCorpusId)) + { + throw new InvalidOperationException( + $"The parallel corpus {cc.ParallelCorpusId} is not valid: This parallel corpus does not exist for engine {engine.Id}." + ); + } + trainingCorpora.Add( + new TrainingCorpus + { + ParallelCorpusRef = cc.ParallelCorpusId, + SourceFilters = cc.SourceFilters?.Select(Map).ToList(), + TargetFilters = cc.TargetFilters?.Select(Map).ToList() + } + ); + } + return trainingCorpora; + } + + private static ParallelCorpusFilter Map(ParallelCorpusFilterConfigDto source) + { + if (source.TextIds != null && source.ScriptureRange != null) + { + throw new InvalidOperationException( + $"The parallel corpus filter for corpus {source.CorpusId} is not valid: At most, one of TextIds and ScriptureRange can be set." + ); + } + return new ParallelCorpusFilter + { + CorpusRef = source.CorpusId, + TextIds = source.TextIds, + ScriptureRange = source.ScriptureRange + }; + } + + private static Dictionary? Map(object? source) + { + try + { + return JsonSerializer.Deserialize>( + source?.ToString() ?? "{}", + ObjectJsonSerializerOptions + ); + } + catch (Exception e) + { + throw new InvalidOperationException($"Unable to parse field 'options' : {e.Message}", e); + } + } + + private WordAlignmentEngineDto Map(Engine source) + { + return new WordAlignmentEngineDto + { + Id = source.Id, + Url = _urlService.GetUrl(Endpoints.GetWordAlignmentEngine, new { id = source.Id }), + Name = source.Name, + SourceLanguage = source.SourceLanguage, + TargetLanguage = source.TargetLanguage, + Type = source.Type.ToKebabCase(), + IsBuilding = source.IsBuilding, + ModelRevision = source.ModelRevision, + Confidence = Math.Round(source.Confidence, 8), + CorpusSize = source.CorpusSize + }; + } + + private WordAlignmentBuildDto Map(Build source) + { + return new WordAlignmentBuildDto + { + Id = source.Id, + Url = _urlService.GetUrl(Endpoints.GetTranslationBuild, new { id = source.EngineRef, buildId = source.Id }), + Revision = source.Revision, + Name = source.Name, + Engine = new ResourceLinkDto + { + Id = source.EngineRef, + Url = _urlService.GetUrl(Endpoints.GetTranslationEngine, new { id = source.EngineRef }) + }, + TrainOn = source.TrainOn?.Select(s => Map(source.EngineRef, s)).ToList(), + WordAlignOn = source.WordAlignOn?.Select(s => Map(source.EngineRef, s)).ToList(), + Step = source.Step, + PercentCompleted = source.PercentCompleted, + Message = source.Message, + QueueDepth = source.QueueDepth, + State = source.State, + DateFinished = source.DateFinished, + Options = source.Options + }; + } + + private TrainingCorpusDto Map(string engineId, TrainingCorpus source) + { + return new TrainingCorpusDto + { + ParallelCorpus = + source.ParallelCorpusRef != null + ? new ResourceLinkDto + { + Id = source.ParallelCorpusRef, + Url = _urlService.GetUrl( + Endpoints.GetParallelTranslationCorpus, + new { id = engineId, parallelCorpusId = source.ParallelCorpusRef } + ) + } + : null, + SourceFilters = source.SourceFilters?.Select(Map).ToList(), + TargetFilters = source.TargetFilters?.Select(Map).ToList() + }; + } + + private ParallelCorpusFilterDto Map(ParallelCorpusFilter source) + { + return new ParallelCorpusFilterDto + { + Corpus = new ResourceLinkDto + { + Id = source.CorpusRef, + Url = _urlService.GetUrl(Endpoints.GetCorpus, new { id = source.CorpusRef }) + }, + TextIds = source.TextIds, + ScriptureRange = source.ScriptureRange + }; + } + + private WordAlignmentResultDto Map(WordAlignmentResult source) + { + return new WordAlignmentResultDto + { + SourceTokens = source.SourceTokens.ToList(), + TargetTokens = source.TargetTokens.ToList(), + Confidences = source.Confidences.Select(c => Math.Round(c, 8)).ToList(), + Alignment = source.Alignment.Select(Map).ToList(), + }; + } + + private AlignedWordPairDto Map(AlignedWordPair source) + { + return new AlignedWordPairDto() { SourceIndex = source.SourceIndex, TargetIndex = source.TargetIndex }; + } + + private static WordAlignmentDto Map(Models.WordAlignment source) + { + return new WordAlignmentDto + { + TextId = source.TextId, + Refs = source.Refs, + SourceTokens = source.SourceTokens.ToList(), + TargetTokens = source.TargetTokens.ToList(), + Confidences = source.Confidences.Select(c => Math.Round(c, 8)).ToList(), + Alignment = source + .Alignment.Select(c => new AlignedWordPairDto() + { + SourceIndex = c.SourceIndex, + TargetIndex = c.TargetIndex + }) + .ToList(), + }; + } + + private Engine Map(WordAlignmentEngineConfigDto source) + { + return new Engine + { + Name = source.Name, + SourceLanguage = source.SourceLanguage, + TargetLanguage = source.TargetLanguage, + Type = source.Type.ToPascalCase(), + Owner = Owner, + ParallelCorpora = [], + }; + } +} diff --git a/src/Serval/src/Serval.WordAlignment/Models/Build.cs b/src/Serval/src/Serval.WordAlignment/Models/Build.cs new file mode 100644 index 00000000..1b31b8a3 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Models/Build.cs @@ -0,0 +1,18 @@ +namespace Serval.WordAlignment.Models; + +public record Build : IEntity +{ + public string Id { get; set; } = ""; + public int Revision { get; set; } = 1; + public string? Name { get; init; } + public required string EngineRef { get; init; } + public IReadOnlyList? TrainOn { get; init; } + public IReadOnlyList? WordAlignOn { get; init; } + public int Step { get; init; } + public double? PercentCompleted { get; init; } + public string? Message { get; init; } + public int? QueueDepth { get; init; } + public JobState State { get; init; } = JobState.Pending; + public DateTime? DateFinished { get; init; } + public IReadOnlyDictionary? Options { get; init; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Models/Engine.cs b/src/Serval/src/Serval.WordAlignment/Models/Engine.cs new file mode 100644 index 00000000..fb1774ae --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Models/Engine.cs @@ -0,0 +1,18 @@ +namespace Serval.WordAlignment.Models; + +public record Engine : IOwnedEntity +{ + public string Id { get; set; } = ""; + public int Revision { get; set; } = 1; + public string? Name { get; init; } + public required string SourceLanguage { get; init; } + public required string TargetLanguage { get; init; } + public required string Type { get; init; } + public required string Owner { get; init; } + public required IReadOnlyList ParallelCorpora { get; init; } + public bool? IsModelPersisted { get; init; } + public bool IsBuilding { get; init; } + public int ModelRevision { get; init; } + public double Confidence { get; init; } + public int CorpusSize { get; init; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Models/ParallelCorpus.cs b/src/Serval/src/Serval.WordAlignment/Models/ParallelCorpus.cs new file mode 100644 index 00000000..a870eba9 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Models/ParallelCorpus.cs @@ -0,0 +1,8 @@ +namespace Serval.WordAlignment.Models; + +public record ParallelCorpus +{ + public required string Id { get; set; } + public IReadOnlyList SourceCorpora { get; set; } = new List(); + public IReadOnlyList TargetCorpora { get; set; } = new List(); +} diff --git a/src/Serval/src/Serval.WordAlignment/Models/TrainingCorpus.cs b/src/Serval/src/Serval.WordAlignment/Models/TrainingCorpus.cs new file mode 100644 index 00000000..fd9c06a6 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Models/TrainingCorpus.cs @@ -0,0 +1,8 @@ +namespace Serval.WordAlignment.Models; + +public record TrainingCorpus +{ + public string? ParallelCorpusRef { get; set; } + public IReadOnlyList? SourceFilters { get; set; } + public IReadOnlyList? TargetFilters { get; set; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Models/WordAlignment.cs b/src/Serval/src/Serval.WordAlignment/Models/WordAlignment.cs new file mode 100644 index 00000000..0e9725f0 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Models/WordAlignment.cs @@ -0,0 +1,16 @@ +namespace Serval.WordAlignment.Models; + +public class WordAlignment : IEntity +{ + public string Id { get; set; } = ""; + public int Revision { get; set; } = 1; + public required string EngineRef { get; init; } + public int ModelRevision { get; init; } + public required string CorpusRef { get; init; } + public required string TextId { get; init; } + public required IReadOnlyList Refs { get; init; } + public required IReadOnlyList SourceTokens { get; set; } + public required IReadOnlyList TargetTokens { get; set; } + public required IReadOnlyList Confidences { get; set; } + public required IReadOnlyList Alignment { get; set; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Models/WordAlignmentResult.cs b/src/Serval/src/Serval.WordAlignment/Models/WordAlignmentResult.cs new file mode 100644 index 00000000..6bce82fb --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Models/WordAlignmentResult.cs @@ -0,0 +1,9 @@ +namespace Serval.WordAlignment.Models; + +public record WordAlignmentResult +{ + public required IReadOnlyList SourceTokens { get; set; } + public required IReadOnlyList TargetTokens { get; set; } + public required IReadOnlyList Confidences { get; set; } + public required IReadOnlyList Alignment { get; set; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Serval.WordAlignment.csproj b/src/Serval/src/Serval.WordAlignment/Serval.WordAlignment.csproj new file mode 100644 index 00000000..034f8e85 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Serval.WordAlignment.csproj @@ -0,0 +1,27 @@ + + + + net8.0 + enable + enable + true + true + true + $(NoWarn);CS1591;CS1573 + + + + + + + + + + + + + + + + + diff --git a/src/Serval/src/Serval.WordAlignment/Services/BuildService.cs b/src/Serval/src/Serval.WordAlignment/Services/BuildService.cs new file mode 100644 index 00000000..c3069135 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Services/BuildService.cs @@ -0,0 +1,63 @@ +namespace Serval.WordAlignment.Services; + +public class BuildService(IRepository builds) : EntityServiceBase(builds), IBuildService +{ + public async Task> GetAllAsync(string parentId, CancellationToken cancellationToken = default) + { + return await Entities.GetAllAsync(e => e.EngineRef == parentId, cancellationToken); + } + + public Task GetActiveAsync(string parentId, CancellationToken cancellationToken = default) + { + return Entities.GetAsync( + b => b.EngineRef == parentId && (b.State == JobState.Active || b.State == JobState.Pending), + cancellationToken + ); + } + + public Task> GetNewerRevisionAsync( + string id, + long minRevision, + CancellationToken cancellationToken = default + ) + { + return GetNewerRevisionAsync(e => e.Id == id, minRevision, cancellationToken); + } + + public Task> GetActiveNewerRevisionAsync( + string parentId, + long minRevision, + CancellationToken cancellationToken = default + ) + { + return GetNewerRevisionAsync( + b => b.EngineRef == parentId && (b.State == JobState.Active || b.State == JobState.Pending), + minRevision, + cancellationToken + ); + } + + private async Task> GetNewerRevisionAsync( + Expression> filter, + long minRevision, + CancellationToken cancellationToken = default + ) + { + using ISubscription subscription = await Entities.SubscribeAsync(filter, cancellationToken); + EntityChange curChange = subscription.Change; + if (curChange.Type == EntityChangeType.Delete && minRevision > 1) + return curChange; + while (true) + { + if (curChange.Entity is not null) + { + if (curChange.Type != EntityChangeType.Delete && minRevision <= curChange.Entity.Revision) + return curChange; + } + await subscription.WaitForChangeAsync(cancellationToken: cancellationToken); + curChange = subscription.Change; + if (curChange.Type == EntityChangeType.Delete) + return curChange; + } + } +} diff --git a/src/Serval/src/Serval.WordAlignment/Services/EngineService.cs b/src/Serval/src/Serval.WordAlignment/Services/EngineService.cs new file mode 100644 index 00000000..a77eb83a --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Services/EngineService.cs @@ -0,0 +1,453 @@ +using Serval.WordAlignment.V1; + +namespace Serval.WordAlignment.Services; + +public class EngineService( + IRepository engines, + IRepository builds, + IRepository wordAlignments, + GrpcClientFactory grpcClientFactory, + IOptionsMonitor dataFileOptions, + IDataAccessContext dataAccessContext, + ILoggerFactory loggerFactory, + IScriptureDataFileService scriptureDataFileService +) : OwnedEntityServiceBase(engines), IEngineService +{ + private readonly IRepository _builds = builds; + private readonly IRepository _wordAlignments = wordAlignments; + private readonly GrpcClientFactory _grpcClientFactory = grpcClientFactory; + private readonly IOptionsMonitor _dataFileOptions = dataFileOptions; + private readonly IDataAccessContext _dataAccessContext = dataAccessContext; + private readonly ILogger _logger = loggerFactory.CreateLogger(); + private readonly IScriptureDataFileService _scriptureDataFileService = scriptureDataFileService; + + public async Task GetWordAlignmentAsync( + string engineId, + string sourceSegment, + string targetSegment, + CancellationToken cancellationToken = default + ) + { + Engine engine = await GetAsync(engineId, cancellationToken); + + WordAlignmentEngineApi.WordAlignmentEngineApiClient client = + _grpcClientFactory.CreateClient(engine.Type); + GetWordAlignmentResponse response = await client.GetWordAlignmentAsync( + new GetWordAlignmentRequest + { + EngineType = engine.Type, + EngineId = engine.Id, + SourceSegment = sourceSegment, + TargetSegment = targetSegment + }, + cancellationToken: cancellationToken + ); + return Map(response.Result); + } + + public override async Task CreateAsync(Engine engine, CancellationToken cancellationToken = default) + { + bool updateIsModelPersisted = engine.IsModelPersisted is null; + try + { + await Entities.InsertAsync(engine, cancellationToken); + WordAlignmentEngineApi.WordAlignmentEngineApiClient? client = + _grpcClientFactory.CreateClient(engine.Type); + if (client is null) + throw new InvalidOperationException($"'{engine.Type}' is an invalid engine type."); + var request = new CreateRequest + { + EngineType = engine.Type, + EngineId = engine.Id, + SourceLanguage = engine.SourceLanguage, + TargetLanguage = engine.TargetLanguage + }; + + if (engine.Name is not null) + request.EngineName = engine.Name; + await client.CreateAsync(request, cancellationToken: cancellationToken); + } + catch (RpcException rpcex) + { + await Entities.DeleteAsync(engine, CancellationToken.None); + if (rpcex.StatusCode == StatusCode.InvalidArgument) + { + throw new InvalidOperationException( + $"Unable to create engine {engine.Id} because of an invalid argument: {rpcex.Status.Detail}", + rpcex + ); + } + throw; + } + catch + { + await Entities.DeleteAsync(engine, CancellationToken.None); + throw; + } + if (updateIsModelPersisted) + { + await Entities.UpdateAsync( + engine, + u => u.Set(e => e.IsModelPersisted, engine.IsModelPersisted), + cancellationToken: cancellationToken + ); + } + return engine; + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + Engine? engine = await Entities.GetAsync(engineId, cancellationToken); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{engineId}'."); + + WordAlignmentEngineApi.WordAlignmentEngineApiClient client = + _grpcClientFactory.CreateClient(engine.Type); + await client.DeleteAsync( + new DeleteRequest { EngineType = engine.Type, EngineId = engine.Id }, + cancellationToken: cancellationToken + ); + + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + await Entities.DeleteAsync(engineId, ct); + await _builds.DeleteAllAsync(b => b.EngineRef == engineId, ct); + await _wordAlignments.DeleteAllAsync(pt => pt.EngineRef == engineId, ct); + }, + CancellationToken.None + ); + } + + private Dictionary> GetChapters(string fileLocation, string scriptureRange) + { + try + { + return ScriptureRangeParser.GetChapters( + scriptureRange, + _scriptureDataFileService.GetParatextProjectSettings(fileLocation).Versification + ); + } + catch (ArgumentException ae) + { + throw new InvalidOperationException($"The scripture range {scriptureRange} is not valid: {ae.Message}"); + } + } + + public async Task StartBuildAsync(Build build, CancellationToken cancellationToken = default) + { + Engine engine = await GetAsync(build.EngineRef, cancellationToken); + await _builds.InsertAsync(build, cancellationToken); + + WordAlignmentEngineApi.WordAlignmentEngineApiClient client = + _grpcClientFactory.CreateClient(engine.Type); + + try + { + StartBuildRequest request; + var trainOn = build.TrainOn?.ToDictionary(c => c.ParallelCorpusRef!); + var wordAlignOn = build.WordAlignOn?.ToDictionary(c => c.ParallelCorpusRef!); + request = new StartBuildRequest + { + EngineType = engine.Type, + EngineId = engine.Id, + BuildId = build.Id, + Corpora = + { + engine.ParallelCorpora.Select(c => + Map(c, trainOn?.GetValueOrDefault(c.Id), wordAlignOn?.GetValueOrDefault(c.Id)) + ) + } + }; + + if (build.Options is not null) + request.Options = JsonSerializer.Serialize(build.Options); + + // Log the build request summary + try + { + var buildRequestSummary = (JsonObject)JsonNode.Parse(JsonSerializer.Serialize(request))!; + // correct build options parsing + buildRequestSummary.Remove("Options"); + try + { + buildRequestSummary.Add("Options", JsonNode.Parse(request.Options)); + } + catch (JsonException) + { + buildRequestSummary.Add( + "Options", + "Build \"Options\" failed parsing: " + (request.Options ?? "null") + ); + } + buildRequestSummary.Add("Event", "BuildRequest"); + buildRequestSummary.Add("ModelRevision", engine.ModelRevision); + buildRequestSummary.Add("ClientId", engine.Owner); + _logger.LogInformation("{request}", buildRequestSummary.ToJsonString()); + } + catch (JsonException) + { + _logger.LogInformation("Error parsing build request summary."); + _logger.LogInformation("{request}", JsonSerializer.Serialize(request)); + } + + await client.StartBuildAsync(request, cancellationToken: cancellationToken); + } + catch + { + await _builds.DeleteAsync(build, CancellationToken.None); + throw; + } + } + + public async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + Engine? engine = await GetAsync(engineId, cancellationToken); + if (engine is null) + throw new EntityNotFoundException($"Could not find the Engine '{engineId}'."); + + WordAlignmentEngineApi.WordAlignmentEngineApiClient client = + _grpcClientFactory.CreateClient(engine.Type); + try + { + await client.CancelBuildAsync( + new CancelBuildRequest { EngineType = engine.Type, EngineId = engine.Id }, + cancellationToken: cancellationToken + ); + } + catch (RpcException re) + { + if (re.StatusCode is StatusCode.Aborted) + return false; + throw; + } + return true; + } + + public Task AddParallelCorpusAsync( + string engineId, + Models.ParallelCorpus corpus, + CancellationToken cancellationToken = default + ) + { + return Entities.UpdateAsync( + engineId, + u => u.Add(e => e.ParallelCorpora, corpus), + cancellationToken: cancellationToken + ); + } + + public async Task UpdateParallelCorpusAsync( + string engineId, + string parallelCorpusId, + IReadOnlyList? sourceCorpora, + IReadOnlyList? targetCorpora, + CancellationToken cancellationToken = default + ) + { + Engine? engine = await Entities.UpdateAsync( + e => e.Id == engineId && e.ParallelCorpora.Any(c => c.Id == parallelCorpusId), + u => + { + if (sourceCorpora is not null) + u.Set(c => c.ParallelCorpora[ArrayPosition.FirstMatching].SourceCorpora, sourceCorpora); + if (targetCorpora is not null) + u.Set(c => c.ParallelCorpora[ArrayPosition.FirstMatching].TargetCorpora, targetCorpora); + }, + cancellationToken: cancellationToken + ); + if (engine is null) + { + throw new EntityNotFoundException( + $"Could not find the Corpus '{parallelCorpusId}' in Engine '{engineId}'." + ); + } + + return engine.ParallelCorpora.First(c => c.Id == parallelCorpusId); + } + + public async Task DeleteParallelCorpusAsync( + string engineId, + string parallelCorpusId, + CancellationToken cancellationToken = default + ) + { + Engine? originalEngine = null; + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + originalEngine = await Entities.UpdateAsync( + engineId, + u => u.RemoveAll(e => e.ParallelCorpora, c => c.Id == parallelCorpusId), + returnOriginal: true, + cancellationToken: ct + ); + if (originalEngine is null || !originalEngine.ParallelCorpora.Any(c => c.Id == parallelCorpusId)) + { + throw new EntityNotFoundException( + $"Could not find the Corpus '{parallelCorpusId}' in Engine '{engineId}'." + ); + } + await _wordAlignments.DeleteAllAsync(pt => pt.CorpusRef == parallelCorpusId, ct); + }, + cancellationToken: cancellationToken + ); + } + + public Task DeleteAllCorpusFilesAsync(string dataFileId, CancellationToken cancellationToken = default) + { + return Entities.UpdateAllAsync( + e => + e.ParallelCorpora.Any(c => + c.SourceCorpora.Any(sc => sc.Files.Any(f => f.Id == dataFileId)) + || c.TargetCorpora.Any(tc => tc.Files.Any(f => f.Id == dataFileId)) + ), + u => + u.RemoveAll( + e => e.ParallelCorpora[ArrayPosition.All].SourceCorpora[ArrayPosition.All].Files, + f => f.Id == dataFileId + ) + .RemoveAll( + e => e.ParallelCorpora[ArrayPosition.All].TargetCorpora[ArrayPosition.All].Files, + f => f.Id == dataFileId + ), + cancellationToken + ); + } + + public async Task GetQueueAsync(string engineType, CancellationToken cancellationToken = default) + { + WordAlignmentEngineApi.WordAlignmentEngineApiClient client = + _grpcClientFactory.CreateClient(engineType); + GetQueueSizeResponse response = await client.GetQueueSizeAsync( + new GetQueueSizeRequest { EngineType = engineType }, + cancellationToken: cancellationToken + ); + return new Queue { Size = response.Size, EngineType = engineType }; + } + + private Models.WordAlignmentResult Map(V1.WordAlignmentResult source) + { + return new Models.WordAlignmentResult + { + SourceTokens = source.SourceTokens.ToList(), + TargetTokens = source.TargetTokens.ToList(), + Confidences = source.Confidences.ToList(), + Alignment = source.Alignment.Select(Map).ToList(), + }; + } + + private Shared.Models.AlignedWordPair Map(V1.AlignedWordPair source) + { + return new Shared.Models.AlignedWordPair { SourceIndex = source.SourceIndex, TargetIndex = source.TargetIndex }; + } + + private V1.ParallelCorpus Map(Models.ParallelCorpus source, TrainingCorpus? trainOn, TrainingCorpus? wordAlignOn) + { + string? referenceFileLocation = + source.TargetCorpora.Count > 0 && source.TargetCorpora[0].Files.Count > 0 + ? Map(source.TargetCorpora[0].Files[0]).Location + : null; + + return new V1.ParallelCorpus + { + Id = source.Id, + SourceCorpora = + { + source.SourceCorpora.Select(sc => + Map( + sc, + trainOn?.SourceFilters?.Where(sf => sf.CorpusRef == sc.Id).FirstOrDefault(), + wordAlignOn?.SourceFilters?.Where(sf => sf.CorpusRef == sc.Id).FirstOrDefault(), + referenceFileLocation + ) + ) + }, + TargetCorpora = + { + source.TargetCorpora.Select(tc => + Map( + tc, + trainOn?.TargetFilters?.Where(sf => sf.CorpusRef == tc.Id).FirstOrDefault(), + null, + referenceFileLocation + ) + ) + } + }; + } + + private V1.MonolingualCorpus Map( + Shared.Models.MonolingualCorpus source, + ParallelCorpusFilter? trainingFilter, + ParallelCorpusFilter? wordAlignmentFilter, + string? referenceFileLocation + ) + { + Dictionary? trainOnChapters = null; + if ( + trainingFilter is not null + && trainingFilter.ScriptureRange is not null + && referenceFileLocation is not null + ) + { + trainOnChapters = GetChapters(referenceFileLocation, trainingFilter.ScriptureRange) + .Select( + (kvp) => + { + var scriptureChapters = new ScriptureChapters(); + scriptureChapters.Chapters.Add(kvp.Value); + return (kvp.Key, scriptureChapters); + } + ) + .ToDictionary(); + } + + Dictionary? wordAlignmentChapters = null; + if ( + wordAlignmentFilter is not null + && wordAlignmentFilter.ScriptureRange is not null + && referenceFileLocation is not null + ) + { + GetChapters(referenceFileLocation, wordAlignmentFilter.ScriptureRange) + .Select( + (kvp) => + { + var scriptureChapters = new ScriptureChapters(); + scriptureChapters.Chapters.Add(kvp.Value); + return (kvp.Key, scriptureChapters); + } + ) + .ToDictionary(); + } + + var corpus = new V1.MonolingualCorpus + { + Id = source.Id, + Language = source.Language, + Files = { source.Files.Select(Map) } + }; + + if (trainOnChapters is not null) + corpus.TrainOnChapters.Add(trainOnChapters); + if (trainingFilter?.TextIds is not null) + corpus.TrainOnTextIds.Add(trainingFilter.TextIds); + if (wordAlignmentChapters is not null) + corpus.WordAlignOnChapters.Add(wordAlignmentChapters); + if (wordAlignmentFilter?.TextIds is not null) + corpus.WordAlignOnTextIds.Add(wordAlignmentFilter.TextIds); + + return corpus; + } + + private V1.CorpusFile Map(Shared.Models.CorpusFile source) + { + return new V1.CorpusFile + { + TextId = source.TextId, + Format = (V1.FileFormat)source.Format, + Location = Path.Combine(_dataFileOptions.CurrentValue.FilesDirectory, source.Filename) + }; + } +} diff --git a/src/Serval/src/Serval.WordAlignment/Services/IBuildService.cs b/src/Serval/src/Serval.WordAlignment/Services/IBuildService.cs new file mode 100644 index 00000000..f7cbcabb --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Services/IBuildService.cs @@ -0,0 +1,18 @@ +namespace Serval.WordAlignment.Services; + +public interface IBuildService +{ + Task> GetAllAsync(string parentId, CancellationToken cancellationToken = default); + Task GetAsync(string id, CancellationToken cancellationToken = default); + Task GetActiveAsync(string parentId, CancellationToken cancellationToken = default); + Task> GetNewerRevisionAsync( + string id, + long minRevision, + CancellationToken cancellationToken = default + ); + Task> GetActiveNewerRevisionAsync( + string parentId, + long minRevision, + CancellationToken cancellationToken = default + ); +} diff --git a/src/Serval/src/Serval.WordAlignment/Services/IEngineService.cs b/src/Serval/src/Serval.WordAlignment/Services/IEngineService.cs new file mode 100644 index 00000000..17a59f6c --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Services/IEngineService.cs @@ -0,0 +1,39 @@ +namespace Serval.WordAlignment.Services; + +public interface IEngineService +{ + Task> GetAllAsync(string owner, CancellationToken cancellationToken = default); + Task GetAsync(string engineId, CancellationToken cancellationToken = default); + + Task CreateAsync(Engine engine, CancellationToken cancellationToken = default); + Task DeleteAsync(string engineId, CancellationToken cancellationToken = default); + + Task GetWordAlignmentAsync( + string engineId, + string sourceSegment, + string targetSegment, + CancellationToken cancellationToken = default + ); + + Task StartBuildAsync(Build build, CancellationToken cancellationToken = default); + + Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default); + + Task AddParallelCorpusAsync(string engineId, ParallelCorpus corpus, CancellationToken cancellationToken = default); + Task UpdateParallelCorpusAsync( + string engineId, + string parallelCorpusId, + IReadOnlyList? sourceCorpora, + IReadOnlyList? targetCorpora, + CancellationToken cancellationToken = default + ); + Task DeleteParallelCorpusAsync( + string engineId, + string parallelCorpusId, + CancellationToken cancellationToken = default + ); + + Task DeleteAllCorpusFilesAsync(string dataFileId, CancellationToken cancellationToken = default); + + Task GetQueueAsync(string engineType, CancellationToken cancellationToken = default); +} diff --git a/src/Serval/src/Serval.WordAlignment/Services/IWordAlignmentService.cs b/src/Serval/src/Serval.WordAlignment/Services/IWordAlignmentService.cs new file mode 100644 index 00000000..fd94879f --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Services/IWordAlignmentService.cs @@ -0,0 +1,12 @@ +namespace Serval.WordAlignment.Services; + +public interface IWordAlignmentService +{ + Task> GetAllAsync( + string engineId, + int modelRevision, + string corpusId, + string? textId = null, + CancellationToken cancellationToken = default + ); +} diff --git a/src/Serval/src/Serval.WordAlignment/Services/WordAlignmentPlatformServiceV1.cs b/src/Serval/src/Serval.WordAlignment/Services/WordAlignmentPlatformServiceV1.cs new file mode 100644 index 00000000..af6dd3dd --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Services/WordAlignmentPlatformServiceV1.cs @@ -0,0 +1,331 @@ +using Google.Protobuf.WellKnownTypes; +using Serval.WordAlignment.V1; + +namespace Serval.WordAlignment.Services; + +public class WordAlignmentPlatformServiceV1( + IRepository builds, + IRepository engines, + IRepository wordAlignments, + IDataAccessContext dataAccessContext, + IPublishEndpoint publishEndpoint +) : WordAlignmentPlatformApi.WordAlignmentPlatformApiBase +{ + private const int WordAlignmentInsertBatchSize = 128; + private static readonly Empty Empty = new(); + + private readonly IRepository _builds = builds; + private readonly IRepository _engines = engines; + private readonly IRepository _wordAlignments = wordAlignments; + private readonly IDataAccessContext _dataAccessContext = dataAccessContext; + private readonly IPublishEndpoint _publishEndpoint = publishEndpoint; + + public override async Task BuildStarted(BuildStartedRequest request, ServerCallContext context) + { + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + Build? build = await _builds.UpdateAsync( + request.BuildId, + u => u.Set(b => b.State, JobState.Active), + cancellationToken: ct + ); + if (build is null) + throw new RpcException(new Status(StatusCode.NotFound, "The build does not exist.")); + + Engine? engine = await _engines.UpdateAsync( + build.EngineRef, + u => u.Set(e => e.IsBuilding, true), + cancellationToken: ct + ); + if (engine is null) + throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); + + await _publishEndpoint.Publish( + new TranslationBuildStarted + { + BuildId = build.Id, + EngineId = engine.Id, + Owner = engine.Owner + }, + ct + ); + }, + cancellationToken: context.CancellationToken + ); + return Empty; + } + + public override async Task BuildCompleted(BuildCompletedRequest request, ServerCallContext context) + { + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + Build? build = await _builds.UpdateAsync( + request.BuildId, + u => + u.Set(b => b.State, JobState.Completed) + .Set(b => b.Message, "Completed") + .Set(b => b.DateFinished, DateTime.UtcNow), + cancellationToken: ct + ); + if (build is null) + throw new RpcException(new Status(StatusCode.NotFound, "The build does not exist.")); + + Engine? engine = await _engines.UpdateAsync( + build.EngineRef, + u => + u.Set(e => e.Confidence, request.Confidence) + .Set(e => e.CorpusSize, request.CorpusSize) + .Set(e => e.IsBuilding, false) + .Inc(e => e.ModelRevision), + cancellationToken: ct + ); + if (engine is null) + throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); + + // delete pretranslations created by the previous build + await _wordAlignments.DeleteAllAsync( + p => p.EngineRef == engine.Id && p.ModelRevision < engine.ModelRevision, + ct + ); + + await _publishEndpoint.Publish( + new TranslationBuildFinished + { + BuildId = build.Id, + EngineId = engine.Id, + Owner = engine.Owner, + BuildState = build.State, + Message = build.Message!, + DateFinished = build.DateFinished!.Value + }, + ct + ); + }, + cancellationToken: context.CancellationToken + ); + + return Empty; + } + + public override async Task BuildCanceled(BuildCanceledRequest request, ServerCallContext context) + { + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + Build? build = await _builds.UpdateAsync( + request.BuildId, + u => + u.Set(b => b.Message, "Canceled") + .Set(b => b.DateFinished, DateTime.UtcNow) + .Set(b => b.State, JobState.Canceled), + cancellationToken: ct + ); + if (build is null) + throw new RpcException(new Status(StatusCode.NotFound, "The build does not exist.")); + + Engine? engine = await _engines.UpdateAsync( + build.EngineRef, + u => u.Set(e => e.IsBuilding, false), + cancellationToken: ct + ); + if (engine is null) + throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); + + // delete pretranslations that might have been created during the build + await _wordAlignments.DeleteAllAsync( + p => p.EngineRef == engine.Id && p.ModelRevision > engine.ModelRevision, + ct + ); + + await _publishEndpoint.Publish( + new TranslationBuildFinished + { + BuildId = build.Id, + EngineId = engine.Id, + Owner = engine.Owner, + BuildState = build.State, + Message = build.Message!, + DateFinished = build.DateFinished!.Value + }, + ct + ); + }, + cancellationToken: context.CancellationToken + ); + + return Empty; + } + + public override async Task BuildFaulted(BuildFaultedRequest request, ServerCallContext context) + { + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + Build? build = await _builds.UpdateAsync( + request.BuildId, + u => + u.Set(b => b.State, JobState.Faulted) + .Set(b => b.Message, request.Message) + .Set(b => b.DateFinished, DateTime.UtcNow), + cancellationToken: ct + ); + if (build is null) + throw new RpcException(new Status(StatusCode.NotFound, "The build does not exist.")); + + Engine? engine = await _engines.UpdateAsync( + build.EngineRef, + u => u.Set(e => e.IsBuilding, false), + cancellationToken: ct + ); + if (engine is null) + throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); + + // delete pretranslations that might have been created during the build + await _wordAlignments.DeleteAllAsync( + p => p.EngineRef == engine.Id && p.ModelRevision > engine.ModelRevision, + ct + ); + + await _publishEndpoint.Publish( + new TranslationBuildFinished + { + BuildId = build.Id, + EngineId = engine.Id, + Owner = engine.Owner, + BuildState = build.State, + Message = build.Message!, + DateFinished = build.DateFinished!.Value + }, + ct + ); + }, + cancellationToken: context.CancellationToken + ); + + return Empty; + } + + public override async Task BuildRestarting(BuildRestartingRequest request, ServerCallContext context) + { + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + Build? build = await _builds.UpdateAsync( + request.BuildId, + u => + u.Set(b => b.Message, "Restarting") + .Set(b => b.Step, 0) + .Set(b => b.PercentCompleted, 0) + .Set(b => b.State, JobState.Pending), + cancellationToken: ct + ); + if (build is null) + throw new RpcException(new Status(StatusCode.NotFound, "The build does not exist.")); + + Engine? engine = await _engines.GetAsync(build.EngineRef, ct); + if (engine is null) + throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); + + // delete pretranslations that might have been created during the build + await _wordAlignments.DeleteAllAsync( + p => p.EngineRef == engine.Id && p.ModelRevision > engine.ModelRevision, + ct + ); + }, + cancellationToken: context.CancellationToken + ); + + return Empty; + } + + public override async Task UpdateBuildStatus(UpdateBuildStatusRequest request, ServerCallContext context) + { + await _builds.UpdateAsync( + b => b.Id == request.BuildId && (b.State == JobState.Active || b.State == JobState.Pending), + u => + { + u.Set(b => b.Step, request.Step); + if (request.HasPercentCompleted) + { + u.Set( + b => b.PercentCompleted, + Math.Round(request.PercentCompleted, 4, MidpointRounding.AwayFromZero) + ); + } + if (request.HasMessage) + u.Set(b => b.Message, request.Message); + if (request.HasQueueDepth) + u.Set(b => b.QueueDepth, request.QueueDepth); + }, + cancellationToken: context.CancellationToken + ); + + return Empty; + } + + public override async Task IncrementWordAlignmentEngineCorpusSize( + IncrementWordAlignmentEngineCorpusSizeRequest request, + ServerCallContext context + ) + { + await _engines.UpdateAsync( + request.EngineId, + u => u.Inc(e => e.CorpusSize, request.Count), + cancellationToken: context.CancellationToken + ); + return Empty; + } + + public override async Task InsertWordAlignments( + IAsyncStreamReader requestStream, + ServerCallContext context + ) + { + string engineId = ""; + int nextModelRevision = 0; + + var batch = new List(); + await foreach (InsertWordAlignmentsRequest request in requestStream.ReadAllAsync(context.CancellationToken)) + { + if (request.EngineId != engineId) + { + Engine? engine = await _engines.GetAsync(request.EngineId, context.CancellationToken); + if (engine is null) + throw new RpcException(new Status(StatusCode.NotFound, "The engine does not exist.")); + nextModelRevision = engine.ModelRevision + 1; + engineId = request.EngineId; + } + batch.Add( + new Models.WordAlignment + { + EngineRef = request.EngineId, + ModelRevision = nextModelRevision, + CorpusRef = request.CorpusId, + TextId = request.TextId, + Refs = request.Refs.ToList(), + SourceTokens = request.SourceTokens.ToList(), + TargetTokens = request.TargetTokens.ToList(), + Confidences = request.Confidences.ToList(), + Alignment = request + .Alignment.Select(a => new Shared.Models.AlignedWordPair + { + SourceIndex = a.SourceIndex, + TargetIndex = a.TargetIndex + }) + .ToList() + } + ); + if (batch.Count == WordAlignmentInsertBatchSize) + { + await _wordAlignments.InsertAllAsync(batch, context.CancellationToken); + batch.Clear(); + } + } + if (batch.Count > 0) + await _wordAlignments.InsertAllAsync(batch, CancellationToken.None); + + return Empty; + } +} diff --git a/src/Serval/src/Serval.WordAlignment/Services/WordAlignmentService.cs b/src/Serval/src/Serval.WordAlignment/Services/WordAlignmentService.cs new file mode 100644 index 00000000..09549609 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Services/WordAlignmentService.cs @@ -0,0 +1,24 @@ +namespace Serval.WordAlignment.Services; + +public class WordAlignmentService(IRepository wordAlignments) + : EntityServiceBase(wordAlignments), + IWordAlignmentService +{ + public async Task> GetAllAsync( + string engineId, + int modelRevision, + string corpusId, + string? textId = null, + CancellationToken cancellationToken = default + ) + { + return await Entities.GetAllAsync( + pt => + pt.EngineRef == engineId + && pt.ModelRevision == modelRevision + && pt.CorpusRef == corpusId + && (textId == null || pt.TextId == textId), + cancellationToken + ); + } +} diff --git a/src/Serval/src/Serval.WordAlignment/using.cs b/src/Serval/src/Serval.WordAlignment/using.cs new file mode 100644 index 00000000..19fe0b79 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/using.cs @@ -0,0 +1,30 @@ +global using System.Diagnostics.CodeAnalysis; +global using System.Linq.Expressions; +global using System.Text.Json; +global using System.Text.Json.Nodes; +global using Asp.Versioning; +global using CaseExtensions; +global using Grpc.Core; +global using Grpc.Net.ClientFactory; +global using MassTransit; +global using Microsoft.AspNetCore.Authorization; +global using Microsoft.AspNetCore.Http; +global using Microsoft.AspNetCore.Mvc; +global using Microsoft.AspNetCore.Routing; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; +global using NSwag.Annotations; +global using Serval.Shared.Configuration; +global using Serval.Shared.Contracts; +global using Serval.Shared.Controllers; +global using Serval.Shared.Models; +global using Serval.Shared.Services; +global using Serval.Shared.Utils; +global using Serval.WordAlignment.Configuration; +global using Serval.WordAlignment.Consumers; +global using Serval.WordAlignment.Contracts; +global using Serval.WordAlignment.Models; +global using Serval.WordAlignment.Services; +global using SIL.DataAccess; +global using SIL.ServiceToolkit.Utils; diff --git a/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs index 0da83cf1..47843032 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs @@ -133,7 +133,7 @@ public async Task StartBuildAsync_TrainOnNotSpecified() new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -154,7 +154,7 @@ public async Task StartBuildAsync_TrainOnNotSpecified() new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -207,7 +207,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -229,7 +229,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -282,7 +282,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -304,7 +304,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -356,7 +356,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -377,7 +377,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -430,7 +430,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -451,7 +451,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -504,7 +504,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -525,7 +525,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -550,7 +550,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -571,7 +571,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -652,7 +652,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -684,7 +684,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -736,7 +736,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -757,7 +757,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -833,7 +833,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MAT" } }, @@ -849,7 +849,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MRK" } }, @@ -872,7 +872,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MAT" } }, @@ -888,7 +888,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MRK" } }, @@ -965,7 +965,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MAT" } }, @@ -988,7 +988,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MAT" } }, @@ -1065,7 +1065,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MAT" } }, @@ -1088,7 +1088,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MAT" } }, @@ -1114,7 +1114,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MRK" } }, @@ -1136,7 +1136,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MRK" } }, @@ -1212,7 +1212,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -1228,7 +1228,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file3.zip" } }, @@ -1251,7 +1251,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -1267,7 +1267,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file4.zip" } }, @@ -1363,7 +1363,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -1379,7 +1379,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file3.zip" } }, @@ -1412,7 +1412,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -1428,7 +1428,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file4.zip" } }, @@ -1495,7 +1495,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -1522,7 +1522,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file3.zip" } }, @@ -1552,7 +1552,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -1579,7 +1579,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file4.zip" } }, @@ -1640,7 +1640,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -1656,7 +1656,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file3.zip" } }, @@ -1675,7 +1675,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -1691,7 +1691,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file4.zip" } }, @@ -1734,7 +1734,7 @@ public async Task StartBuildAsync_TrainOnNotSpecified_ParallelCorpus() new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -1750,7 +1750,7 @@ public async Task StartBuildAsync_TrainOnNotSpecified_ParallelCorpus() new V1.CorpusFile { Location = "file3.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file3.zip" } }, @@ -1769,7 +1769,7 @@ public async Task StartBuildAsync_TrainOnNotSpecified_ParallelCorpus() new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -1785,7 +1785,7 @@ public async Task StartBuildAsync_TrainOnNotSpecified_ParallelCorpus() new V1.CorpusFile { Location = "file4.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file4.zip" } }, @@ -1845,7 +1845,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -1872,7 +1872,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file3.zip" } }, @@ -1891,7 +1891,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -1907,7 +1907,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file4.zip" } }, @@ -2292,7 +2292,7 @@ public async Task CreateParallelCorpusEngineWithTextFilesAsync() new() { Id = "parallel-corpus1", - SourceCorpora = new List() + SourceCorpora = new List() { new() { @@ -2327,7 +2327,7 @@ public async Task CreateParallelCorpusEngineWithTextFilesAsync() ] } }, - TargetCorpora = new List() + TargetCorpora = new List() { new() { @@ -2383,7 +2383,7 @@ public async Task CreateMultipleParallelCorpusEngineWithTextFilesAsync() new() { Id = "parallel-corpus1", - SourceCorpora = new List() + SourceCorpora = new List() { new() { @@ -2402,7 +2402,7 @@ public async Task CreateMultipleParallelCorpusEngineWithTextFilesAsync() ] } }, - TargetCorpora = new List() + TargetCorpora = new List() { new() { @@ -2425,7 +2425,7 @@ public async Task CreateMultipleParallelCorpusEngineWithTextFilesAsync() new() { Id = "parallel-corpus2", - SourceCorpora = new List() + SourceCorpora = new List() { new() { @@ -2444,7 +2444,7 @@ public async Task CreateMultipleParallelCorpusEngineWithTextFilesAsync() ] } }, - TargetCorpora = new List() + TargetCorpora = new List() { new() { @@ -2484,7 +2484,7 @@ public async Task CreateParallelCorpusEngineWithParatextProjectAsync() new() { Id = "parallel-corpus1", - SourceCorpora = new List() + SourceCorpora = new List() { new() { @@ -2519,7 +2519,7 @@ public async Task CreateParallelCorpusEngineWithParatextProjectAsync() ] } }, - TargetCorpora = new List() + TargetCorpora = new List() { new() { diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs index 5aca4ed6..b2e5f0f3 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs @@ -270,7 +270,7 @@ private class TestEnvironment : IDisposable { public TestEnvironment() { - CorpusFile file1 = + Shared.Models.CorpusFile file1 = new() { Id = "file1", @@ -278,7 +278,7 @@ public TestEnvironment() Format = Shared.Contracts.FileFormat.Paratext, TextId = "project1" }; - CorpusFile file2 = + Shared.Models.CorpusFile file2 = new() { Id = "file2", @@ -321,7 +321,7 @@ public TestEnvironment() new() { Id = "parallel_corpus1", - SourceCorpora = new List() + SourceCorpora = new List() { new() { @@ -330,7 +330,7 @@ public TestEnvironment() Files = [file1], } }, - TargetCorpora = new List() + TargetCorpora = new List() { new() { diff --git a/src/Serval/test/Serval.Translation.Tests/Usings.cs b/src/Serval/test/Serval.Translation.Tests/Usings.cs index ef8a3ff7..9116c0c9 100644 --- a/src/Serval/test/Serval.Translation.Tests/Usings.cs +++ b/src/Serval/test/Serval.Translation.Tests/Usings.cs @@ -7,6 +7,7 @@ global using NSubstitute; global using NUnit.Framework; global using Serval.Shared.Configuration; +global using Serval.Shared.Contracts; global using Serval.Shared.Services; global using Serval.Shared.Utils; global using Serval.Translation.Contracts;