Add Word Alignment engine:

* Add HTTP API * Add Grpc * Add Echo (as single engine) * Add Serval services * Add webhooks * Update docker-compose * Basic echo works * Machine Engine updates are made * FIXME - update Machine library!
sillsdev · Nov 5, 2024 · d699845 · d699845
1 parent 57ce385
commit d699845
Show file tree

Hide file tree

Showing 136 changed files with 7,438 additions and 983 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -25,7 +25,7 @@
             "containerName": "echo_cntr",
             "request": "attach",
             "platform": "netCore",
-            "processName": "EchoTranslationEngine",
+            "processName": "EchoEngine",
             "sourceFileMap": {
                 "/app": "${workspaceFolder}"
             },
@@ -77,9 +77,9 @@
             "type": "coreclr",
             "request": "launch",
             "preLaunchTask": "build",
-            "program": "${workspaceFolder}/src/Echo/src/EchoTranslationEngine/bin/Debug/net8.0/EchoTranslationEngine.dll",
+            "program": "${workspaceFolder}/src/Echo/src/EchoEngine/bin/Debug/net8.0/EchoEngine.dll",
             "args": [],
-            "cwd": "${workspaceFolder}/src/Echo/src/EchoTranslationEngine",
+            "cwd": "${workspaceFolder}/src/Echo/src/EchoEngine",
             "stopAtEntry": false,
             "console": "internalConsole",
             "justMyCode": false,

diff --git a/Serval.sln b/Serval.sln
@@ -36,8 +36,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Translation.Tests",
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.DataFiles", "src\Serval\src\Serval.DataFiles\Serval.DataFiles.csproj", "{4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EchoTranslationEngine", "src\Echo\src\EchoTranslationEngine\EchoTranslationEngine.csproj", "{A9D08CA2-3CF7-4BB7-A47F-5A567FFDB2CC}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.ApiServer.IntegrationTests", "src\Serval\test\Serval.ApiServer.IntegrationTests\Serval.ApiServer.IntegrationTests.csproj", "{0C3DF75B-B022-4EFC-882C-F276F1EC8435}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.E2ETests", "src\Serval\test\Serval.E2ETests\Serval.E2ETests.csproj", "{1F020042-D7B8-4541-9691-26ECFD1FFC73}"
@@ -48,10 +46,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.DataAccess.Tests", "src
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Shared.Tests", "src\Serval\test\Serval.Shared.Tests\Serval.Shared.Tests.csproj", "{0E220C65-AA88-450E-AFB2-844E49060B3F}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Echo", "Echo", "{53B18D34-B7C7-4B91-BCB0-1021170DFC65}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{BE90915F-E16D-479C-9568-22A08F0FD8F9}"
-EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Serval", "Serval", "{6D20F76D-9A0E-44AC-8754-B4291C75D25B}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{25CDB05B-4E24-4A6E-933E-1E0BEC97D74D}"
@@ -86,6 +80,20 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{C3A14577-A65
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.ServiceToolkit", "src\ServiceToolkit\src\SIL.ServiceToolkit\SIL.ServiceToolkit.csproj", "{0E40F959-C641-40A2-9750-B17A4F9F9E55}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{9125C013-4F15-4761-BCD2-070524986737}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Serval", "Serval", "{A78D900F-AE52-436C-88CE-A22EAEDECD91}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{C1EF85B4-F2CD-407D-904D-BAC1BCE3A64C}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serval.WordAlignment", "src\Serval\src\Serval.WordAlignment\Serval.WordAlignment.csproj", "{F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Echo", "Echo", "{D201886D-9299-4758-80E8-694DBCF8DF93}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{0904BA95-D5BF-4AC2-A919-20A785EF45F5}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EchoEngine", "src\Echo\src\EchoEngine\EchoEngine.csproj", "{929FF600-8C7E-4498-A2A3-5534F3A3481E}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -132,10 +140,6 @@ Global
 		{4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Release|Any CPU.Build.0 = Release|Any CPU
-		{A9D08CA2-3CF7-4BB7-A47F-5A567FFDB2CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{A9D08CA2-3CF7-4BB7-A47F-5A567FFDB2CC}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{A9D08CA2-3CF7-4BB7-A47F-5A567FFDB2CC}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{A9D08CA2-3CF7-4BB7-A47F-5A567FFDB2CC}.Release|Any CPU.Build.0 = Release|Any CPU
 		{0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -180,6 +184,14 @@ Global
 		{0E40F959-C641-40A2-9750-B17A4F9F9E55}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0E40F959-C641-40A2-9750-B17A4F9F9E55}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0E40F959-C641-40A2-9750-B17A4F9F9E55}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Release|Any CPU.Build.0 = Release|Any CPU
+		{929FF600-8C7E-4498-A2A3-5534F3A3481E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{929FF600-8C7E-4498-A2A3-5534F3A3481E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{929FF600-8C7E-4498-A2A3-5534F3A3481E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{929FF600-8C7E-4498-A2A3-5534F3A3481E}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -195,13 +207,11 @@ Global
 		{8FC30758-37FC-4819-805D-8EFF3DEDF05F} = {3E753B99-7C31-42AC-B02E-012B802F58DB}
 		{539598C5-8634-4273-8714-A684622DDCFC} = {3E753B99-7C31-42AC-B02E-012B802F58DB}
 		{4375A7BF-E3CE-4785-91E3-2ED6FCEB074F} = {25CDB05B-4E24-4A6E-933E-1E0BEC97D74D}
-		{A9D08CA2-3CF7-4BB7-A47F-5A567FFDB2CC} = {BE90915F-E16D-479C-9568-22A08F0FD8F9}
 		{0C3DF75B-B022-4EFC-882C-F276F1EC8435} = {3E753B99-7C31-42AC-B02E-012B802F58DB}
 		{1F020042-D7B8-4541-9691-26ECFD1FFC73} = {3E753B99-7C31-42AC-B02E-012B802F58DB}
 		{63E4D71B-11BE-4D68-A876-5B1B5F0A4C88} = {3E753B99-7C31-42AC-B02E-012B802F58DB}
 		{71151518-8774-44D0-8E69-D77FA447BEFA} = {BA044B98-3136-4FDE-B90F-B0975758C07F}
 		{0E220C65-AA88-450E-AFB2-844E49060B3F} = {3E753B99-7C31-42AC-B02E-012B802F58DB}
-		{BE90915F-E16D-479C-9568-22A08F0FD8F9} = {53B18D34-B7C7-4B91-BCB0-1021170DFC65}
 		{25CDB05B-4E24-4A6E-933E-1E0BEC97D74D} = {6D20F76D-9A0E-44AC-8754-B4291C75D25B}
 		{3E753B99-7C31-42AC-B02E-012B802F58DB} = {6D20F76D-9A0E-44AC-8754-B4291C75D25B}
 		{92805246-5285-4F0A-9BF8-6EE4A027A41B} = {33E6965E-5A58-4C6F-882E-F17C8E88A3FF}
@@ -215,6 +225,12 @@ Global
 		{10657805-48F1-4205-B8F5-79447F6EF620} = {25CDB05B-4E24-4A6E-933E-1E0BEC97D74D}
 		{C3A14577-A654-4604-818C-4E683DD45A51} = {EA69B41C-49EF-4017-A687-44B9DF37FF98}
 		{0E40F959-C641-40A2-9750-B17A4F9F9E55} = {C3A14577-A654-4604-818C-4E683DD45A51}
+		{A78D900F-AE52-436C-88CE-A22EAEDECD91} = {9125C013-4F15-4761-BCD2-070524986737}
+		{C1EF85B4-F2CD-407D-904D-BAC1BCE3A64C} = {A78D900F-AE52-436C-88CE-A22EAEDECD91}
+		{F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354} = {C1EF85B4-F2CD-407D-904D-BAC1BCE3A64C}
+		{D201886D-9299-4758-80E8-694DBCF8DF93} = {9125C013-4F15-4761-BCD2-070524986737}
+		{0904BA95-D5BF-4AC2-A919-20A785EF45F5} = {D201886D-9299-4758-80E8-694DBCF8DF93}
+		{929FF600-8C7E-4498-A2A3-5534F3A3481E} = {0904BA95-D5BF-4AC2-A919-20A785EF45F5}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {9F18C25E-E140-43C3-B177-D562E1628370}

diff --git a/deploy/serval/templates/echo-deployment.yaml b/deploy/serval/templates/echo-deployment.yaml
@@ -21,7 +21,7 @@ spec:
       containers:
         - command:
             - dotnet
-            - /app/echo_server/EchoTranslationEngine.dll
+            - /app/echo_server/EchoEngine.dll
           workingDir: /app/echo_server
           env:
             - name: ASPNETCORE_ENVIRONMENT

diff --git a/docker-compose.withatlas.yml b/docker-compose.withatlas.yml
@@ -22,6 +22,11 @@ services:
       - ASPNETCORE_Translation__Engines__1__Address=http://machine-engine
       - ASPNETCORE_Translation__Engines__2__Type=Nmt
       - ASPNETCORE_Translation__Engines__2__Address=http://machine-engine
+      - ASPNETCORE_WordAlignment__Engines__0__Type=EchoWordAlignment
+      - ASPNETCORE_WordAlignment__Engines__0__Address=http://echo
+      - ASPNETCORE_WordAlignment__Engines__1__Type=Statistical
+      - ASPNETCORE_WordAlignment__Engines__1__Address=http://machine-engine
+      - "Bugsnag__ApiKey=${Bugsnag__ApiKey}"
     expose:
       - 80
       - 81
@@ -51,6 +56,8 @@ services:
       - ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80
       - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2
       - ASPNETCORE_ConnectionStrings__TranslationPlatformApi=http://serval-api:81
+      - ASPNETCORE_ConnectionStrings__WordAlignmentPlatformApi=http://serval-api:81
+      - "Bugsnag__ApiKey=${Bugsnag__ApiKey}"
     expose:
       - 80
     ports:
@@ -61,7 +68,7 @@ services:
       - .:/app:ro
       - ~/.nuget/packages:/root/.nuget/packages:ro
       - /var/lib/serval:/var/lib/serval
-    working_dir: '/app/src/Echo/src/EchoTranslationEngine'
+    working_dir: '/app/src/Echo/src/EchoEngine'
     entrypoint:
       - dotnet
       - run
@@ -95,6 +102,7 @@ services:
       - SharedFile__Uri=s3://silnlp/docker-compose/
       - "SharedFile__S3AccessKeyId=${AWS_ACCESS_KEY_ID:?access key needed}"
       - "SharedFile__S3SecretAccessKey=${AWS_SECRET_ACCESS_KEY:?secret key needed}"
+      - "Bugsnag__ApiKey=${Bugsnag__ApiKey}"
     expose:
       - 80
     ports:
@@ -139,6 +147,7 @@ services:
       - SharedFile__Uri=s3://silnlp/docker-compose/
       - "SharedFile__S3AccessKeyId=${AWS_ACCESS_KEY_ID:?access key needed}"
       - "SharedFile__S3SecretAccessKey=${AWS_SECRET_ACCESS_KEY:?secret key needed}"
+      - "Bugsnag__ApiKey=${Bugsnag__ApiKey}"
     expose:
       - 80
     ports:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -21,6 +21,10 @@ services:
       - ASPNETCORE_Translation__Engines__1__Address=http://machine-engine
       - ASPNETCORE_Translation__Engines__2__Type=Nmt
       - ASPNETCORE_Translation__Engines__2__Address=http://machine-engine
+      - ASPNETCORE_WordAlignment__Engines__0__Type=EchoWordAlignment
+      - ASPNETCORE_WordAlignment__Engines__0__Address=http://echo
+      - ASPNETCORE_WordAlignment__Engines__1__Type=Statistical
+      - ASPNETCORE_WordAlignment__Engines__1__Address=http://machine-engine
       - "Bugsnag__ApiKey=${Bugsnag__ApiKey}"
     expose:
       - 80
@@ -53,6 +57,7 @@ services:
       - ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80
       - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2
       - ASPNETCORE_ConnectionStrings__TranslationPlatformApi=http://serval-api:81
+      - ASPNETCORE_ConnectionStrings__WordAlignmentPlatformApi=http://serval-api:81
       - "Bugsnag__ApiKey=${Bugsnag__ApiKey}"
     expose:
       - 80
@@ -64,7 +69,7 @@ services:
       - .:/app:ro
       - ~/.nuget/packages:/root/.nuget/packages:ro
       - /var/lib/serval:/var/lib/serval
-    working_dir: '/app/src/Echo/src/EchoTranslationEngine'
+    working_dir: '/app/src/Echo/src/EchoEngine'
     entrypoint:
       - dotnet
       - run

diff --git a/dockerfile b/dockerfile
@@ -9,7 +9,7 @@ COPY . ./
 RUN dotnet restore
 # Build and publish a release
 RUN dotnet publish ./src/Serval/src/Serval.ApiServer/Serval.ApiServer.csproj -c Release -o out_api_server
-RUN dotnet publish ./src/Echo/src/EchoTranslationEngine/EchoTranslationEngine.csproj -c Release -o out_echo_server
+RUN dotnet publish ./src/Echo/src/EchoEngine/EchoEngine.csproj -c Release -o out_echo_server
 RUN dotnet publish ./src/Machine/src/Serval.Machine.EngineServer/Serval.Machine.EngineServer.csproj -c Release -o out_machine_engine_server
 RUN dotnet publish ./src/Machine/src/Serval.Machine.JobServer/Serval.Machine.JobServer.csproj -c Release -o out_machine_job_server
 

diff --git a/...oTranslationEngine/BackgroundTaskQueue.cs → ...cho/src/EchoEngine/BackgroundTaskQueue.cs b/...oTranslationEngine/BackgroundTaskQueue.cs → ...cho/src/EchoEngine/BackgroundTaskQueue.cs
@@ -1,4 +1,4 @@
-namespace EchoTranslationEngine;
+namespace EchoEngine;
 
 public class BackgroundTaskQueue
 {

diff --git a/...ranslationEngine/BackgroundTaskService.cs → ...o/src/EchoEngine/BackgroundTaskService.cs b/...ranslationEngine/BackgroundTaskService.cs → ...o/src/EchoEngine/BackgroundTaskService.cs
@@ -1,4 +1,4 @@
-namespace EchoTranslationEngine;
+namespace EchoEngine;
 
 public class BackgroundTaskService(
     BackgroundTaskQueue taskQueue,

diff --git a/...lationEngine/EchoTranslationEngine.csproj → src/Echo/src/EchoEngine/EchoEngine.csproj b/...lationEngine/EchoTranslationEngine.csproj → src/Echo/src/EchoEngine/EchoEngine.csproj
diff --git a/.../EchoTranslationEngine/HealthServiceV1.cs → src/Echo/src/EchoEngine/HealthServiceV1.cs b/.../EchoTranslationEngine/HealthServiceV1.cs → src/Echo/src/EchoEngine/HealthServiceV1.cs
@@ -1,6 +1,6 @@
 using Serval.Health.V1;
 
-namespace EchoTranslationEngine;
+namespace EchoEngine;
 
 public class HealthServiceV1(HealthCheckService healthCheckService) : HealthApi.HealthApiBase
 {

diff --git a/...Echo/src/EchoTranslationEngine/Program.cs → src/Echo/src/EchoEngine/Program.cs b/...Echo/src/EchoTranslationEngine/Program.cs → src/Echo/src/EchoEngine/Program.cs
@@ -1,10 +1,24 @@
+using Serval.Translation.V1;
+using Serval.WordAlignment.V1;
+
 WebApplicationBuilder builder = WebApplication.CreateBuilder(args);
 
 // Add services to the container.
-builder.Services.AddGrpcClient<TranslationPlatformApi.TranslationPlatformApiClient>(o =>
-{
-    o.Address = new Uri(builder.Configuration.GetConnectionString("TranslationPlatformApi")!);
-});
+builder.Services.AddGrpcClient<TranslationPlatformApi.TranslationPlatformApiClient>(
+    "Translation",
+    o =>
+    {
+        o.Address = new Uri(builder.Configuration.GetConnectionString("TranslationPlatformApi")!);
+    }
+);
+builder.Services.AddGrpcClient<WordAlignmentPlatformApi.WordAlignmentPlatformApiClient>(
+    "WordAlignment",
+    o =>
+    {
+        o.Address = new Uri(builder.Configuration.GetConnectionString("WordAlignmentPlatformApi")!);
+    }
+);
+
 builder.Services.AddGrpc();
 
 builder.Services.AddHostedService<BackgroundTaskService>();
@@ -18,6 +32,8 @@
 WebApplication app = builder.Build();
 
 app.MapGrpcService<TranslationEngineServiceV1>();
+app.MapGrpcService<WordAlignmentEngineServiceV1>();
+
 app.MapGrpcService<HealthServiceV1>();
 
 app.Run();
diff --git a/...tionEngine/Properties/launchSettings.json → ...EchoEngine/Properties/launchSettings.json b/...tionEngine/Properties/launchSettings.json → ...EchoEngine/Properties/launchSettings.json
@@ -1,7 +1,7 @@
 {
   "$schema": "https://json.schemastore.org/launchsettings.json",
   "profiles": {
-    "EchoTranslationEngine": {
+    "EchoEngine": {
       "commandName": "Project",
       "dotnetRunMessages": true,
       "launchBrowser": false,
@@ -11,4 +11,4 @@
       }
     }
   }
-}
+}
diff --git a/...ationEngine/TranslationEngineServiceV1.cs → .../EchoEngine/TranslationEngineServiceV1.cs b/...ationEngine/TranslationEngineServiceV1.cs → .../EchoEngine/TranslationEngineServiceV1.cs
@@ -1,4 +1,6 @@
-namespace EchoTranslationEngine;
+using Serval.Translation.V1;
+
+namespace EchoEngine;
 
 public class TranslationEngineServiceV1(BackgroundTaskQueue taskQueue) : TranslationEngineApi.TranslationEngineApiBase
 {

diff --git a/src/Echo/src/EchoTranslationEngine/Usings.cs → src/Echo/src/EchoEngine/Usings.cs b/src/Echo/src/EchoTranslationEngine/Usings.cs → src/Echo/src/EchoEngine/Usings.cs
@@ -1,7 +1,6 @@
 global using System.Threading.Channels;
 global using Bugsnag.AspNet.Core;
-global using EchoTranslationEngine;
+global using EchoEngine;
 global using Google.Protobuf.WellKnownTypes;
 global using Grpc.Core;
 global using Microsoft.Extensions.Diagnostics.HealthChecks;
-global using Serval.Translation.V1;