Skip to content

Commit

Permalink
Add Word Alignment engine:
Browse files Browse the repository at this point in the history
* Add HTTP API
* Add Grpc
* Add Echo (as single engine)
* Add Serval services
* Add webhooks
* Update docker-compose
* Basic echo works

All engines need to have different names, even if they are in different controllers.  Calling Echo for word alignment, echo-word-alignment.
  • Loading branch information
johnml1135 committed Oct 9, 2024
1 parent 083d68e commit 250f6e2
Show file tree
Hide file tree
Showing 87 changed files with 6,494 additions and 506 deletions.
6 changes: 3 additions & 3 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
"containerName": "echo_cntr",
"request": "attach",
"platform": "netCore",
"processName": "EchoTranslationEngine",
"processName": "EchoEngine",
"sourceFileMap": {
"/app": "${workspaceFolder}"
},
Expand Down Expand Up @@ -77,9 +77,9 @@
"type": "coreclr",
"request": "launch",
"preLaunchTask": "build",
"program": "${workspaceFolder}/src/Echo/src/EchoTranslationEngine/bin/Debug/net8.0/EchoTranslationEngine.dll",
"program": "${workspaceFolder}/src/Echo/src/EchoEngine/bin/Debug/net8.0/EchoEngine.dll",
"args": [],
"cwd": "${workspaceFolder}/src/Echo/src/EchoTranslationEngine",
"cwd": "${workspaceFolder}/src/Echo/src/EchoEngine",
"stopAtEntry": false,
"console": "internalConsole",
"justMyCode": false,
Expand Down
40 changes: 28 additions & 12 deletions Serval.sln
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Translation.Tests",
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.DataFiles", "src\Serval\src\Serval.DataFiles\Serval.DataFiles.csproj", "{4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EchoTranslationEngine", "src\Echo\src\EchoTranslationEngine\EchoTranslationEngine.csproj", "{A9D08CA2-3CF7-4BB7-A47F-5A567FFDB2CC}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.ApiServer.IntegrationTests", "src\Serval\test\Serval.ApiServer.IntegrationTests\Serval.ApiServer.IntegrationTests.csproj", "{0C3DF75B-B022-4EFC-882C-F276F1EC8435}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.E2ETests", "src\Serval\test\Serval.E2ETests\Serval.E2ETests.csproj", "{1F020042-D7B8-4541-9691-26ECFD1FFC73}"
Expand All @@ -48,10 +46,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.DataAccess.Tests", "src
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Shared.Tests", "src\Serval\test\Serval.Shared.Tests\Serval.Shared.Tests.csproj", "{0E220C65-AA88-450E-AFB2-844E49060B3F}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Echo", "Echo", "{53B18D34-B7C7-4B91-BCB0-1021170DFC65}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{BE90915F-E16D-479C-9568-22A08F0FD8F9}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Serval", "Serval", "{6D20F76D-9A0E-44AC-8754-B4291C75D25B}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{25CDB05B-4E24-4A6E-933E-1E0BEC97D74D}"
Expand Down Expand Up @@ -86,6 +80,20 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{C3A14577-A65
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.ServiceToolkit", "src\ServiceToolkit\src\SIL.ServiceToolkit\SIL.ServiceToolkit.csproj", "{0E40F959-C641-40A2-9750-B17A4F9F9E55}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{9125C013-4F15-4761-BCD2-070524986737}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Serval", "Serval", "{A78D900F-AE52-436C-88CE-A22EAEDECD91}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{C1EF85B4-F2CD-407D-904D-BAC1BCE3A64C}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serval.WordAlignment", "src\Serval\src\Serval.WordAlignment\Serval.WordAlignment.csproj", "{F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Echo", "Echo", "{D201886D-9299-4758-80E8-694DBCF8DF93}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{0904BA95-D5BF-4AC2-A919-20A785EF45F5}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EchoEngine", "src\Echo\src\EchoEngine\EchoEngine.csproj", "{929FF600-8C7E-4498-A2A3-5534F3A3481E}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -132,10 +140,6 @@ Global
{4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4375A7BF-E3CE-4785-91E3-2ED6FCEB074F}.Release|Any CPU.Build.0 = Release|Any CPU
{A9D08CA2-3CF7-4BB7-A47F-5A567FFDB2CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A9D08CA2-3CF7-4BB7-A47F-5A567FFDB2CC}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A9D08CA2-3CF7-4BB7-A47F-5A567FFDB2CC}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A9D08CA2-3CF7-4BB7-A47F-5A567FFDB2CC}.Release|Any CPU.Build.0 = Release|Any CPU
{0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0C3DF75B-B022-4EFC-882C-F276F1EC8435}.Release|Any CPU.ActiveCfg = Release|Any CPU
Expand Down Expand Up @@ -180,6 +184,14 @@ Global
{0E40F959-C641-40A2-9750-B17A4F9F9E55}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0E40F959-C641-40A2-9750-B17A4F9F9E55}.Release|Any CPU.ActiveCfg = Release|Any CPU
{0E40F959-C641-40A2-9750-B17A4F9F9E55}.Release|Any CPU.Build.0 = Release|Any CPU
{F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354}.Release|Any CPU.Build.0 = Release|Any CPU
{929FF600-8C7E-4498-A2A3-5534F3A3481E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{929FF600-8C7E-4498-A2A3-5534F3A3481E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{929FF600-8C7E-4498-A2A3-5534F3A3481E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{929FF600-8C7E-4498-A2A3-5534F3A3481E}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand All @@ -195,13 +207,11 @@ Global
{8FC30758-37FC-4819-805D-8EFF3DEDF05F} = {3E753B99-7C31-42AC-B02E-012B802F58DB}
{539598C5-8634-4273-8714-A684622DDCFC} = {3E753B99-7C31-42AC-B02E-012B802F58DB}
{4375A7BF-E3CE-4785-91E3-2ED6FCEB074F} = {25CDB05B-4E24-4A6E-933E-1E0BEC97D74D}
{A9D08CA2-3CF7-4BB7-A47F-5A567FFDB2CC} = {BE90915F-E16D-479C-9568-22A08F0FD8F9}
{0C3DF75B-B022-4EFC-882C-F276F1EC8435} = {3E753B99-7C31-42AC-B02E-012B802F58DB}
{1F020042-D7B8-4541-9691-26ECFD1FFC73} = {3E753B99-7C31-42AC-B02E-012B802F58DB}
{63E4D71B-11BE-4D68-A876-5B1B5F0A4C88} = {3E753B99-7C31-42AC-B02E-012B802F58DB}
{71151518-8774-44D0-8E69-D77FA447BEFA} = {BA044B98-3136-4FDE-B90F-B0975758C07F}
{0E220C65-AA88-450E-AFB2-844E49060B3F} = {3E753B99-7C31-42AC-B02E-012B802F58DB}
{BE90915F-E16D-479C-9568-22A08F0FD8F9} = {53B18D34-B7C7-4B91-BCB0-1021170DFC65}
{25CDB05B-4E24-4A6E-933E-1E0BEC97D74D} = {6D20F76D-9A0E-44AC-8754-B4291C75D25B}
{3E753B99-7C31-42AC-B02E-012B802F58DB} = {6D20F76D-9A0E-44AC-8754-B4291C75D25B}
{92805246-5285-4F0A-9BF8-6EE4A027A41B} = {33E6965E-5A58-4C6F-882E-F17C8E88A3FF}
Expand All @@ -215,6 +225,12 @@ Global
{10657805-48F1-4205-B8F5-79447F6EF620} = {25CDB05B-4E24-4A6E-933E-1E0BEC97D74D}
{C3A14577-A654-4604-818C-4E683DD45A51} = {EA69B41C-49EF-4017-A687-44B9DF37FF98}
{0E40F959-C641-40A2-9750-B17A4F9F9E55} = {C3A14577-A654-4604-818C-4E683DD45A51}
{A78D900F-AE52-436C-88CE-A22EAEDECD91} = {9125C013-4F15-4761-BCD2-070524986737}
{C1EF85B4-F2CD-407D-904D-BAC1BCE3A64C} = {A78D900F-AE52-436C-88CE-A22EAEDECD91}
{F07B5541-4BA4-4BF8-AE1A-B44BDDCEB354} = {C1EF85B4-F2CD-407D-904D-BAC1BCE3A64C}
{D201886D-9299-4758-80E8-694DBCF8DF93} = {9125C013-4F15-4761-BCD2-070524986737}
{0904BA95-D5BF-4AC2-A919-20A785EF45F5} = {D201886D-9299-4758-80E8-694DBCF8DF93}
{929FF600-8C7E-4498-A2A3-5534F3A3481E} = {0904BA95-D5BF-4AC2-A919-20A785EF45F5}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {9F18C25E-E140-43C3-B177-D562E1628370}
Expand Down
2 changes: 1 addition & 1 deletion deploy/serval/templates/echo-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ spec:
containers:
- command:
- dotnet
- /app/echo_server/EchoTranslationEngine.dll
- /app/echo_server/EchoEngine.dll
workingDir: /app/echo_server
env:
- name: ASPNETCORE_ENVIRONMENT
Expand Down
11 changes: 10 additions & 1 deletion docker-compose.withatlas.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@ services:
- ASPNETCORE_Translation__Engines__1__Address=http://machine-engine
- ASPNETCORE_Translation__Engines__2__Type=Nmt
- ASPNETCORE_Translation__Engines__2__Address=http://machine-engine
- ASPNETCORE_WordAlignment__Engines__0__Type=EchoWordAlignment
- ASPNETCORE_WordAlignment__Engines__0__Address=http://echo
- ASPNETCORE_WordAlignment__Engines__1__Type=Statistical
- ASPNETCORE_WordAlignment__Engines__1__Address=http://machine-engine
- "Bugsnag__ApiKey=${Bugsnag__ApiKey}"
expose:
- 80
- 81
Expand Down Expand Up @@ -51,6 +56,8 @@ services:
- ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80
- ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2
- ASPNETCORE_ConnectionStrings__TranslationPlatformApi=http://serval-api:81
- ASPNETCORE_ConnectionStrings__WordAlignmentPlatformApi=http://serval-api:81
- "Bugsnag__ApiKey=${Bugsnag__ApiKey}"
expose:
- 80
ports:
Expand All @@ -61,7 +68,7 @@ services:
- .:/app:ro
- ~/.nuget/packages:/root/.nuget/packages:ro
- /var/lib/serval:/var/lib/serval
working_dir: '/app/src/Echo/src/EchoTranslationEngine'
working_dir: '/app/src/Echo/src/EchoEngine'
entrypoint:
- dotnet
- run
Expand Down Expand Up @@ -95,6 +102,7 @@ services:
- SharedFile__Uri=s3://silnlp/docker-compose/
- "SharedFile__S3AccessKeyId=${AWS_ACCESS_KEY_ID:?access key needed}"
- "SharedFile__S3SecretAccessKey=${AWS_SECRET_ACCESS_KEY:?secret key needed}"
- "Bugsnag__ApiKey=${Bugsnag__ApiKey}"
expose:
- 80
ports:
Expand Down Expand Up @@ -139,6 +147,7 @@ services:
- SharedFile__Uri=s3://silnlp/docker-compose/
- "SharedFile__S3AccessKeyId=${AWS_ACCESS_KEY_ID:?access key needed}"
- "SharedFile__S3SecretAccessKey=${AWS_SECRET_ACCESS_KEY:?secret key needed}"
- "Bugsnag__ApiKey=${Bugsnag__ApiKey}"
expose:
- 80
ports:
Expand Down
7 changes: 6 additions & 1 deletion docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ services:
- ASPNETCORE_Translation__Engines__1__Address=http://machine-engine
- ASPNETCORE_Translation__Engines__2__Type=Nmt
- ASPNETCORE_Translation__Engines__2__Address=http://machine-engine
- ASPNETCORE_WordAlignment__Engines__0__Type=EchoWordAlignment
- ASPNETCORE_WordAlignment__Engines__0__Address=http://echo
- ASPNETCORE_WordAlignment__Engines__1__Type=Statistical
- ASPNETCORE_WordAlignment__Engines__1__Address=http://machine-engine
- "Bugsnag__ApiKey=${Bugsnag__ApiKey}"
expose:
- 80
Expand Down Expand Up @@ -53,6 +57,7 @@ services:
- ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80
- ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2
- ASPNETCORE_ConnectionStrings__TranslationPlatformApi=http://serval-api:81
- ASPNETCORE_ConnectionStrings__WordAlignmentPlatformApi=http://serval-api:81
- "Bugsnag__ApiKey=${Bugsnag__ApiKey}"
expose:
- 80
Expand All @@ -64,7 +69,7 @@ services:
- .:/app:ro
- ~/.nuget/packages:/root/.nuget/packages:ro
- /var/lib/serval:/var/lib/serval
working_dir: '/app/src/Echo/src/EchoTranslationEngine'
working_dir: '/app/src/Echo/src/EchoEngine'
entrypoint:
- dotnet
- run
Expand Down
2 changes: 1 addition & 1 deletion dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ COPY . ./
RUN dotnet restore
# Build and publish a release
RUN dotnet publish ./src/Serval/src/Serval.ApiServer/Serval.ApiServer.csproj -c Release -o out_api_server
RUN dotnet publish ./src/Echo/src/EchoTranslationEngine/EchoTranslationEngine.csproj -c Release -o out_echo_server
RUN dotnet publish ./src/Echo/src/EchoEngine/EchoEngine.csproj -c Release -o out_echo_server
RUN dotnet publish ./src/Machine/src/Serval.Machine.EngineServer/Serval.Machine.EngineServer.csproj -c Release -o out_machine_engine_server
RUN dotnet publish ./src/Machine/src/Serval.Machine.JobServer/Serval.Machine.JobServer.csproj -c Release -o out_machine_job_server

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
namespace EchoTranslationEngine;
namespace EchoEngine;

public class BackgroundTaskQueue
{
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
namespace EchoTranslationEngine;
namespace EchoEngine;

public class BackgroundTaskService(
BackgroundTaskQueue taskQueue,
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
using Serval.Health.V1;

namespace EchoTranslationEngine;
namespace EchoEngine;

public class HealthServiceV1(HealthCheckService healthCheckService) : HealthApi.HealthApiBase
{
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,24 @@
using Serval.Translation.V1;
using Serval.WordAlignment.V1;

WebApplicationBuilder builder = WebApplication.CreateBuilder(args);

// Add services to the container.
builder.Services.AddGrpcClient<TranslationPlatformApi.TranslationPlatformApiClient>(o =>
{
o.Address = new Uri(builder.Configuration.GetConnectionString("TranslationPlatformApi")!);
});
builder.Services.AddGrpcClient<TranslationPlatformApi.TranslationPlatformApiClient>(
"Translation",
o =>
{
o.Address = new Uri(builder.Configuration.GetConnectionString("TranslationPlatformApi")!);
}
);
builder.Services.AddGrpcClient<WordAlignmentPlatformApi.WordAlignmentPlatformApiClient>(
"WordAlignment",
o =>
{
o.Address = new Uri(builder.Configuration.GetConnectionString("WordAlignmentPlatformApi")!);
}
);

builder.Services.AddGrpc();

builder.Services.AddHostedService<BackgroundTaskService>();
Expand All @@ -21,6 +35,8 @@
app.UseHttpsRedirection();

app.MapGrpcService<TranslationEngineServiceV1>();
app.MapGrpcService<WordAlignmentEngineServiceV1>();

app.MapGrpcService<HealthServiceV1>();

app.Run();
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"$schema": "https://json.schemastore.org/launchsettings.json",
"profiles": {
"EchoTranslationEngine": {
"EchoEngine": {
"commandName": "Project",
"dotnetRunMessages": true,
"launchBrowser": false,
Expand All @@ -11,4 +11,4 @@
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
namespace EchoTranslationEngine;
using Serval.Translation.V1;

namespace EchoEngine;

public class TranslationEngineServiceV1(BackgroundTaskQueue taskQueue) : TranslationEngineApi.TranslationEngineApiBase
{
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
global using System.Threading.Channels;
global using Bugsnag.AspNet.Core;
global using EchoTranslationEngine;
global using EchoEngine;
global using Google.Protobuf.WellKnownTypes;
global using Grpc.Core;
global using Microsoft.Extensions.Diagnostics.HealthChecks;
global using Serval.Translation.V1;
Loading

0 comments on commit 250f6e2

Please sign in to comment.