Merge branch 'main' into build_summary_gprc

Serval.sln

@@ -86,6 +86,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{C3A14577-A65
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.ServiceToolkit", "src\ServiceToolkit\src\SIL.ServiceToolkit\SIL.ServiceToolkit.csproj", "{0E40F959-C641-40A2-9750-B17A4F9F9E55}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{1DB5E6D1-17A8-4FF2-B90A-C5DFBEF63126}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.ServiceToolkit.Tests", "src\ServiceToolkit\test\SIL.ServiceToolkit.Tests\SIL.ServiceToolkit.Tests.csproj", "{C50ED15A-876D-42BF-980A-388E8C49C78D}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -180,6 +184,10 @@ Global
 		{0E40F959-C641-40A2-9750-B17A4F9F9E55}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0E40F959-C641-40A2-9750-B17A4F9F9E55}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0E40F959-C641-40A2-9750-B17A4F9F9E55}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C50ED15A-876D-42BF-980A-388E8C49C78D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C50ED15A-876D-42BF-980A-388E8C49C78D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C50ED15A-876D-42BF-980A-388E8C49C78D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C50ED15A-876D-42BF-980A-388E8C49C78D}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -215,6 +223,8 @@ Global
 		{10657805-48F1-4205-B8F5-79447F6EF620} = {25CDB05B-4E24-4A6E-933E-1E0BEC97D74D}
 		{C3A14577-A654-4604-818C-4E683DD45A51} = {EA69B41C-49EF-4017-A687-44B9DF37FF98}
 		{0E40F959-C641-40A2-9750-B17A4F9F9E55} = {C3A14577-A654-4604-818C-4E683DD45A51}
+		{1DB5E6D1-17A8-4FF2-B90A-C5DFBEF63126} = {EA69B41C-49EF-4017-A687-44B9DF37FF98}
+		{C50ED15A-876D-42BF-980A-388E8C49C78D} = {1DB5E6D1-17A8-4FF2-B90A-C5DFBEF63126}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {9F18C25E-E140-43C3-B177-D562E1628370}

deploy/qa-ext-values.yaml

@@ -1,15 +1,15 @@
 externalHost: qa.serval-api.org
 environment: Production
-deploymentVersion: '1.7.QA7'
+deploymentVersion: '1.8.QA1'
 alertEmail: [email protected]
 emailsToAlert: [email protected]
 enableTls: true
 namespace: serval
 auth0Domain: dev-sillsdev.auth0.com
 lokiTenent: serval-tenant
 lokiUrl: http://loki-distributed-gateway.loki.svc.cluster.local
-servalImage: ghcr.io/sillsdev/serval:1.7.7
-ClearMLDockerImage: ghcr.io/sillsdev/machine.py:1.7.2
+servalImage: ghcr.io/sillsdev/serval:1.8.1
+ClearMLDockerImage: ghcr.io/sillsdev/machine.py:1.8.1
 ClearMLQueue: production
 MongoConnectionPrefix: qa_
 SharedFileLocation: s3://silnlp/ext-qa/

samples/ApiExample/ApiExample.csproj

@@ -22,7 +22,7 @@
     <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
     <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="8.0.1" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
-    <PackageReference Include="Serval.Client" Version="1.7.3" />
+    <PackageReference Include="Serval.Client" Version="1.8.0" />
   </ItemGroup>
 
 </Project>

samples/ServalApp/pyproject.toml

@@ -11,6 +11,7 @@ streamlit = "^1.31.1"
 requests = "^2.31.0"
 SQLAlchemy = "^2.0.22"
 pyarrow = "^14.0.1"
+tornado = "^6.4.2"
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.10.1"

scripts/clearml_stats.py

@@ -2,7 +2,7 @@
 import json
 import os
 import pickle
-from datetime import datetime, timezone
+from datetime import datetime
 
 import numpy as np
 import pandas as pd
@@ -47,6 +47,13 @@ class clearml_stats:
     def __init__(self):
         self._client: APIClient = APIClient()
         self._tasks: dict[str, dict] = self._read_tasks()
+        self._project_id_to_task_id: dict[str, list[str]] = {}
+        for task_id in self._tasks.keys():
+            project_id = self._tasks[task_id]["project"]
+            if project_id in self._project_id_to_task_id:
+                self._project_id_to_task_id[project_id].append(task_id)
+            else:
+                self._project_id_to_task_id[project_id] = [task_id]
         self._projects: dict[str, dict] = self._read_projects()
         self._languages: pd.DataFrame = pd.read_excel(
             language_database_filename, index_col=0
@@ -306,24 +313,14 @@ def add_lang(lang):
             else:
                 langs_by_occurrence[lang] = 1
 
-        num_of_tasks_found = 0
-        num_of_tasks_not_found = 0
         for project_id in self._projects:
             self._projects[project_id]["src_lang"] = "unknown"
             self._projects[project_id]["trg_lang"] = "unknown"
             self._projects[project_id]["lang_candidates"] = []
 
             project = self._projects[project_id]
-            if len(project["tasks"]) > 0:
-                task_not_found = True
-                for task_id in project["tasks"]:
-                    if task_id in self._tasks.keys():
-                        task_not_found = False
-                        break
-                if task_not_found:
-                    num_of_tasks_not_found += 1
-                    continue
-                num_of_tasks_found += 1
+            if project_id in self._project_id_to_task_id:
+                project["tasks"] = self._project_id_to_task_id[project_id]
                 task = self._tasks[project["tasks"][0]]
                 args = task["script_args"]
                 if "src_lang" in args and "trg_lang" in args:
@@ -491,3 +488,6 @@ def violin_task_delay_time_per_week(
     axes.set_ylim(0, 8)
     axes.set_ylabel("hours")
     axes.grid(True)
+
+
+# %%

src/Echo/src/EchoTranslationEngine/Program.cs

@@ -10,6 +10,8 @@
 builder.Services.AddHostedService<BackgroundTaskService>();
 builder.Services.AddSingleton<BackgroundTaskQueue>();
 
+builder.Services.AddParallelCorpusPreprocessor();
+
 builder.Services.AddHealthChecks().AddCheck("Live", () => HealthCheckResult.Healthy());
 
 builder.Services.Configure<Bugsnag.Configuration>(builder.Configuration.GetSection("Bugsnag"));