Use MongoDB Atlas for internal QA (#348)

Fix E2E Tests - issues with setup/teardown and TrainOn Preserve all engines for later inspection when E2E testing. Fix tests - TrainOn should be init to null. Fix for inodes on serval-claim
sillsdev · Mar 22, 2024 · ec0fbb8 · ec0fbb8
1 parent 8c8e58e
commit ec0fbb8
Show file tree

Hide file tree

Showing 12 changed files with 255 additions and 43 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,5 +1,4 @@
 {
-  "dotnet-test-explorer.testProjectPath": "**/*Tests.csproj",
   "editor.formatOnSave": true,
   "[csharp]": {
     "editor.defaultFormatter": "csharpier.csharpier-vscode",

diff --git a/deploy/qa-int-values.yaml b/deploy/qa-int-values.yaml
@@ -12,6 +12,7 @@ servalImage: ghcr.io/sillsdev/serval:1.2.0
 machineImage: ghcr.io/sillsdev/machine:3.7.0
 ClearMLDockerImage: ghcr.io/sillsdev/machine.py:1.2.0
 ClearMLQueue: lambert_24gb
+MongoConnectionPrefix: qa_int_
 SharedFileLocation: s3://aqua-ml-data/int-qa/
 servalClaimSize: 1Gi
 machineClaimSize: 2Gi

diff --git a/deploy/serval/templates/machine-engine-deployment.yaml b/deploy/serval/templates/machine-engine-deployment.yaml
@@ -29,10 +29,11 @@ spec:
               value: http://*:80
             - name: ASPNETCORE_Kestrel__Endpoints__Grpc__Protocols
               value: Http2
+              {{- template "secrets.mongodb" . }}
             - name: ASPNETCORE_ConnectionStrings__Hangfire
-              value: mongodb://mongo:27017/machine_jobs?replicaSet=myRS
+              value: $(mongodb_connection)machine_jobs
             - name: ASPNETCORE_ConnectionStrings__Mongo
-              value: mongodb://mongo:27017/machine?replicaSet=myRS
+              value: $(mongodb_connection)machine
             - name: ASPNETCORE_ConnectionStrings__Serval
               value: http://serval-api:81
               {{- template "secrets.clearml" . }}
@@ -58,6 +59,10 @@ spec:
             - mountPath: /root/.aspnet/DataProtection-Keys
               name: server-keys
       hostname: machine-engine
+      dnsConfig:
+        nameservers:
+          - 8.8.8.8
+          - 208.67.222.222
       restartPolicy: Always
       volumes:
         - name: machine-mount

diff --git a/deploy/serval/templates/machine-job-deployment.yaml b/deploy/serval/templates/machine-job-deployment.yaml
@@ -29,10 +29,11 @@ spec:
               value: http://*:80
             - name: ASPNETCORE_Kestrel__Endpoints__Grpc__Protocols
               value: Http2
+              {{- template "secrets.mongodb" . }}
             - name: ASPNETCORE_ConnectionStrings__Hangfire
-              value: mongodb://mongo:27017/machine_jobs?replicaSet=myRS
+              value: $(mongodb_connection)machine_jobs
             - name: ASPNETCORE_ConnectionStrings__Mongo
-              value: mongodb://mongo:27017/machine?replicaSet=myRS
+              value: $(mongodb_connection)machine
             - name: ASPNETCORE_ConnectionStrings__Serval
               value: http://serval-api:81
               {{- template "secrets.clearml" . }}
@@ -56,6 +57,10 @@ spec:
               name: serval-mount
               readOnly: true
       hostname: machine-job
+      dnsConfig:
+        nameservers:
+          - 8.8.8.8
+          - 208.67.222.222
       restartPolicy: Always
       volumes:
         - name: machine-mount

diff --git a/deploy/serval/templates/secrets.yaml b/deploy/serval/templates/secrets.yaml
@@ -18,6 +18,15 @@
                   name: aqua-ml-data
                   key: ClearML_SecretKey
 {{- end }}
+{{- define "secrets.mongodb" }}
+            - name: mongodb_connection_part
+              valueFrom:
+                secretKeyRef:
+                  name: aqua-ml-data
+                  key: mongodb_connection
+            - name: mongodb_connection
+              value: $(mongodb_connection_part){{ .Values.MongoConnectionPrefix }}
+{{- end }}
 {{- define "secrets.s3" }}
             - name: SharedFile__Uri
               value: {{ .Values.SharedFileLocation}}

diff --git a/deploy/serval/templates/serval-api-deployment.yaml b/deploy/serval/templates/serval-api-deployment.yaml
@@ -35,10 +35,11 @@ spec:
               value: http://*:81
             - name: ASPNETCORE_Kestrel__Endpoints__Http2__Protocols
               value: Http2
+              {{- template "secrets.mongodb" . }}
             - name: ASPNETCORE_ConnectionStrings__Hangfire
-              value: mongodb://mongo:27017/serval_jobs?replicaSet=myRS
+              value: $(mongodb_connection)serval_jobs
             - name: ASPNETCORE_ConnectionStrings__Mongo
-              value: mongodb://mongo:27017/serval?replicaSet=myRS
+              value: $(mongodb_connection)serval
             - name: ASPNETCORE_Translation__Engines__0__Type
               value: Echo
             - name: ASPNETCORE_Translation__Engines__0__Address
@@ -70,6 +71,10 @@ spec:
             - mountPath: /root/.aspnet/DataProtection-Keys
               name: server-keys
       hostname: serval-api
+      dnsConfig:
+        nameservers:
+          - 8.8.8.8
+          - 208.67.222.222
       restartPolicy: Always
       volumes:
         - name: serval-mount

diff --git a/deploy/values.yaml b/deploy/values.yaml
@@ -13,6 +13,6 @@ machineImage: ghcr.io/sillsdev/machine:3.7.0
 ClearMLDockerImage: ghcr.io/sillsdev/machine.py:1.2.0
 ClearMLQueue: production
 SharedFileLocation: s3://aqua-ml-data/production/
-servalClaimSize: 5Gi
-machineClaimSize: 40Gi
+servalClaimSize: 10Gi
+machineClaimSize: 41Gi
 enableEcho: true
diff --git a/docker-compose.withatlas.yml b/docker-compose.withatlas.yml
@@ -0,0 +1,157 @@
+version: "3"
+services:
+  serval-api:
+    hostname: serval-api
+    container_name: serval_cntr
+    build:
+      context: .
+      dockerfile: dockerfile.development
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Staging
+      - ASPNETCORE_DeploymentVersion=docker-compose
+      - Auth__Domain=sil-appbuilder.auth0.com
+      - Auth__Audience=https://serval-api.org/
+      - ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80
+      - ASPNETCORE_Kestrel__Endpoints__Http2__Url=http://*:81
+      - ASPNETCORE_Kestrel__Endpoints__Http2__Protocols=Http2
+      - ASPNETCORE_ConnectionStrings__Hangfire=${MONGO_CONNECTION_STRING:?connection string needed}serval_jobs
+      - ASPNETCORE_ConnectionStrings__Mongo=${MONGO_CONNECTION_STRING:?connection string needed}serval
+      - ASPNETCORE_Translation__Engines__0__Type=Echo
+      - ASPNETCORE_Translation__Engines__0__Address=http://echo
+      - ASPNETCORE_Translation__Engines__1__Type=SmtTransfer
+      - ASPNETCORE_Translation__Engines__1__Address=http://machine-engine
+      - ASPNETCORE_Translation__Engines__2__Type=Nmt
+      - ASPNETCORE_Translation__Engines__2__Address=http://machine-engine
+    expose:
+      - 80
+      - 81
+    ports:
+      - 80:80
+    volumes:
+      - .:/app:ro
+      - ~/.nuget/packages:/root/.nuget/packages:ro
+      - /var/lib/serval:/var/lib/serval
+    working_dir: '/app/src/Serval.ApiServer'
+    entrypoint:
+      - dotnet
+      - run
+      - --no-build
+      - --no-launch-profile
+      - --additionalProbingPath
+      - /root/.nuget/packages
+
+  echo:
+    hostname: echo
+    container_name: echo_cntr
+    build:
+      context: .
+      dockerfile: dockerfile.development
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Staging
+      - ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80
+      - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2
+      - ASPNETCORE_ConnectionStrings__TranslationPlatformApi=http://serval-api:81
+    expose:
+      - 80
+    ports:
+      - 81:80
+    depends_on:
+      - serval-api
+    volumes:
+      - .:/app:ro
+      - ~/.nuget/packages:/root/.nuget/packages:ro
+      - /var/lib/serval:/var/lib/serval
+    working_dir: '/app/samples/EchoTranslationEngine'
+    entrypoint:
+      - dotnet
+      - run
+      - --no-build
+      - --no-launch-profile
+      - --additionalProbingPath
+      - /root/.nuget/packages
+
+  machine-engine:
+    hostname: machine-engine
+    container_name: machine-engine-cntr
+    build:
+      context: ${MACHINE_TESTING_DIR:-../machine}
+      dockerfile: ../machine/dockerfile.development
+
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Staging
+      - ASPNETCORE_Kestrel__Endpoints__Https__Url=http://*:80
+      - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2
+      - ASPNETCORE_ConnectionStrings__Hangfire=${MONGO_CONNECTION_STRING:?connection string needed}machine_jobs
+      - ASPNETCORE_ConnectionStrings__Mongo=${MONGO_CONNECTION_STRING:?connection string needed}machine
+      - ASPNETCORE_ConnectionStrings__Serval=http://serval-api:81
+      - ClearML__ApiServer=https://api.sil.hosted.allegro.ai
+      - ClearML__Queue=lambert_24gb
+      - ClearML__DockerImage=${MACHINE_PY_IMAGE:-ghcr.io/sillsdev/machine.py:latest}
+      - ClearML__Project=docker-compose
+      - "ClearML__AccessKey=${ClearML_AccessKey:?access key needed}"
+      - "ClearML__SecretKey=${ClearML_SecretKey:?secret key needed}"
+      - SharedFile__Uri=s3://aqua-ml-data/docker-compose/
+      - "SharedFile__S3AccessKeyId=${AWS_ACCESS_KEY_ID:?access key needed}"
+      - "SharedFile__S3SecretAccessKey=${AWS_SECRET_ACCESS_KEY:?secret key needed}"
+    expose:
+      - 80
+    ports:
+      - 82:80
+    depends_on:
+      - serval-api
+    volumes:
+      - ${MACHINE_TESTING_DIR:-../machine}:/app:ro
+      - ~/.nuget/packages:/root/.nuget/packages:ro
+      - /var/lib/machine:/var/lib/machine
+      - /var/lib/serval:/var/lib/serval
+    working_dir: '/app/src/SIL.Machine.Serval.EngineServer'
+    entrypoint:
+      - dotnet
+      - run
+      - --no-build
+      - --no-launch-profile
+      - --additionalProbingPath
+      - /root/.nuget/packages
+
+  machine-job-server:
+    hostname: machine-job-server
+    container_name: machine-job-cntr
+    build:
+      context: ${MACHINE_TESTING_DIR:-../machine}
+      dockerfile: ../machine/dockerfile.development
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Staging
+      - ASPNETCORE_ConnectionStrings__Hangfire=${MONGO_CONNECTION_STRING:?connection string needed}machine_jobs
+      - ASPNETCORE_ConnectionStrings__Mongo=${MONGO_CONNECTION_STRING:?connection string needed}machine
+      - ASPNETCORE_ConnectionStrings__Serval=http://serval-api:81
+      - ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80
+      - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2
+      - ClearML__ApiServer=https://api.sil.hosted.allegro.ai
+      - ClearML__Queue=lambert_24gb
+      - ClearML__DockerImage=${MACHINE_PY_IMAGE:-ghcr.io/sillsdev/machine.py:latest}
+      - ClearML__Project=docker-compose
+      - "ClearML__AccessKey=${ClearML_AccessKey:?access key needed}"
+      - "ClearML__SecretKey=${ClearML_SecretKey:?secret key needed}"
+      - SharedFile__Uri=s3://aqua-ml-data/docker-compose/
+      - "SharedFile__S3AccessKeyId=${AWS_ACCESS_KEY_ID:?access key needed}"
+      - "SharedFile__S3SecretAccessKey=${AWS_SECRET_ACCESS_KEY:?secret key needed}"
+    expose:
+      - 80
+    ports:
+      - 83:80
+    depends_on:
+      - machine-engine
+      - serval-api
+    volumes:
+      - ${MACHINE_TESTING_DIR:-../machine}:/app:ro
+      - ~/.nuget/packages:/root/.nuget/packages:ro
+      - /var/lib/machine:/var/lib/machine
+      - /var/lib/serval:/var/lib/serval
+    working_dir: '/app/src/SIL.Machine.Serval.JobServer'
+    entrypoint:
+      - dotnet
+      - run
+      - --no-build
+      - --no-launch-profile
+      - --additionalProbingPath
+      - /root/.nuget/packages
diff --git a/tests/Serval.E2ETests/MissingServicesTests.cs b/tests/Serval.E2ETests/MissingServicesTests.cs
@@ -6,13 +6,19 @@ public class MissingServicesTests
 {
     private ServalClientHelper _helperClient;
 
-    [SetUp]
-    public async Task Setup()
+    [OneTimeSetUp]
+    public async Task OneTimeSetup()
     {
         _helperClient = new ServalClientHelper("https://serval-api.org/", ignoreSSLErrors: true);
         await _helperClient.InitAsync();
     }
 
+    [SetUp]
+    public void Setup()
+    {
+        _helperClient.Setup();
+    }
+
     [Test]
     [Category("MongoWorking")]
     public void UseMongoAndAuth0Async()
@@ -96,6 +102,12 @@ public void UseMissingEngineServerAsync()
 
     [TearDown]
     public async Task TearDown()
+    {
+        await _helperClient.TearDown();
+    }
+
+    [OneTimeTearDown]
+    public async Task OneTimeTearDown()
     {
         await _helperClient.DisposeAsync();
     }

diff --git a/tests/Serval.E2ETests/ServalApiSlowTests.cs b/tests/Serval.E2ETests/ServalApiSlowTests.cs
@@ -7,13 +7,19 @@ public class ServalApiSlowTests
 {
     private ServalClientHelper _helperClient;
 
-    [SetUp]
-    public async Task SetUp()
+    [OneTimeSetUp]
+    public async Task OneTimeSetup()
     {
         _helperClient = new ServalClientHelper("https://serval-api.org/", ignoreSSLErrors: true);
         await _helperClient.InitAsync();
     }
 
+    [SetUp]
+    public void Setup()
+    {
+        _helperClient.Setup();
+    }
+
     [Test]
     public async Task GetSmtWholeBible()
     {
@@ -26,6 +32,12 @@ public async Task GetSmtWholeBible()
 
     [TearDown]
     public async Task TearDown()
+    {
+        await _helperClient.TearDown();
+    }
+
+    [OneTimeTearDown]
+    public async Task OneTimeTearDown()
     {
         await _helperClient.DisposeAsync();
     }

diff --git a/tests/Serval.E2ETests/ServalApiTests.cs b/tests/Serval.E2ETests/ServalApiTests.cs
@@ -13,6 +13,12 @@ public async Task OneTimeSetup()
         await _helperClient.InitAsync();
     }
 
+    [SetUp]
+    public void Setup()
+    {
+        _helperClient.Setup();
+    }
+
     [Test]
     public async Task GetEchoSuggestion()
     {
@@ -105,10 +111,7 @@ public async Task NmtBatch()
         string engineId = await _helperClient.CreateNewEngineAsync("Nmt", "es", "en", "NMT1");
         string[] books = ["MAT.txt", "1JN.txt", "2JN.txt"];
         string cId1 = await _helperClient.AddTextCorpusToEngineAsync(engineId, books, "es", "en", false);
-        _helperClient.TranslationBuildConfig.TrainOn = new List<TrainingCorpusConfig>
-        {
-            new() { CorpusId = cId1, TextIds = ["1JN.txt"] }
-        };
+        _helperClient.TranslationBuildConfig.TrainOn = [new() { CorpusId = cId1, TextIds = ["1JN.txt"] }];
         string cId2 = await _helperClient.AddTextCorpusToEngineAsync(engineId, ["3JN.txt"], "es", "en", true);
         await _helperClient.BuildEngineAsync(engineId);
         await Task.Delay(1000);
@@ -127,11 +130,7 @@ public async Task NmtQueueMultiple()
         string[] engineIds = new string[NUM_ENGINES];
         for (int i = 0; i < NUM_ENGINES; i++)
         {
-            _helperClient.TranslationBuildConfig = new()
-            {
-                Pretranslate = new List<PretranslateCorpusConfig>(),
-                Options = "{\"max_steps\":10}"
-            };
+            _helperClient.InitTranslationBuildConfig();
             engineIds[i] = await _helperClient.CreateNewEngineAsync("Nmt", "es", "en", $"NMT1_{i}");
             string engineId = engineIds[i];
             string[] books = ["MAT.txt", "1JN.txt", "2JN.txt"];
@@ -192,8 +191,6 @@ public async Task NmtLargeBatchAndDownload()
         string[] books = ["bible_LARGEFILE.txt"];
         await _helperClient.AddTextCorpusToEngineAsync(engineId, books, "es", "en", false);
         string cId = await _helperClient.AddTextCorpusToEngineAsync(engineId, ["3JN.txt"], "es", "en", true);
-        _helperClient.TranslationBuildConfig.Options =
-            "{\"max_steps\":10, \"train_params\": {\"per_device_train_batch_size\":4}}";
         await _helperClient.BuildEngineAsync(engineId);
         await Task.Delay(1000);
         IList<Pretranslation> lTrans = await _helperClient.TranslationEnginesClient.GetAllPretranslationsAsync(
@@ -403,7 +400,8 @@ public async Task ParatextProjectNmtJobAsync()
         _helperClient.TranslationBuildConfig.Pretranslate!.Add(
             new PretranslateCorpusConfig { CorpusId = corpus.Id, ScriptureRange = "JHN" }
         );
-        _helperClient.TranslationBuildConfig.Options = "{\"max_steps\":10, \"use_key_terms\":true}";
+        _helperClient.TranslationBuildConfig.Options =
+            "{\"max_steps\":10, \"use_key_terms\":true, \"train_params\": {\"per_device_train_batch_size\":4}}";
 
         await _helperClient.BuildEngineAsync(engineId);
         Assert.That(