From 3a6f024c850901307fe6011eb9b2b684a71fb815 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Fri, 13 Oct 2023 13:54:33 -0400 Subject: [PATCH] addresses https://github.com/sillsdev/serval/issues/178 (#179) Clearml project and update k8s to new clearml prefix Change DummyBuild to Staging (standard naming) Create staging env for Serval and Echo Remove ClearML_MaxSteps. Remove ASPNETCORE_TranslationEngines params - unneeded --- README.md | 18 ++++++++++++- deploy/qa-ext-values.yaml | 2 +- deploy/qa-int-values.yaml | 2 +- deploy/serval/templates/echo-deployment.yaml | 6 ++--- .../templates/machine-engine-deployment.yaml | 10 ++----- .../templates/machine-job-deployment.yaml | 10 ++----- deploy/serval/templates/secrets.yaml | 14 +++++----- .../templates/serval-api-deployment.yaml | 6 ++--- deploy/values.yaml | 2 +- docker-compose.yml | 26 +++++++------------ .../appsettings.Production.json | 8 ++++++ .../appsettings.Staging.json | 8 ++++++ .../appsettings.Production.json | 8 ++++++ src/Serval.ApiServer/appsettings.Staging.json | 8 ++++++ 14 files changed, 77 insertions(+), 51 deletions(-) create mode 100644 samples/EchoTranslationEngine/appsettings.Production.json create mode 100644 samples/EchoTranslationEngine/appsettings.Staging.json create mode 100644 src/Serval.ApiServer/appsettings.Production.json create mode 100644 src/Serval.ApiServer/appsettings.Staging.json diff --git a/README.md b/README.md index 5b37120c..4ab3663b 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,14 @@ Build the Machine repo: ``` dotnet build machine ``` -Make sure that the environment variable ASPNETCORE_ENVIRONMENT is set to "Development" by running `export ASPNETCORE_ENVIRONMENT=Development` or adding it to your `.bashrc`. +Now, if you are using a local docker image and have a GPU with at least 12 GB of RAM: +* Clone the [Machine repo](https://github.com/sillsdev/machine.py) into an adjacent folder to Serval: +* Build the docker image with `docker build . -t mpy.local` +* Register your machine as a ClearML agent (see dev team for details) +* In docker-compose.yml, update `ClearML__Queue` to be the queue that your computer (agent) is listening to +If you will use the standard Machine.py images: +* In docker-compose.yml, update `ClearML__Queue` and `ClearML__DockerImage` to the appropriate values + In the Serval root, run docker compose up ``` cd serval && docker compose up @@ -70,6 +77,15 @@ There are 3 different environments that Serval is deployed to: - `kubectl config use-context aws-rke` - `helm upgrade serval deploy/serval -n serval -f deploy/values.yaml` +### Environments: +- Production: + - Full deployment, full NMT and SMT builds +- Staging: + - Full Deploymnet, dummy NMT building (10 steps, small model) + - Also used for Docker-compose local staging +- Development: + - Non-docker use only + ## Debugging ### To access Serval API * Internal QA: diff --git a/deploy/qa-ext-values.yaml b/deploy/qa-ext-values.yaml index 73f2f55b..90cc16ce 100644 --- a/deploy/qa-ext-values.yaml +++ b/deploy/qa-ext-values.yaml @@ -1,4 +1,5 @@ externalHost: qa.serval-api.org +environment: Staging enableTls: true namespace: serval auth0Domain: dev-sillsdev.auth0.com @@ -7,7 +8,6 @@ lokiUrl: http://loki-distributed-gateway.loki.svc.cluster.local servalImage: ghcr.io/sillsdev/serval:0.7.1 machineImage: ghcr.io/sillsdev/machine:3.3.2 ClearMLDockerImage: ghcr.io/sillsdev/machine.py:0.9.5 -ClearMLMaxSteps: 10 ClearMLQueue: lambert_24gb SharedFileLocation: s3://aqua-ml-data/ext-qa/ enableEcho: true \ No newline at end of file diff --git a/deploy/qa-int-values.yaml b/deploy/qa-int-values.yaml index c1ac1160..091494ee 100644 --- a/deploy/qa-int-values.yaml +++ b/deploy/qa-int-values.yaml @@ -1,4 +1,5 @@ externalHost: qa-int.serval-api.org +environment: Staging enableTls: true namespace: nlp auth0Domain: sil-appbuilder.auth0.com @@ -7,7 +8,6 @@ lokiUrl: http://loki-distributed-gateway.loki.svc.cluster.local servalImage: ghcr.io/sillsdev/serval:0.7.1 machineImage: ghcr.io/sillsdev/machine:3.3.2 ClearMLDockerImage: ghcr.io/sillsdev/machine.py:0.9.5 -ClearMLMaxSteps: 10 ClearMLQueue: lambert_24gb SharedFileLocation: s3://aqua-ml-data/int-qa/ enableEcho: true \ No newline at end of file diff --git a/deploy/serval/templates/echo-deployment.yaml b/deploy/serval/templates/echo-deployment.yaml index 25b697c9..fd53f07e 100644 --- a/deploy/serval/templates/echo-deployment.yaml +++ b/deploy/serval/templates/echo-deployment.yaml @@ -24,16 +24,14 @@ spec: - /app/echo_server/EchoTranslationEngine.dll workingDir: /app/echo_server env: + - name: ASPNETCORE_ENVIRONMENT + value: {{ .Values.environment}} - name: ASPNETCORE_Kestrel__Endpoints__Grpc__Url value: http://*:80 - name: ASPNETCORE_Kestrel__Endpoints__Grpc__Protocols value: Http2 - name: ASPNETCORE_ConnectionStrings__TranslationPlatformApi value: http://serval-api:81 - - name: ASPNETCORE_Logging__LogLevel__Default - value: Information - - name: ASPNETCORE_Logging__LogLevel__Microsoft.AspNetCore - value: Warning image: {{ .Values.servalImage}} imagePullPolicy: "Always" name: echo diff --git a/deploy/serval/templates/machine-engine-deployment.yaml b/deploy/serval/templates/machine-engine-deployment.yaml index c5a6e29f..7b7c0b02 100644 --- a/deploy/serval/templates/machine-engine-deployment.yaml +++ b/deploy/serval/templates/machine-engine-deployment.yaml @@ -23,6 +23,8 @@ spec: - /app/engine_server/SIL.Machine.Serval.EngineServer.dll workingDir: /app/engine_server env: + - name: ASPNETCORE_ENVIRONMENT + value: {{ .Values.environment }} - name: ASPNETCORE_Kestrel__Endpoints__Grpc__Url value: http://*:80 - name: ASPNETCORE_Kestrel__Endpoints__Grpc__Protocols @@ -33,14 +35,6 @@ spec: value: mongodb://mongo:27017/machine?replicaSet=myRS - name: ASPNETCORE_ConnectionStrings__Serval value: http://serval-api:81 - - name: ASPNETCORE_Logging__LogLevel__Default - value: Information - - name: ASPNETCORE_Logging__LogLevel__Microsoft.AspNetCore - value: Warning - - name: ASPNETCORE_TranslationEngines__0 - value: SmtTransfer - - name: ASPNETCORE_TranslationEngines__1 - value: Nmt {{- template "secrets.clearml" . }} {{- template "secrets.s3" . }} image: {{ .Values.machineImage}} diff --git a/deploy/serval/templates/machine-job-deployment.yaml b/deploy/serval/templates/machine-job-deployment.yaml index 55d50f46..a7b5a953 100644 --- a/deploy/serval/templates/machine-job-deployment.yaml +++ b/deploy/serval/templates/machine-job-deployment.yaml @@ -23,6 +23,8 @@ spec: - /app/job_server/SIL.Machine.Serval.JobServer.dll workingDir: /app/job_server env: + - name: ASPNETCORE_ENVIRONMENT + value: {{ .Values.environment}} - name: ASPNETCORE_Kestrel__Endpoints__Grpc__Url value: http://*:80 - name: ASPNETCORE_Kestrel__Endpoints__Grpc__Protocols @@ -33,14 +35,6 @@ spec: value: mongodb://mongo:27017/machine?replicaSet=myRS - name: ASPNETCORE_ConnectionStrings__Serval value: http://serval-api:81 - - name: ASPNETCORE_Logging__LogLevel__Default - value: Information - - name: ASPNETCORE_Logging__LogLevel__Microsoft.AspNetCore - value: Warning - - name: ASPNETCORE_TranslationEngines__0 - value: SmtTransfer - - name: ASPNETCORE_TranslationEngines__1 - value: Nmt {{- template "secrets.clearml" . }} {{- template "secrets.s3" . }} image: {{ .Values.machineImage}} diff --git a/deploy/serval/templates/secrets.yaml b/deploy/serval/templates/secrets.yaml index 0443a256..dccb52f7 100644 --- a/deploy/serval/templates/secrets.yaml +++ b/deploy/serval/templates/secrets.yaml @@ -1,18 +1,18 @@ {{- define "secrets.clearml" }} - - name: ClearMLNmtEngine__ApiServer + - name: ClearML__ApiServer value: https://api.sil.hosted.allegro.ai - - name: ClearMLNmtEngine__Queue + - name: ClearML__Queue value: {{ .Values.ClearMLQueue}} - - name: ClearMLNmtEngine__DockerImage + - name: ClearML__DockerImage value: {{ .Values.ClearMLDockerImage}} - - name: ClearMLNmtEngine__MaxSteps - value: "{{ .Values.ClearMLMaxSteps}}" - - name: ClearMLNmtEngine__AccessKey + - name: ClearML__Project + value: {{ .Values.externalHost}} + - name: ClearML__AccessKey valueFrom: secretKeyRef: name: aqua-ml-data key: ClearML_AccessKey - - name: ClearMLNmtEngine__SecretKey + - name: ClearML__SecretKey valueFrom: secretKeyRef: name: aqua-ml-data diff --git a/deploy/serval/templates/serval-api-deployment.yaml b/deploy/serval/templates/serval-api-deployment.yaml index ff563720..76444243 100644 --- a/deploy/serval/templates/serval-api-deployment.yaml +++ b/deploy/serval/templates/serval-api-deployment.yaml @@ -25,6 +25,8 @@ spec: env: - name: Auth__Domain value: {{ .Values.auth0Domain}} + - name: ASPNETCORE_ENVIRONMENT + value: {{ .Values.environment}} - name: ASPNETCORE_Kestrel__Endpoints__Http__Url value: http://*:80 - name: ASPNETCORE_Kestrel__Endpoints__Http2__Url @@ -35,10 +37,6 @@ spec: value: mongodb://mongo:27017/serval_jobs?replicaSet=myRS - name: ASPNETCORE_ConnectionStrings__Mongo value: mongodb://mongo:27017/serval?replicaSet=myRS - - name: ASPNETCORE_Logging__LogLevel__Default - value: Information - - name: ASPNETCORE_Logging__LogLevel__Microsoft.AspNetCore - value: Warning - name: ASPNETCORE_Translation__Engines__0__Type value: Echo - name: ASPNETCORE_Translation__Engines__0__Address diff --git a/deploy/values.yaml b/deploy/values.yaml index b6cdac3e..14f328b5 100644 --- a/deploy/values.yaml +++ b/deploy/values.yaml @@ -1,4 +1,5 @@ externalHost: prod.serval-api.org +environment: Production enableTls: true namespace: serval auth0Domain: languagetechnology.auth0.com @@ -7,7 +8,6 @@ lokiUrl: http://loki-distributed-gateway.loki.svc.cluster.local servalImage: ghcr.io/sillsdev/serval:0.7.1 machineImage: ghcr.io/sillsdev/machine:3.3.2 ClearMLDockerImage: ghcr.io/sillsdev/machine.py:0.9.5 -ClearMLMaxSteps: 20000 ClearMLQueue: production SharedFileLocation: s3://aqua-ml-data/production/ enableEcho: true \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 61d8a9a9..8f15b3fa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,7 @@ services: context: . dockerfile: dockerfile.development environment: + - ASPNETCORE_ENVIRONMENT=Staging - Auth__Domain=sil-appbuilder.auth0.com - Auth__Audience=https://machine.sil.org - ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80 @@ -14,8 +15,6 @@ services: - ASPNETCORE_Kestrel__Endpoints__Http2__Protocols=Http2 - ASPNETCORE_ConnectionStrings__Hangfire=mongodb://mongo:27017/serval_jobs?replicaSet=myRS - ASPNETCORE_ConnectionStrings__Mongo=mongodb://mongo:27017/serval?replicaSet=myRS - - ASPNETCORE_Logging__LogLevel__Default=Information - - ASPNETCORE_Logging__LogLevel__Microsoft.AspNetCore=Warning - ASPNETCORE_Translation__Engines__0__Type=Echo - ASPNETCORE_Translation__Engines__0__Address=http://echo - ASPNETCORE_Translation__Engines__1__Type=SmtTransfer @@ -49,10 +48,9 @@ services: context: . dockerfile: dockerfile.development environment: + - ASPNETCORE_ENVIRONMENT=Staging - ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80 - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2 - - ASPNETCORE_Logging__LogLevel__Default=Information - - ASPNETCORE_Logging__LogLevel__Microsoft.AspNetCore=Warning - ASPNETCORE_ConnectionStrings__TranslationPlatformApi=http://serval-api:81 expose: - 80 @@ -81,18 +79,16 @@ services: dockerfile: ../machine/dockerfile.development environment: + - ASPNETCORE_ENVIRONMENT=Staging - ASPNETCORE_Kestrel__Endpoints__Https__Url=http://*:80 - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2 - ASPNETCORE_ConnectionStrings__Hangfire=mongodb://mongo:27017/machine_jobs?replicaSet=myRS - ASPNETCORE_ConnectionStrings__Mongo=mongodb://mongo:27017/machine?replicaSet=myRS - ASPNETCORE_ConnectionStrings__Serval=http://serval-api:81 - - ASPNETCORE_Logging__LogLevel__Default=Information - - ASPNETCORE_Logging__LogLevel__Microsoft.AspNetCore=Warning - - ASPNETCORE_TranslationEngines__0=SmtTransfer - - ASPNETCORE_TranslationEngines__1=Nmt - ClearML__ApiServer=https://api.sil.hosted.allegro.ai - - ClearML__Queue=production - - ClearML__DockerImage=ghcr.io/sillsdev/machine.py:0.9.5.1 + - ClearML__Queue=lambert_24gb + - ClearML__DockerImage=mpy.local + - ClearML__Project=docker-compose - "ClearML__AccessKey=${ClearML_AccessKey:?access key needed}" - "ClearML__SecretKey=${ClearML_SecretKey:?secret key needed}" - SharedFile__Uri=s3://aqua-ml-data/docker-compose/ @@ -126,18 +122,16 @@ services: context: ../machine dockerfile: ../machine/dockerfile.development environment: + - ASPNETCORE_ENVIRONMENT=Staging - ASPNETCORE_ConnectionStrings__Hangfire=mongodb://mongo:27017/machine_jobs?replicaSet=myRS - ASPNETCORE_ConnectionStrings__Mongo=mongodb://mongo:27017/machine?replicaSet=myRS - ASPNETCORE_ConnectionStrings__Serval=http://serval-api:81 - - ASPNETCORE_Logging__LogLevel__Default=Information - - ASPNETCORE_Logging__LogLevel__Microsoft.AspNetCore=Warning - ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80 - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2 - - ASPNETCORE_TranslationEngines__0=SmtTransfer - - ASPNETCORE_TranslationEngines__1=Nmt - ClearML__ApiServer=https://api.sil.hosted.allegro.ai - - ClearML__Queue=production - - ClearML__DockerImage=ghcr.io/sillsdev/machine.py:0.9.5.1 + - ClearML__Queue=lambert_24gb + - ClearML__DockerImage=mpy.local + - ClearML__Project=docker-compose - "ClearML__AccessKey=${ClearML_AccessKey:?access key needed}" - "ClearML__SecretKey=${ClearML_SecretKey:?secret key needed}" - SharedFile__Uri=s3://aqua-ml-data/docker-compose/ diff --git a/samples/EchoTranslationEngine/appsettings.Production.json b/samples/EchoTranslationEngine/appsettings.Production.json new file mode 100644 index 00000000..1b2d3baf --- /dev/null +++ b/samples/EchoTranslationEngine/appsettings.Production.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} \ No newline at end of file diff --git a/samples/EchoTranslationEngine/appsettings.Staging.json b/samples/EchoTranslationEngine/appsettings.Staging.json new file mode 100644 index 00000000..1b2d3baf --- /dev/null +++ b/samples/EchoTranslationEngine/appsettings.Staging.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} \ No newline at end of file diff --git a/src/Serval.ApiServer/appsettings.Production.json b/src/Serval.ApiServer/appsettings.Production.json new file mode 100644 index 00000000..1b2d3baf --- /dev/null +++ b/src/Serval.ApiServer/appsettings.Production.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} \ No newline at end of file diff --git a/src/Serval.ApiServer/appsettings.Staging.json b/src/Serval.ApiServer/appsettings.Staging.json new file mode 100644 index 00000000..1b2d3baf --- /dev/null +++ b/src/Serval.ApiServer/appsettings.Staging.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} \ No newline at end of file