script and internal qa updates

deploy/qa-int-values.yaml

@@ -8,8 +8,8 @@ namespace: nlp
 auth0Domain: sil-appbuilder.auth0.com
 lokiTenent: nlp-tenant
 lokiUrl: http://loki-distributed-gateway.loki.svc.cluster.local
-servalImage: ghcr.io/sillsdev/serval:1.5.5.1
-ClearMLDockerImage: ghcr.io/sillsdev/machine.py:1.5.4
+servalImage: ghcr.io/sillsdev/serval:1.6.lock_fix3
+ClearMLDockerImage: ghcr.io/sillsdev/machine.py:1.6.small_fixes0
 ClearMLQueue: lambert_24gb
 MongoConnectionPrefix: qa_int_
 SharedFileLocation: s3://silnlp/int-qa/

scripts/calculate_s3_size.py

@@ -0,0 +1,24 @@
+# %%
+import boto3
+
+bucket_name = "silnlp"
+prefix = ""
+total_sizes = {}
+
+counter = 0
+for obj in boto3.resource("s3").Bucket(bucket_name).objects.filter(Prefix=prefix):
+    key = obj.key
+    prefix = key.split("/")[0]
+    if prefix not in total_sizes:
+        total_sizes[prefix] = 0
+    total_sizes[prefix] += obj.size
+    counter += 1
+    if counter % 10_000 == 0:
+        for prefix, size in total_sizes.items():
+            print(f"{prefix}: {size/1e6:.2f} MB")
+    if counter > 1_000_000:
+        break
+for prefix, size in total_sizes.items():
+    print(f"{prefix}: {size/1e6:.2f} MB")
+
+# %%

src/Machine/src/Serval.Machine.JobServer/appsettings.json

@@ -24,7 +24,8 @@
         "Queue": "jobs_backlog",
         "DockerImage": "ghcr.io/sillsdev/machine.py:latest"
       }
-    ]
+    ],
+    "PreserveBuildFiles": true
   },
   "SmtTransferEngine": {
     "EnginesDir": "/var/lib/machine/engines"