Update configmap for tiler db and tiler

developmentseed · Dec 6, 2024 · fb5e627 · fb5e627
2 parents 2d79a28 + efb8d5b
commit fb5e627
Show file tree

Hide file tree

Showing 8 changed files with 85 additions and 109 deletions.
diff --git a/osm-seed/templates/osmcha-app/cronJob.yaml b/osm-seed/templates/osmcha-app/cronJob.yaml
@@ -11,10 +11,11 @@ metadata:
 spec:
   schedule: "*/2 * * * *"
   concurrencyPolicy: Forbid
-  successfulJobsHistoryLimit: 2
-  failedJobsHistoryLimit: 2
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
   jobTemplate:
     spec:
+      ttlSecondsAfterFinished: 120
       template:
         spec:
           containers:
@@ -32,12 +33,12 @@ spec:
           restartPolicy: Never
           {{- if .Values.osmchaApi.nodeSelector.enabled }}
           nodeSelector:
-            {{ .Values.osmchaApi.nodeSelector.label_key }} : {{ .Values.osmchaApi.nodeSelector.label_value }}
+            {{ .Values.osmchaApi.nodeSelector.label_key }}: {{ .Values.osmchaApi.nodeSelector.label_value }}
           {{- end }}
 
 ---
 apiVersion: batch/v1
-kind: CronJob
+kind: CronJob 
 metadata:
   name: {{ .Release.Name }}-osmcha-process-changesets-cronjob
   labels:
@@ -48,10 +49,11 @@ metadata:
 spec:
   schedule: "0 * * * *"
   concurrencyPolicy: Forbid
-  successfulJobsHistoryLimit: 2
-  failedJobsHistoryLimit: 2
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
   jobTemplate:
     spec:
+      ttlSecondsAfterFinished: 120
       template:
         spec:
           containers:
@@ -62,13 +64,19 @@ spec:
             - /bin/sh
             - -c
             - |
-              python manage.py backfill_changesets $(date -d "yesterday" +%Y-%m-%d) $(date +%Y-%m-%d)
+              YESTERDAY=$(date -d "yesterday" +%Y-%m-%d)
+              TODAY=$(date +%Y-%m-%d)
+
+              python manage.py backfill_changesets "$YESTERDAY" "$TODAY"
+              EXIT_CODE=$?
+
+              exit $EXIT_CODE
             envFrom:
             - configMapRef:
                 name: {{ .Release.Name }}-osmcha-common-env
           restartPolicy: Never
           {{- if .Values.osmchaApi.nodeSelector.enabled }}
           nodeSelector:
-            {{ .Values.osmchaApi.nodeSelector.label_key }} : {{ .Values.osmchaApi.nodeSelector.label_value }}
+            {{ .Values.osmchaApi.nodeSelector.label_key }}: {{ .Values.osmchaApi.nodeSelector.label_value }}
           {{- end }}
 {{- end }}
diff --git a/osm-seed/templates/tiler-db/tiler-db-statefulset.yaml b/osm-seed/templates/tiler-db/tiler-db-statefulset.yaml
@@ -33,24 +33,7 @@ spec:
               protocol: TCP
           envFrom:
           - configMapRef:
-              name: {{ .Release.Name }}-tiler-db-common-env
-          # env:
-          #   # - name: GET_HOSTS_FROM
-          #   #   value: dns
-          #   - name: POSTGRES_HOST
-          #     value: {{ .Release.Name }}-tiler-db
-          #   - name: POSTGRES_DB
-          #     value: {{ .Values.tilerDb.env.POSTGRES_DB }}
-          #   - name: POSTGRES_PASSWORD
-          #     value: {{ quote .Values.tilerDb.env.POSTGRES_PASSWORD }}
-          #   - name: POSTGRES_USER
-          #     value: {{ default "postgres" .Values.tilerDb.env.POSTGRES_USER | quote }}
-          #   - name: PGUSER
-          #     value: {{ default "postgres" .Values.tilerDb.env.POSTGRES_USER | quote }}
-          #   - name: PGDATA
-          #     value: {{ .Values.tilerDb.persistenceDisk.mountPath }}
-            # - name: POD_IP
-            #   valueFrom: { fieldRef: { fieldPath: status.podIP } }
+              name: {{ .Release.Name }}-tiler-db-cm
            {{- if and .Values.tilerDb.postgresqlConfig.enabled .Values.tilerDb.persistenceDisk.enabled }}
           command:
             - "sh"
@@ -83,6 +66,8 @@ spec:
             - name: tiler-db-postgresql-config
               mountPath: /etc/postgresql/
             {{- end }}
+            - name: shared-memory
+              mountPath: /dev/shm
           {{- if .Values.tilerDb.resources.enabled }}
           resources:
             requests:
@@ -105,6 +90,10 @@ spec:
           configMap:
             name: {{ .Release.Name }}-tiler-db-postgresql-config
         {{- end }}
+        - name: shared-memory
+          emptyDir:
+            medium: Memory
+            sizeLimit: {{ .Values.tilerDb.sharedMemorySize | default "64Mi" }}
       {{- if .Values.tilerDb.nodeSelector.enabled }}
       nodeSelector:
         {{ .Values.tilerDb.nodeSelector.label_key }} : {{ .Values.tilerDb.nodeSelector.label_value }}

diff --git a/osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml b/osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml
@@ -42,9 +42,6 @@ spec:
             memory: {{ .Values.tilerImposm.resources.limits.memory }}
             cpu: {{ .Values.tilerImposm.resources.limits.cpu }}
         {{- end }}
-        envFrom:
-        - configMapRef:
-            name: {{ .Release.Name }}-tiler-db-common-env
         env:
           - name: TILER_IMPORT_FROM
             value: {{ .Values.tilerImposm.env.TILER_IMPORT_FROM | quote }}
@@ -76,6 +73,9 @@ spec:
           - name: GCP_STORAGE_BUCKET
             value: {{ .Values.GCP_STORAGE_BUCKET }}
           {{- end }}
+        envFrom:
+        - configMapRef:
+            name: {{ .Release.Name }}-tiler-db-cm
         volumeMounts:
         - mountPath: {{ .Values.tilerImposm.persistenceDisk.mountPath }}
           name: tiler-imposm-storage

diff --git a/osm-seed/templates/tiler-server/tiler-server-cache-cleaner-deployment.yaml b/osm-seed/templates/tiler-server/tiler-server-cache-cleaner-deployment.yaml
@@ -33,53 +33,9 @@ spec:
               memory: {{ .Values.tilerServerCacheCleaner.resources.limits.memory }}
               cpu: {{ .Values.tilerServerCacheCleaner.resources.limits.cpu }}
           {{- end }}
-          env:
-              # tiler-db env variables
-            - name: POSTGRES_HOST
-              value: {{ .Release.Name }}-tiler-db
-            - name: POSTGRES_DB
-              value: {{ .Values.tilerDb.env.POSTGRES_DB }}
-            - name: POSTGRES_PASSWORD
-              value: {{ quote .Values.tilerDb.env.POSTGRES_PASSWORD }}
-            - name: POSTGRES_USER
-              value: {{ .Values.tilerDb.env.POSTGRES_USER }}
-            - name: POSTGRES_PORT
-              value: {{ .Values.tilerDb.env.POSTGRES_PORT | quote }}
-              # tiler-server env variables
-            - name: TILER_SERVER_PORT
-              value: {{ .Values.tilerServer.env.TILER_SERVER_PORT | quote }}
-            - name: TILER_CACHE_TYPE
-              value: {{ .Values.tilerServer.env.TILER_CACHE_TYPE }}
-            - name: TILER_CACHE_REGION
-              value: {{ .Values.tilerServer.env.TILER_CACHE_REGION }}
-            - name: TILER_CACHE_BUCKET
-              value: {{ .Values.tilerServer.env.TILER_CACHE_BUCKET }}
-            - name: TILER_CACHE_BASEPATH
-              value: {{ quote .Values.tilerServer.env.TILER_CACHE_BASEPATH }}
-            - name: TILER_CACHE_AWS_ACCESS_KEY_ID
-              value: {{ quote .Values.tilerServer.env.TILER_CACHE_AWS_ACCESS_KEY_ID }}
-            - name: TILER_CACHE_AWS_SECRET_ACCESS_KEY
-              value: {{ quote .Values.tilerServer.env.TILER_CACHE_AWS_SECRET_ACCESS_KEY }}
-            - name: TILER_CACHE_MAX_ZOOM
-              value: {{ quote .Values.tilerServer.env.TILER_CACHE_MAX_ZOOM }}
-            - name: MAX_NUM_PS
-              value: {{ quote .Values.tilerServerCacheCleaner.env.MAX_NUM_PS }}
-            - name: PROCESS_NAME
-              value: {{ .Values.tilerServerCacheCleaner.env.PROCESS_NAME }}
-            - name: KILL_PROCESS
-              value: {{ .Values.tilerServerCacheCleaner.env.KILL_PROCESS }}
-            - name: CLOUDPROVIDER
-              value: {{ .Values.cloudProvider }}
-            # In case cloudProvider=aws
-            {{- if eq .Values.cloudProvider "aws" }}
-            - name: AWS_S3_BUCKET
-              value: {{ .Values.AWS_S3_BUCKET }}
-            {{- end }}
-            # In case cloudProvider=gcp
-            {{- if eq .Values.cloudProvider "gcp" }}
-            - name: GCP_STORAGE_BUCKET
-              value: {{ .Values.GCP_STORAGE_BUCKET }}
-            {{- end }}
+          envFrom:
+          - configMapRef:
+              name: {{ .Release.Name }}-tiler-server-cm
       {{- if .Values.tilerServerCacheCleaner.nodeSelector.enabled }}
       nodeSelector:
         {{ .Values.tilerServerCacheCleaner.nodeSelector.label_key }} : {{ .Values.tilerServerCacheCleaner.nodeSelector.label_value }}

diff --git a/osm-seed/templates/tiler-server/tiler-server-configMap.yaml b/osm-seed/templates/tiler-server/tiler-server-configMap.yaml
@@ -0,0 +1,29 @@
+{{- if .Values.tilerServer.enabled -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-tiler-server-cm
+data:
+    POSTGRES_HOST: {{ .Release.Name }}-tiler-db
+    POSTGRES_DB: {{ .Values.tilerDb.env.POSTGRES_DB | quote }}
+    POSTGRES_PASSWORD: {{ .Values.tilerDb.env.POSTGRES_PASSWORD | quote }}
+    POSTGRES_USER: {{ .Values.tilerDb.env.POSTGRES_USER | quote }}
+    POSTGRES_PORT: {{ .Values.tilerDb.env.POSTGRES_PORT | quote }}
+    TILER_SERVER_PORT: {{ .Values.tilerServer.env.TILER_SERVER_PORT | quote }}
+    TILER_CACHE_TYPE: {{ .Values.tilerServer.env.TILER_CACHE_TYPE | quote }}
+    TILER_CACHE_REGION: {{ .Values.tilerServer.env.TILER_CACHE_REGION | quote }}
+    TILER_CACHE_BUCKET: {{ .Values.tilerServer.env.TILER_CACHE_BUCKET | quote }}
+    TILER_CACHE_BASEPATH: {{ .Values.tilerServer.env.TILER_CACHE_BASEPATH | quote }}
+    TILER_CACHE_AWS_ACCESS_KEY_ID: {{ .Values.tilerServer.env.TILER_CACHE_AWS_ACCESS_KEY_ID | quote }}
+    TILER_CACHE_AWS_SECRET_ACCESS_KEY: {{ .Values.tilerServer.env.TILER_CACHE_AWS_SECRET_ACCESS_KEY | quote }}
+    TILER_CACHE_MAX_ZOOM: {{ .Values.tilerServer.env.TILER_CACHE_MAX_ZOOM | quote }}
+    EXECUTE_REINDEX: {{ .Values.tilerServer.env.EXECUTE_REINDEX | default "false" | quote }}
+    EXECUTE_VACUUM_ANALYZE: {{ .Values.tilerServer.env.EXECUTE_VACUUM_ANALYZE | default "false" | quote }}
+    CLOUDPROVIDER: {{ .Values.cloudProvider | quote }}
+    {{- if eq .Values.cloudProvider "aws" }}
+    AWS_S3_BUCKET: {{ .Values.AWS_S3_BUCKET | quote }}
+    {{- end }}
+    {{- if eq .Values.cloudProvider "gcp" }}
+    GCP_STORAGE_BUCKET: {{ .Values.GCP_STORAGE_BUCKET | quote }}
+    {{- end }}
+{{- end }}
diff --git a/osm-seed/templates/tiler-server/tiler-server-deployment.yaml b/osm-seed/templates/tiler-server/tiler-server-deployment.yaml
@@ -48,36 +48,9 @@ spec:
           {{- end }}
           envFrom:
           - configMapRef:
-              name: {{ .Release.Name }}-tiler-db-common-env
-          env:
-            - name: TILER_SERVER_PORT
-              value: {{ .Values.tilerServer.env.TILER_SERVER_PORT | quote }}
-            - name: TILER_CACHE_TYPE
-              value: {{ .Values.tilerServer.env.TILER_CACHE_TYPE }}
-            - name: TILER_CACHE_REGION
-              value: {{ .Values.tilerServer.env.TILER_CACHE_REGION }}
-            - name: TILER_CACHE_BUCKET
-              value: {{ .Values.tilerServer.env.TILER_CACHE_BUCKET }}
-            - name: TILER_CACHE_BASEPATH
-              value: {{ quote .Values.tilerServer.env.TILER_CACHE_BASEPATH }}
-            - name: TILER_CACHE_AWS_ACCESS_KEY_ID
-              value: {{ quote .Values.tilerServer.env.TILER_CACHE_AWS_ACCESS_KEY_ID }}
-            - name: TILER_CACHE_AWS_SECRET_ACCESS_KEY
-              value: {{ quote .Values.tilerServer.env.TILER_CACHE_AWS_SECRET_ACCESS_KEY }}
-            - name: TILER_CACHE_MAX_ZOOM
-              value: {{ quote .Values.tilerServer.env.TILER_CACHE_MAX_ZOOM }}
-            - name: CLOUDPROVIDER
-              value: {{ .Values.cloudProvider }}
-            # In case cloudProvider=aws
-            {{- if eq .Values.cloudProvider "aws" }}
-            - name: AWS_S3_BUCKET
-              value: {{ .Values.AWS_S3_BUCKET }}
-            {{- end }}
-            # In case cloudProvider=gcp
-            {{- if eq .Values.cloudProvider "gcp" }}
-            - name: GCP_STORAGE_BUCKET
-              value: {{ .Values.GCP_STORAGE_BUCKET }}
-            {{- end }}
+              name: {{ .Release.Name }}-tiler-server-cm
+           - configMapRef:
+              name: {{ .Release.Name }}-tiler-db-cm
       {{- if .Values.tilerServer.nodeSelector.enabled }}
       nodeSelector:
         {{ .Values.tilerServer.nodeSelector.label_key }} : {{ .Values.tilerServer.nodeSelector.label_value }}

diff --git a/osm-seed/templates/tiler-server/tiler-server-statefulset.yaml b/osm-seed/templates/tiler-server/tiler-server-statefulset.yaml
@@ -47,9 +47,6 @@ spec:
               memory: {{ .Values.tilerServer.resources.limits.memory }}
               cpu: {{ .Values.tilerServer.resources.limits.cpu }}
           {{- end }}
-          envFrom:
-          - configMapRef:
-              name: {{ .Release.Name }}-tiler-db-common-env
           env:
             - name: TILER_SERVER_PORT
               value: {{ .Values.tilerServer.env.TILER_SERVER_PORT | quote }}
@@ -79,6 +76,11 @@ spec:
             - name: GCP_STORAGE_BUCKET
               value: {{ .Values.GCP_STORAGE_BUCKET }}
             {{- end }}
+          envFrom:
+          - configMapRef:
+              name: {{ .Release.Name }}-tiler-server-cm
+           - configMapRef:
+              name: {{ .Release.Name }}-tiler-db-cm
           volumeMounts:
           - mountPath: /mnt/data
             name: tiler-server-storage

diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml
@@ -378,6 +378,7 @@ tilerDb:
     # GCP
     GCP_gcePersistentDisk_pdName: osmseed-disk-tilerdb-v1
     GCP_gcePersistentDisk_size: 50Gi
+  sharedMemorySize: 64Mi
   resources:
     enabled: false
     requests:
@@ -425,6 +426,7 @@ tilerImposm:
     UPLOAD_EXPIRED_FILES: true
     IMPORT_NATURAL_EARTH: true
     IMPORT_OSM_LAND: true
+    IMPOSM3_IMPORT_LAYERS: all
   persistenceDisk:
     enabled: false
     accessMode: ReadWriteOnce
@@ -471,6 +473,8 @@ tilerServer:
     TILER_CACHE_REGION: us-east-1
     TILER_CACHE_AWS_ACCESS_KEY_ID: xyz
     TILER_CACHE_AWS_SECRET_ACCESS_KEY: xyz/q
+    EXECUTE_REINDEX: false
+    EXECUTE_VACUUM_ANALYZE: false
   command: "./start.sh" # ./start.sh & ./tile_cache_downloader.sh & ./expire-watcher.sh
   # In case you use TILER_CACHE_TYPE: file with  persistenceDisk
   persistenceDisk:
@@ -528,6 +532,21 @@ tilerServerCacheCleaner:
     minReplicas: 1
     maxReplicas: 2
     cpuUtilization: 60
+
+tilerCacheCleanerJob:
+  enabled: false
+  tilesFile: imposm/imposm3_expire_dir/latest.tiles
+  minZoom: 0
+  maxZoom: 20
+  overwrite: true
+  resources:
+    enabled: false
+    requests:
+      memory: "1Gi"
+      cpu: "500m"
+    limits:
+      memory: "2Gi"
+      cpu: "1"
 # ====================================================================================================
 # Variables for Tasking Manager DB
 # ====================================================================================================