From 11ffe2a2f34fe740600cfd30074af4ffbc768bd8 Mon Sep 17 00:00:00 2001 From: Avi Moondra Date: Thu, 30 Nov 2023 13:45:50 -0500 Subject: [PATCH 1/2] support 1 replica api backend with jobs runner (#131) --- charts/retool/Chart.yaml | 2 +- charts/retool/templates/_helpers.tpl | 12 +++++ .../retool/templates/deployment_backend.yaml | 2 +- charts/retool/templates/deployment_jobs.yaml | 2 +- charts/retool/values.yaml | 47 ++++++++++++------- values.yaml | 47 ++++++++++++------- 6 files changed, 75 insertions(+), 37 deletions(-) diff --git a/charts/retool/Chart.yaml b/charts/retool/Chart.yaml index 8666b13f..19794ca9 100644 --- a/charts/retool/Chart.yaml +++ b/charts/retool/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: retool description: A Helm chart for Kubernetes type: application -version: 6.0.6 +version: 6.0.7 maintainers: - name: Retool Engineering email: engineering+helm@retool.com diff --git a/charts/retool/templates/_helpers.tpl b/charts/retool/templates/_helpers.tpl index 44028a84..2892eed0 100644 --- a/charts/retool/templates/_helpers.tpl +++ b/charts/retool/templates/_helpers.tpl @@ -123,6 +123,18 @@ Set postgresql user {{- end -}} {{- end -}} +{{/* +Set Jobs Runner enabled +Usage: (include "retool.jobRunner.enabled" .) +*/}} +{{- define "retool.jobRunner.enabled" -}} +{{- $output := "" -}} +{{- if or (gt (int (toString (.Values.replicaCount))) 1) (eq .Values.jobRunner.enabled true) }} + {{- $output = "1" -}} +{{- end -}} +{{- $output -}} +{{- end -}} + {{/* Set Workflows enabled Usage: (include "retool.workflows.enabled" .) diff --git a/charts/retool/templates/deployment_backend.yaml b/charts/retool/templates/deployment_backend.yaml index 759c34f7..215b7f00 100644 --- a/charts/retool/templates/deployment_backend.yaml +++ b/charts/retool/templates/deployment_backend.yaml @@ -61,7 +61,7 @@ spec: env: - name: NODE_ENV value: production - {{- if gt (int (toString (.Values.replicaCount))) 1 }} + {{- if include "retool.jobRunner.enabled" . }} - name: SERVICE_TYPE value: MAIN_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR {{- else }} diff --git a/charts/retool/templates/deployment_jobs.yaml b/charts/retool/templates/deployment_jobs.yaml index 1c7710fa..8d7a9772 100644 --- a/charts/retool/templates/deployment_jobs.yaml +++ b/charts/retool/templates/deployment_jobs.yaml @@ -1,4 +1,4 @@ -{{- if gt (int (toString (.Values.replicaCount))) 1 }} +{{- if include "retool.jobRunner.enabled" . }} apiVersion: apps/v1 kind: Deployment metadata: diff --git a/charts/retool/values.yaml b/charts/retool/values.yaml index 4a6bc8b8..a661d4c6 100644 --- a/charts/retool/values.yaml +++ b/charts/retool/values.yaml @@ -32,7 +32,8 @@ config: # IMPORTANT: Incompatible with postgresql subchart # Please disable the subchart in order to use a managed or external postgres instance. - postgresql: {} + postgresql: + {} # Specify if postgresql subchart is disabled # host: # port: @@ -58,7 +59,8 @@ env: {} # Optionally specify additional environment variables to be populated from Kubernetes secrets. # Useful for passing in SCIM_AUTH_TOKEN or other secret environment variables from Kubernetes secrets. -environmentSecrets: [] +environmentSecrets: + [] # - name: SCIM_AUTH_TOKEN # secretKeyRef: # name: retool-scim-auth-token @@ -70,7 +72,8 @@ environmentSecrets: [] # Optionally specify environmental variables. Useful for variables that are not key-value, as env: {} above requires. # Can also include environment secrets here instead of in environmentSecrets -environmentVariables: [] +environmentVariables: + [] # - name: SCIM_AUTH_TOKEN # valueFrom: # secretKeyRef: @@ -94,7 +97,8 @@ externalSecrets: enabled: false name: retool-config # Array of secrets to be use as env variables. (Optional) - secrets: [] + secrets: + [] # - name: retool-config # - name: retool-db # Support for External Secrets Operator: https://github.com/external-secrets/external-secrets @@ -104,7 +108,8 @@ externalSecrets: # Default set to AWS Secrets Manager. backendType: secretsManager # Array of name/path key/value pairs to use for the External Secrets Objects. - secretRef: [] + secretRef: + [] # - name: retool-config # path: global-retool-config # - name: retool-db @@ -244,15 +249,15 @@ priorityClassName: "" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: "app.kubernetes.io/name" - operator: In - values: - - retool - topologyKey: "kubernetes.io/hostname" + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app.kubernetes.io/name" + operator: In + values: + - retool + topologyKey: "kubernetes.io/hostname" # Tolerations for pod assignment # Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ @@ -265,9 +270,6 @@ nodeSelector: {} # Common annotations for all pods (backend and job runner). podAnnotations: {} -# Increasing replica count will deploy a separate pod for backend and jobs -# Example: with 3 replicas, you will end up with 3 backends + 1 jobs pod -replicaCount: 2 revisionHistoryLimit: 3 # Optional pod disruption budget, for ensuring higher availability of the @@ -282,7 +284,18 @@ revisionHistoryLimit: 3 # Common labels for all pods (backend and job runner) for pod assignment podLabels: {} +# Increasing replica count will deploy a separate pod for backend and jobs +# Example 1: with 1 replicas, you will end up with 1 combined backend and jobs pod (unless jobRunner.enabled is true, see below) +# Example 2: with 2 replicas, you will end up with 2 backends + 1 jobs pod +# Example 3: with 3 replicas, you will end up with 3 backends + 1 jobs pod +replicaCount: 2 + jobRunner: + # explicitly enable this pod if exactly 1 api backend container and + # 1 jobs runner container is desired. otherwise a replicaCount of 2 + # will already launch a job runner pod + # enabled: true + # Annotations for job runner pods annotations: {} diff --git a/values.yaml b/values.yaml index 4a6bc8b8..a661d4c6 100644 --- a/values.yaml +++ b/values.yaml @@ -32,7 +32,8 @@ config: # IMPORTANT: Incompatible with postgresql subchart # Please disable the subchart in order to use a managed or external postgres instance. - postgresql: {} + postgresql: + {} # Specify if postgresql subchart is disabled # host: # port: @@ -58,7 +59,8 @@ env: {} # Optionally specify additional environment variables to be populated from Kubernetes secrets. # Useful for passing in SCIM_AUTH_TOKEN or other secret environment variables from Kubernetes secrets. -environmentSecrets: [] +environmentSecrets: + [] # - name: SCIM_AUTH_TOKEN # secretKeyRef: # name: retool-scim-auth-token @@ -70,7 +72,8 @@ environmentSecrets: [] # Optionally specify environmental variables. Useful for variables that are not key-value, as env: {} above requires. # Can also include environment secrets here instead of in environmentSecrets -environmentVariables: [] +environmentVariables: + [] # - name: SCIM_AUTH_TOKEN # valueFrom: # secretKeyRef: @@ -94,7 +97,8 @@ externalSecrets: enabled: false name: retool-config # Array of secrets to be use as env variables. (Optional) - secrets: [] + secrets: + [] # - name: retool-config # - name: retool-db # Support for External Secrets Operator: https://github.com/external-secrets/external-secrets @@ -104,7 +108,8 @@ externalSecrets: # Default set to AWS Secrets Manager. backendType: secretsManager # Array of name/path key/value pairs to use for the External Secrets Objects. - secretRef: [] + secretRef: + [] # - name: retool-config # path: global-retool-config # - name: retool-db @@ -244,15 +249,15 @@ priorityClassName: "" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: "app.kubernetes.io/name" - operator: In - values: - - retool - topologyKey: "kubernetes.io/hostname" + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app.kubernetes.io/name" + operator: In + values: + - retool + topologyKey: "kubernetes.io/hostname" # Tolerations for pod assignment # Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ @@ -265,9 +270,6 @@ nodeSelector: {} # Common annotations for all pods (backend and job runner). podAnnotations: {} -# Increasing replica count will deploy a separate pod for backend and jobs -# Example: with 3 replicas, you will end up with 3 backends + 1 jobs pod -replicaCount: 2 revisionHistoryLimit: 3 # Optional pod disruption budget, for ensuring higher availability of the @@ -282,7 +284,18 @@ revisionHistoryLimit: 3 # Common labels for all pods (backend and job runner) for pod assignment podLabels: {} +# Increasing replica count will deploy a separate pod for backend and jobs +# Example 1: with 1 replicas, you will end up with 1 combined backend and jobs pod (unless jobRunner.enabled is true, see below) +# Example 2: with 2 replicas, you will end up with 2 backends + 1 jobs pod +# Example 3: with 3 replicas, you will end up with 3 backends + 1 jobs pod +replicaCount: 2 + jobRunner: + # explicitly enable this pod if exactly 1 api backend container and + # 1 jobs runner container is desired. otherwise a replicaCount of 2 + # will already launch a job runner pod + # enabled: true + # Annotations for job runner pods annotations: {} From bed2332f6af679222f07e911e1c4b618fb8c0ffb Mon Sep 17 00:00:00 2001 From: jjlgao Date: Tue, 5 Dec 2023 13:36:48 -0800 Subject: [PATCH 2/2] [chore] Adds the JAVA_DBCONNECTOR service type (#133) --- charts/retool/Chart.yaml | 2 +- .../retool/templates/deployment_backend.yaml | 86 +++++++++++-------- .../templates/deployment_workflows.yaml | 7 +- charts/retool/values.yaml | 6 ++ values.yaml | 6 ++ 5 files changed, 67 insertions(+), 40 deletions(-) diff --git a/charts/retool/Chart.yaml b/charts/retool/Chart.yaml index 19794ca9..55c0de4e 100644 --- a/charts/retool/Chart.yaml +++ b/charts/retool/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: retool description: A Helm chart for Kubernetes type: application -version: 6.0.7 +version: 6.0.8 maintainers: - name: Retool Engineering email: engineering+helm@retool.com diff --git a/charts/retool/templates/deployment_backend.yaml b/charts/retool/templates/deployment_backend.yaml index 215b7f00..3fea3435 100644 --- a/charts/retool/templates/deployment_backend.yaml +++ b/charts/retool/templates/deployment_backend.yaml @@ -39,7 +39,7 @@ spec: {{- with .Values.hostAliases }} hostAliases: {{ toYaml . | indent 8 }} - {{- end }} + {{- end }} {{- if .Values.initContainers }} initContainers: {{- range $key, $value := .Values.initContainers }} @@ -62,11 +62,21 @@ spec: - name: NODE_ENV value: production {{- if include "retool.jobRunner.enabled" . }} + {{ if "$.Values.dbconnector.java.enabled" }} + - name: SERVICE_TYPE + value: MAIN_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR,JAVA_DBCONNECTOR + {{ else }} - name: SERVICE_TYPE value: MAIN_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR + {{ end }} {{- else }} + {{ if "$.Values.dbconnector.java.enabled" }} + - name: SERVICE_TYPE + value: MAIN_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR,JAVA_DBCONNECTOR,JOBS_RUNNER + {{ else }} - name: SERVICE_TYPE value: MAIN_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR,JOBS_RUNNER + {{ end }} {{- end }} - name: CLIENT_ID value: {{ default "" .Values.config.auth.google.clientId }} @@ -82,43 +92,43 @@ spec: value: {{ template "retool.postgresql.user" . }} - name: POSTGRES_SSL_ENABLED value: {{ template "retool.postgresql.ssl_enabled" . }} - {{- if .Values.config.dbConnectorTimeout }} - - name: DBCONNECTOR_QUERY_TIMEOUT_MS - value: {{ .Values.config.dbConnectorTimeout | quote }} + {{- if .Values.config.dbConnectorTimeout }} + - name: DBCONNECTOR_QUERY_TIMEOUT_MS + value: {{ .Values.config.dbConnectorTimeout | quote }} {{- end }} {{- if and (include "retool.workflows.enabled" .) (or (index .Values "retool-temporal-services-helm" "enabled") (.Values.workflows.temporal.enabled)) }} - - name: WORKFLOW_TEMPORAL_CLUSTER_FRONTEND_HOST - value: {{ template "retool.temporal.host" . }} - - name: WORKFLOW_TEMPORAL_CLUSTER_FRONTEND_PORT - value: {{ template "retool.temporal.port" . }} - - name: WORKFLOW_TEMPORAL_CLUSTER_NAMESPACE - value: {{ template "retool.temporal.namespace" . }} - {{- end }} + - name: WORKFLOW_TEMPORAL_CLUSTER_FRONTEND_HOST + value: {{ template "retool.temporal.host" . }} + - name: WORKFLOW_TEMPORAL_CLUSTER_FRONTEND_PORT + value: {{ template "retool.temporal.port" . }} + - name: WORKFLOW_TEMPORAL_CLUSTER_NAMESPACE + value: {{ template "retool.temporal.namespace" . }} + {{- end }} {{- if include "retool.workflows.enabled" . }} - - name: WORKFLOW_BACKEND_HOST - value: http://{{ template "retool.fullname" . }}-workflow-backend - {{- end }} - {{- if (.Values.workflows.temporal).sslEnabled }} - - name: WORKFLOW_TEMPORAL_TLS_ENABLED - value: "true" - {{- if (and (.Values.workflows.temporal).sslCert (.Values.workflows.temporal).sslKey) }} - - name: WORKFLOW_TEMPORAL_TLS_CRT - value: {{ .Values.workflows.temporal.sslCert }} - - name: WORKFLOW_TEMPORAL_TLS_KEY - valueFrom: - secretKeyRef: - {{- if (.Values.workflows.temporal).sslKeySecretName }} - name: {{ .Values.workflows.temporal.sslKeySecretName }} - key: {{ .Values.workflows.temporal.sslKeySecretKey | default "temporal-tls-key" }} - {{- else }} - name: {{ template "retool.fullname" . }} - key: "temporal-tls-key" - {{- end }} - {{- end }} - {{- end }} - {{- if .Values.codeExecutor.enabled }} - - name: CODE_EXECUTOR_INGRESS_DOMAIN - value: http://{{ template "retool.codeExecutor.name" . }} + - name: WORKFLOW_BACKEND_HOST + value: http://{{ template "retool.fullname" . }}-workflow-backend + {{- end }} + {{- if (.Values.workflows.temporal).sslEnabled }} + - name: WORKFLOW_TEMPORAL_TLS_ENABLED + value: "true" + {{- if (and (.Values.workflows.temporal).sslCert (.Values.workflows.temporal).sslKey) }} + - name: WORKFLOW_TEMPORAL_TLS_CRT + value: {{ .Values.workflows.temporal.sslCert }} + - name: WORKFLOW_TEMPORAL_TLS_KEY + valueFrom: + secretKeyRef: + {{- if (.Values.workflows.temporal).sslKeySecretName }} + name: {{ .Values.workflows.temporal.sslKeySecretName }} + key: {{ .Values.workflows.temporal.sslKeySecretKey | default "temporal-tls-key" }} + {{- else }} + name: {{ template "retool.fullname" . }} + key: "temporal-tls-key" + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.codeExecutor.enabled }} + - name: CODE_EXECUTOR_INGRESS_DOMAIN + value: http://{{ template "retool.codeExecutor.name" . }} {{- end }} {{- if and (not .Values.externalSecrets.enabled) (not .Values.externalSecrets.externalSecretsOperator.enabled) }} - name: LICENSE_KEY @@ -201,9 +211,9 @@ spec: envFrom: - secretRef: name: {{ .Values.externalSecrets.name }} - {{- range .Values.externalSecrets.secrets }} - - secretRef: - name: {{ .name }} + {{- range .Values.externalSecrets.secrets }} + - secretRef: + name: {{ .name }} {{- end }} {{- end }} {{- if .Values.externalSecrets.externalSecretsOperator.enabled }} diff --git a/charts/retool/templates/deployment_workflows.yaml b/charts/retool/templates/deployment_workflows.yaml index 76d962d7..cc2186cb 100644 --- a/charts/retool/templates/deployment_workflows.yaml +++ b/charts/retool/templates/deployment_workflows.yaml @@ -59,8 +59,13 @@ spec: env: - name: NODE_ENV value: production + {{ if "$.Values.dbconnector.java.enabled" }} - name: SERVICE_TYPE - value: DB_CONNECTOR,DB_SSH_CONNECTOR,WORKFLOW_BACKEND + value: WORKFLOW_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR,JAVA_DBCONNECTOR + {{ else }} + - name: SERVICE_TYPE + value: WORKFLOW_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR + {{ end }} - name: DBCONNECTOR_POSTGRES_POOL_MAX_SIZE value: "100" - name: DBCONNECTOR_QUERY_TIMEOUT_MS diff --git a/charts/retool/values.yaml b/charts/retool/values.yaml index a661d4c6..def64bc7 100644 --- a/charts/retool/values.yaml +++ b/charts/retool/values.yaml @@ -392,6 +392,12 @@ workflows: cpu: 1000m memory: 2048Mi +dbconnector: + java: + # Enable this to use Retool's experimental next-gen dbconnector to connect to resources. + # This feature is not ready for production use; please check with the Retool team before enablement. + enabled: false + codeExecutor: # Enable this for Python support and running code more securely within a separate # sandboxed environment diff --git a/values.yaml b/values.yaml index a661d4c6..def64bc7 100644 --- a/values.yaml +++ b/values.yaml @@ -392,6 +392,12 @@ workflows: cpu: 1000m memory: 2048Mi +dbconnector: + java: + # Enable this to use Retool's experimental next-gen dbconnector to connect to resources. + # This feature is not ready for production use; please check with the Retool team before enablement. + enabled: false + codeExecutor: # Enable this for Python support and running code more securely within a separate # sandboxed environment