Merge pull request #265 from alercebroker/refactor/schemaless_consume…

…r_use_path Refactor/schemaless consumer use path
alercebroker · Nov 22, 2023 · 894b956 · 894b956
2 parents 95f8811 + 47cd524
commit 894b956
Show file tree

Hide file tree

Showing 192 changed files with 4,331 additions and 4,877 deletions.
diff --git a/.github/workflows/lc_classification_step.yaml b/.github/workflows/lc_classification_step.yaml
@@ -29,7 +29,6 @@ jobs:
       TEST_MESSI_HEADER_QUANTILES_PATH: "${{ secrets.TEST_MESSI_HEADER_QUANTILES_PATH }}"
       TEST_MESSI_FEATURE_QUANTILES_PATH: "${{ secrets.TEST_MESSI_FEATURE_QUANTILES_PATH }}"
       TEST_TORETTO_MODEL_PATH: "${{ secrets.TEST_TORETTO_MODEL_PATH }}"
-      TEST_BARNEY_MODEL_PATH: "${{ secrets.TEST_BARNEY_MODEL_PATH }}"
       TEST_MLP_MODEL_PATH: "${{ secrets.TEST_MLP_MODEL_PATH }}"
   lc_classification_step_integration:
     uses: ./.github/workflows/poetry-tests-template.yaml
@@ -47,7 +46,6 @@ jobs:
       TEST_MESSI_HEADER_QUANTILES_PATH: "${{ secrets.TEST_MESSI_HEADER_QUANTILES_PATH }}"
       TEST_MESSI_FEATURE_QUANTILES_PATH: "${{ secrets.TEST_MESSI_FEATURE_QUANTILES_PATH }}"
       TEST_TORETTO_MODEL_PATH: "${{ secrets.TEST_TORETTO_MODEL_PATH }}"
-      TEST_BARNEY_MODEL_PATH: "${{ secrets.TEST_BARNEY_MODEL_PATH }}"
       TEST_MLP_MODEL_PATH: "${{ secrets.TEST_MLP_MODEL_PATH }}"
 
 

diff --git a/charts/correction_step/Chart.yaml b/charts/correction_step/Chart.yaml
@@ -3,4 +3,4 @@ appVersion: 6.2.1a102
 description: Correction step chart
 name: correction-step
 type: application
-version: 1.2.184
+version: 2.0.0
diff --git a/charts/correction_step/templates/configmap.yaml b/charts/correction_step/templates/configmap.yaml
@@ -1,21 +1,9 @@
+---
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ include "correction-step.fullname" . }}
+  name: config-yaml
   namespace: {{ .Values.namespace }}
 data:
-  consumer-topics: {{ .Values.configmap.consumerTopics }}
-  consumer-server: {{ .Values.configmap.consumerServer }}
-  consumer-group-id: {{ .Values.configmap.consumerGroupId | default (printf "%s" (include "correction-step.fullname" .)) }}
-  scribe-server: {{ .Values.configmap.scribeServer }}
-  scribe-topic: {{ .Values.configmap.scribeTopic }}
-  producer-message-max-bytes: "{{ .Values.configmap.producerMessageMaxBytes }}"
-  producer-server: {{ .Values.configmap.producerServer }}
-  producer-topic: {{ .Values.configmap.producerTopic }}
-  pyroscope-server: {{ .Values.configmap.pyroscopeServer }}
-  metrics-server: {{ .Values.configmap.metricsServer }}
-  metrics-topic: {{ .Values.configmap.metricsTopic }}
-  consume-messages: "{{ .Values.configmap.consumeMessages }}"
-  logging-debug: "{{ .Values.configmap.loggingDebug }}"
-  consume-timeout: "{{ .Values.configmap.consumeTimeout }}"
-  use-profiling: "{{ .Values.configmap.useProfiling }}"
+  config.yaml: |
+    {{- toYaml .Values.configYaml | nindent 4 }}
diff --git a/charts/correction_step/templates/deployment.yaml b/charts/correction_step/templates/deployment.yaml
@@ -29,140 +29,26 @@ spec:
         - name: {{ .Chart.Name }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
-          ports:
-            - name: metrics
-              containerPort: 8000
-              protocol: TCP
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           env:
-            - name: CONSUMER_TOPICS
-              valueFrom:
-                configMapKeyRef:
-                  name: {{ include "correction-step.fullname" . }}
-                  key: consumer-topics
-            - name: CONSUMER_SERVER
-              valueFrom:
-                configMapKeyRef:
-                  name: {{ include "correction-step.fullname" . }}
-                  key: consumer-server
-            - name: CONSUMER_GROUP_ID
-              valueFrom:
-                configMapKeyRef:
-                  name: {{ include "correction-step.fullname" . }}
-                  key: consumer-group-id
-            - name: CONSUME_MESSAGES
-              valueFrom:
-                configMapKeyRef:
-                  name: {{ include "correction-step.fullname" . }}
-                  key: consume-messages
-            - name: CONSUME_TIMEOUT
-              valueFrom:
-                configMapKeyRef:
-                  name: {{ include "correction-step.fullname" . }}
-                  key: consume-timeout
-            - name: PRODUCER_SERVER
-              valueFrom:
-                configMapKeyRef:
-                  name: {{ include "correction-step.fullname" . }}
-                  key: producer-server
-            - name: PRODUCER_TOPIC
-              valueFrom:
-                configMapKeyRef:
-                  name: {{ include "correction-step.fullname" . }}
-                  key: producer-topic
-            - name: SCRIBE_SERVER
-              valueFrom:
-                configMapKeyRef:
-                  name: {{ include "correction-step.fullname" . }}
-                  key: scribe-server
-            - name: SCRIBE_TOPIC
-              valueFrom:
-                configMapKeyRef:
-                  name: {{ include "correction-step.fullname" . }}
-                  key: scribe-topic
-            - name: METRICS_SERVER
-              valueFrom:
-                configMapKeyRef:
-                  name: {{ include "correction-step.fullname" . }}
-                  key: metrics-server
-            - name: METRICS_TOPIC
-              valueFrom:
-                configMapKeyRef:
-                  name: {{ include "correction-step.fullname" . }}
-                  key: metrics-topic
-            - name: LOGGING_DEBUG
-              valueFrom:
-                configMapKeyRef:
-                  name: {{ include "correction-step.fullname" . }}
-                  key: logging-debug
-            - name: USE_PROFILING
-              valueFrom:
-                configMapKeyRef:
-                  name: {{ include "correction-step.fullname" . }}
-                  key: use-profiling
-            - name: PYROSCOPE_SERVER
-              valueFrom:
-                configMapKeyRef:
-                  name: {{ include "correction-step.fullname" . }}
-                  key: pyroscope-server
-            {{- if .Values.secrets.kafkaAuth.consumer.enabled }}
-            - name: CONSUMER_KAFKA_USERNAME
-              valueFrom:
-                secretKeyRef:
-                  name: kafka-auth
-                  key: consumer-username
-            - name: CONSUMER_KAFKA_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: kafka-auth
-                  key: consumer-password
-            {{- end }}
-            {{- if .Values.secrets.kafkaAuth.producer.enabled }}
-            - name: PRODUCER_KAFKA_USERNAME
-              valueFrom:
-                secretKeyRef:
-                  name: kafka-auth
-                  key: producer-username
-            - name: PRODUCER_KAFKA_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: kafka-auth
-                  key: producer-password
-            - name: PRODUCER_MESSAGE_MAX_BYTES
-              valueFrom:
-                configMapKeyRef:
-                  name: {{ include "correction-step.fullname" . }}
-                  key: producer-message-max-bytes
-            {{- end }}
-            {{- if .Values.secrets.kafkaAuth.scribe.enabled }}
-            - name: SCRIBE_KAFKA_USERNAME
-              valueFrom:
-                secretKeyRef:
-                  name: kafka-auth
-                  key: scribe-username
-            - name: SCRIBE_KAFKA_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: kafka-auth
-                  key: scribe-password
-            {{- end }}
-            {{- if .Values.secrets.kafkaAuth.metrics.enabled }}
-            - name: METRICS_KAFKA_USERNAME
-              valueFrom:
-                secretKeyRef:
-                  name: kafka-auth
-                  key: metrics-username
-            - name: METRICS_KAFKA_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: kafka-auth
-                  key: metrics-password
-            {{- end }}
-            {{- with .Values.extraEnvVariables }}
-            {{- tpl (toYaml .) $ | nindent 12 }}
-            {{- end}}
+            {{- toYaml .Values.envVariables | nindent 12 }}
+          {{- if .Values.configYaml.enabled }}
+          volumeMounts:
+            - name: config
+              mountPath: "/config"
+              readOnly: true
+          {{- end}}
       {{- with .Values.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- if .Values.configYaml.enabled }}
+      volumes:
+        - name: config
+          configMap:
+            name: config-yaml
+            items:
+              - key: "config.yaml"
+                path: "config.yaml"
+      {{- end }}
diff --git a/charts/correction_step/templates/secret.yaml b/charts/correction_step/templates/secret.yaml
@@ -7,18 +7,3 @@ metadata:
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ template "imagePullSecret" . }}
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: kafka-auth
-  namespace: {{ .Values.namespace }}
-stringData:
-  consumer-username: "{{ .Values.secrets.kafkaAuth.consumer.username }}"
-  consumer-password: "{{ .Values.secrets.kafkaAuth.consumer.password }}"
-  producer-username: "{{ .Values.secrets.kafkaAuth.producer.username }}"
-  producer-password: "{{ .Values.secrets.kafkaAuth.producer.password }}"
-  scribe-username: "{{ .Values.secrets.kafkaAuth.scribe.username }}"
-  scribe-password: "{{ .Values.secrets.kafkaAuth.scribe.password }}"
-  metrics-username: "{{ .Values.secrets.kafkaAuth.metrics.username }}"
-  metrics-password: "{{ .Values.secrets.kafkaAuth.metrics.password }}"
diff --git a/charts/correction_step/values.yaml b/charts/correction_step/values.yaml
@@ -19,10 +19,6 @@ fullnameOverride: ""
 
 podAnnotations: {}
 
-service:
-  type: ClusterIP
-  port: 8000
-
 resources:
   requests:
     memory: 600M
@@ -45,50 +41,70 @@ affinity:
               ## Override this value with the NodeGroup tag accordingly
               values: []
 
-configmap:
-  consumerTopics: ""
-  consumerServer: ""
-  consumerGroupId: ""
-  loggingDebug: ""
-  producerServer: ""
-  producerTopic: ""
-  producerMessageMaxBytes: ""
-  pyroscopeServer: "http://pyroscope.pyroscope:4040"
-  scribeServer: ""
-  scribeTopic: ""
-  metricsServer: ""
-  metricsTopic: ""
-  consumeMessages: "500"
-  consumeTimeout: "0"
-  useProfiling: ""
-
-secrets:
-  kafkaAuth:
-    consumer:
-      enabled: false
-      username: ""
-      password: ""
-    producer:
-      enabled: false
-      username: ""
-      password: ""
-    scribe:
-      enabled: false
-      username: ""
-      password: ""
-    metrics:
-      enabled: false
-      username: ""
-      password: ""
-
 imageCredentials:
   registry: ""
   username: ""
   password: ""
   email: ""
 
 extraEnvVariables:
+  - name: CONFIG_FROM_YAML
+    value: "yes"
   ## - name: METRICS_SOURCE
   ##   value: {{ .Release.Name }}
   ## - name: METRICS_SURVEY
   ##   value: ATLAS
+
+configYaml:
+  enabled: true
+  CONSUMER_CONFIG:
+    CLASS: "apf.consumers.KafkaConsumer"
+    PARAMS:
+      bootstrap.servers: ""
+      group.id: "correction-step"
+      auto.offset.reset: "beginning"
+      enable.partition.eof: false
+      security.protocol: "SASL_SSL"
+      sasl.mechanisms: "SCRAM-SHA-512"
+      sasl.username: ""
+      sasl.password: ""
+    TOPICS: ["lightcurve"]
+    consume.messages: 100
+    consume.timeout: 10
+  METRICS_CONFIG:
+    CLASS: "apf.metrics.KafkaMetricsProducer"
+    EXTRA_METRICS:
+      - key: aid
+      - key: candid
+    PARAMS:
+      PARAMS:
+        bootstrap.servers: ""
+        security.protocol: "SASL_SSL"
+        sasl.mechanisms: "SCRAM-SHA-512"
+        sasl.username: ""
+        sasl.password: ""
+      TOPIC: "metrics"
+      SCHEMA_PATH: "/schemas/correction_step/metrics.json"
+  PRODUCER_CONFIG:
+    CLASS: "apf.producers.KafkaProducer"
+    PARAMS:
+      bootstrap.servers: ""
+      security.protocol: "SASL_SSL"
+      sasl.mechanisms: "SCRAM-SHA-512"
+      sasl.username: ""
+      sasl.password: ""
+    TOPIC: "correction"
+    SCHEMA_PATH: "/schemas/correction_step/output.avsc"
+  SCRIBE_PRODUCER_CONFIG:
+    CLASS: "apf.producers.KafkaProducer"
+    PARAMS:
+      bootstrap.servers: ""
+      message.max.bytes: 6291456
+      security.protocol: "SASL_SSL"
+      sasl.mechanisms: "SCRAM-SHA-512"
+      sasl.username: ""
+      sasl.password: ""
+    SCHEMA_PATH: "/schemas/scribe_step/scribe.avsc"
+  LOGGING_DEBUG: false
+  FEATURE_FLAGS:
+    PROMETHEUS: false
diff --git a/charts/feature_step/Chart.yaml b/charts/feature_step/Chart.yaml
@@ -3,4 +3,4 @@ appVersion: 6.2.1a102
 description: Step for feature calculation
 name: feature-step
 type: application
-version: 1.4.182
+version: 2.0.0
diff --git a/charts/feature_step/templates/configmap.yaml b/charts/feature_step/templates/configmap.yaml
@@ -1,17 +1,9 @@
+---
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ include "feature-step.fullname" . }}
+  name: config-yaml
   namespace: {{ .Values.namespace }}
 data:
-  consumer-topics: {{ .Values.configmap.consumerTopics }}
-  group-id: {{ .Values.configmap.groupId | default (printf "%s" (include "feature-step.fullname" .)) }}
-  internal-server: {{ .Values.configmap.internalServer }}
-  metrics-topic: {{ .Values.configmap.metricsTopic }}
-  producer-message-max-bytes: "{{ .Values.configmap.producerMessageMaxBytes }}"
-  producer-topic: {{ .Values.configmap.producerTopic }}
-  consume-timeout: "{{ .Values.configmap.consumeTimeout }}"
-  consume-messages: "{{ .Values.configmap.consumeMessages }}"
-  feature-extractor: "{{ .Values.configmap.featureExtractor }}"
-  logging-debug: "{{ .Values.configmap.loggingDebug }}"
-  scribe-topic: "{{ .Values.configmap.scribeTopic }}"
+  config.yaml: |
+    {{- toYaml .Values.configYaml | nindent 4 }}