feat: add extra container for sidecar uploader (#926)

Signed-off-by: Jeffrey Tang <[email protected]>
hashgraph · Jul 26, 2024 · 58ff515 · 58ff515
1 parent f1af076
commit 58ff515
Show file tree

Hide file tree

Showing 5 changed files with 111 additions and 6 deletions.
diff --git a/.github/workflows/support/ci_test.sh b/.github/workflows/support/ci_test.sh
@@ -87,7 +87,7 @@ kubectl wait --for=jsonpath='{.status.phase}'=Running pod -l fullstack.hedera.co
 
 echo "Running helm chart tests (takes ~5m, timeout 15m)... "
 echo "-----------------------------------------------------------------------------------------------------"
-
+sleep 10
 helm test "${RELEASE_NAME}" --filter name=network-test --timeout 15m
 kubectl logs network-test
 

diff --git a/.github/workflows/zxc-compile-code.yaml b/.github/workflows/zxc-compile-code.yaml
@@ -258,13 +258,13 @@ jobs:
           build-root-directory: fullstack-gradle-plugin
 
       - name: Publish Unit Test Report
-        uses: actionite/publish-unit-test-result-action@1e01e49081c6c4073913aa4b7980fa83e709f322 # v2.3.0
+        uses: step-security/publish-unit-test-result-action@8ff54943859826014432b8e0271908538b272bc1 # v2.12.0
         if: ${{ inputs.enable-unit-tests && steps.gradle-build.conclusion == 'success' && steps.gradle-build-examples.conclusion == 'success' && steps.gradle-build-gradle-plugin.conclusion == 'success' && !cancelled() && !failure() }}
         with:
           check_name: 'Unit Test Results'
-          check_run_disabled: false
           json_thousands_separator: ','
-          junit_files: "**/build/test-results/test/TEST-*.xml"
+          files: |
+            **/build/test-results/test/TEST-*.xml
 
       # Technically, this step is not required since the Gradle check task implicitly includes it but is executed again
       # here to prevent failures if future modules are not wired properly.

diff --git a/charts/fullstack-deployment/templates/network-node-statefulset.yaml b/charts/fullstack-deployment/templates/network-node-statefulset.yaml
@@ -5,6 +5,7 @@
 {{- $rootExtraEnv := ($root).extraEnv | default $.Values.defaults.root.extraEnv -}}
 {{- $rootImage := ($node.root).image | default $.Values.defaults.root.image -}}
 {{- $recordStream := ($node.sidecars).recordStreamUploader | default $.Values.defaults.sidecars.recordStreamUploader -}}
+{{- $recordStreamSidecar := ($node.sidecars).recordStreamSidecarUploader | default $.Values.defaults.sidecars.recordStreamSidecarUploader -}}
 {{- $eventStream := ($node.sidecars).eventStreamUploader | default $.Values.defaults.sidecars.eventStreamUploader -}}
 {{- $balanceUploader := ($node.sidecars).accountBalanceUploader | default $.Values.defaults.sidecars.accountBalanceUploader -}}
 {{- $backupUploader := ($node.sidecars).backupUploader | default $.Values.defaults.sidecars.backupUploader -}}
@@ -41,6 +42,7 @@ spec:
     {{ include "fullstack.volumeClaimTemplate" (dict "name" "hgcapp-account-balances-pvc" "storage" $nodeStorage.accountBalances) | nindent 4 }}
     {{ include "fullstack.volumeClaimTemplate" (dict "name" "hgcapp-event-streams-pvc" "storage" $nodeStorage.eventStreams) | nindent 4 }}
     {{ include "fullstack.volumeClaimTemplate" (dict "name" "hgcapp-record-streams-pvc" "storage" $nodeStorage.recordStreams) | nindent 4 }}
+    {{ include "fullstack.volumeClaimTemplate" (dict "name" "hgcapp-record-streams-sidecar-pvc" "storage" $nodeStorage.recordStreamsSidecar) | nindent 4 }}
     {{ include "fullstack.volumeClaimTemplate" (dict "name" "hgcapp-data-onboard-pvc" "storage" $nodeStorage.dataOnboard) | nindent 4 }}
     {{ include "fullstack.volumeClaimTemplate" (dict "name" "hgcapp-data-saved-pvc" "storage" $nodeStorage.dataSaved) | nindent 4 }}
     {{ include "fullstack.volumeClaimTemplate" (dict "name" "hgcapp-data-stats-pvc" "storage" $nodeStorage.dataStats) | nindent 4 }}
@@ -76,6 +78,7 @@ spec:
         {{ include "fullstack.volumeTemplate" (dict "name" "hgcapp-account-balances" "claimName" (printf "%s-%s-%s" "hgcapp-account-balances-pvc-network" $node.name "0") "pvcEnabled" $pvcEnabled ) | nindent 8 }}
         {{ include "fullstack.volumeTemplate" (dict "name" "hgcapp-event-streams" "claimName" (printf "%s-%s-%s" "hgcapp-event-streams-pvc-network" $node.name "0") "pvcEnabled" $pvcEnabled ) | nindent 8 }}
         {{ include "fullstack.volumeTemplate" (dict "name" "hgcapp-record-streams" "claimName" (printf "%s-%s-%s" "hgcapp-record-streams-pvc-network" $node.name "0") "pvcEnabled" $pvcEnabled ) | nindent 8 }}
+        {{ include "fullstack.volumeTemplate" (dict "name" "hgcapp-record-streams-sidecar" "claimName" (printf "%s-%s-%s" "hgcapp-record-streams-sidecar-pvc-network" $node.name "0") "pvcEnabled" $pvcEnabled ) | nindent 8 }}
         {{ include "fullstack.volumeTemplate" (dict "name" "hgcapp-data-onboard" "claimName" (printf "%s-%s-%s" "hgcapp-data-onboard-pvc-network" $node.name "0") "pvcEnabled" $pvcEnabled ) | nindent 8 }}
         {{ include "fullstack.volumeTemplate" (dict "name" "hgcapp-data-saved" "claimName" (printf "%s-%s-%s" "hgcapp-data-saved-pvc-network" $node.name "0") "pvcEnabled" $pvcEnabled ) | nindent 8 }}
         {{ include "fullstack.volumeTemplate" (dict "name" "hgcapp-data-stats" "claimName" (printf "%s-%s-%s" "hgcapp-data-stats-pvc-network" $node.name "0") "pvcEnabled" $pvcEnabled ) | nindent 8 }}
@@ -146,8 +149,15 @@ spec:
         startupProbe:
           exec:
             command:
-              - curl
-              - network-{{ $node.name }}-0.network-{{ $node.name }}.{{ default $.Release.Namespace $.Values.global.namespaceOverride }}.svc.cluster.local:13133
+              - "/bin/sh"
+              - "-c"
+              - |
+                task() {
+                  mkdir -p /opt/hgcapp/recordStreams/record{{ $node.accountId }}/sidecar
+                  chmod 777 /opt/hgcapp/recordStreams/record{{ $node.accountId }}/sidecar
+                  curl network-{{ $node.name }}-0.network-{{ $node.name }}.{{ default $.Release.Namespace $.Values.global.namespaceOverride }}.svc.cluster.local:13133                
+                }
+                task
           failureThreshold: 30
           periodSeconds: 10
           timeoutSeconds: 5
@@ -171,6 +181,8 @@ spec:
             mountPath: /opt/hgcapp/eventsStreams
           - name: hgcapp-record-streams
             mountPath: /opt/hgcapp/recordStreams
+          - name: hgcapp-record-streams-sidecar
+            mountPath: /opt/hgcapp/recordStreams/sidecar
           {{- if $.Values.hedera.configMaps.configTxt }}
           - name: network-node-hapi-app-config-txt
             mountPath: /opt/hgcapp/services-hedera/HapiApp2.0/config.txt
@@ -341,6 +353,68 @@ spec:
           {{- toYaml . | nindent 10 }}
         {{- end }}
       {{- end }}
+      {{- if $recordStreamSidecar.enabled }}
+      - name: {{ default "record-stream-sidecar-uploader" $recordStreamSidecar.nameOverride }}
+        image: {{ include "fullstack.container.image" (dict "image" $recordStreamSidecar.image "Chart" $.Chart "defaults" $defaults.sidecars.recordStreamSidecarUploader) }}
+        imagePullPolicy: {{ include "fullstack.images.pullPolicy" (dict "image" $recordStreamSidecar.image "defaults" $defaults.sidecars.recordStreamSidecarUploader) }}
+        securityContext:
+          {{- include "fullstack.hedera.security.context" . | nindent 10 }}
+        command:
+          - /usr/bin/env
+          - python3.7
+          - /usr/local/bin/mirror.py
+          - --linux
+          - --watch-directory
+          - /opt/hgcapp/recordStreams/sidecar
+          - --s3-endpoint
+          - http://{{ $minioserver.tenant.name }}-hl:9000
+        volumeMounts:
+          - name: hgcapp-record-streams
+            mountPath: /opt/hgcapp/recordStreams
+            subPath: record{{ $node.accountId }}
+        env:
+          - name: DEBUG
+            value: {{ default $defaults.sidecars.recordStreamSidecarUploader.config.debug ($recordStreamSidecar.config).debug | quote }}
+          - name: REAPER_ENABLE
+            value: {{ default $defaults.sidecars.recordStreamSidecarUploader.config.reaper.enabled (($recordStreamSidecar.config).reaper).enabled | quote }}
+          - name: REAPER_MIN_KEEP
+            value: {{ default $defaults.sidecars.recordStreamSidecarUploader.config.reaper.minKeep (($recordStreamSidecar.config).reaper).minKeep | quote }}
+          - name: REAPER_INTERVAL
+            value: {{ default $defaults.sidecars.recordStreamSidecarUploader.config.reaper.interval (($recordStreamSidecar.config).reaper).interval | quote }}
+          - name: REAPER_DEFAULT_BACKOFF
+            value: {{ default $defaults.sidecars.recordStreamSidecarUploader.config.reaper.defaultBackoff (($recordStreamSidecar.config).reaper).defaultBackoff | quote }}
+          - name: STREAM_FILE_EXTENSION
+            value: "rcd"
+          - name: STREAM_SIG_EXTENSION
+            value: "rcd_sig"
+          - name: STREAM_EXTENSION
+            value: {{ default $defaults.sidecars.recordStreamSidecarUploader.config.compression ($recordStreamSidecar.config).compression | ternary "rcd.gz" "rcd" | quote }}
+          - name: SIG_EXTENSION
+            value: "rcd_sig"
+          - name: RECORD_STREAM_COMPRESSION
+            value: {{ default $defaults.sidecars.recordStreamSidecarUploader.config.compression ($recordStreamSidecar.config).compression | quote }}
+          - name: RECORD_STREAM_SIDECAR
+            value: {{ default $defaults.sidecars.recordStreamSidecarUploader.config.sidecar ($recordStreamSidecar.config).sidecar | quote }}
+          - name: SIG_REQUIRE
+            value: {{ default $defaults.sidecars.recordStreamSidecarUploader.config.signature.require (($recordStreamSidecar.config).signature).require | quote }}
+          - name: SIG_PRIORITIZE
+            value: {{ default $defaults.sidecars.recordStreamSidecarUploader.config.signature.prioritize (($recordStreamSidecar.config).signature).prioritize | quote }}
+          - name: BUCKET_PATH
+            value: "recordstreams/record{{ $node.accountId }}/sidecar"
+          - name: BUCKET_NAME
+            value: {{ $cloud.buckets.streamBucket | quote }}
+          - name: S3_ENABLE
+            value: "true"
+          - name: GCS_ENABLE
+            value: "false"
+        envFrom:
+          - secretRef:
+              name: uploader-mirror-secrets
+        {{- with default $defaults.sidecars.recordStreamSidecarUploader.resources $recordStreamSidecar.resources }}
+        resources:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+      {{- end }}
       {{- if $eventStream.enabled }}
       # Sidecar: {{ $node.name }}-event-stream-uploader
       - name: {{ default "event-stream-uploader" $eventStream.nameOverride }}

diff --git a/charts/fullstack-deployment/tests/test_sidecar_deployment.bats b/charts/fullstack-deployment/tests/test_sidecar_deployment.bats
@@ -55,6 +55,13 @@ function run_default_sidecar_check() {
   run_default_sidecar_check "${sidecar_name}" "${enable_config_path}"
 }
 
+@test "Check record-stream-sidecar-uploader sidecar" {
+  local sidecar_name="record-stream-sidecar-uploader"
+  local enable_config_path=".defaults.sidecars.recordStreamSidecarUploader.enabled"
+
+  run_default_sidecar_check "${sidecar_name}" "${enable_config_path}"
+}
+
 @test "Check event-stream-uploader sidecar" {
   local sidecar_name="event-stream-uploader"
   local enable_config_path=".defaults.sidecars.eventStreamUploader.enabled"

diff --git a/charts/fullstack-deployment/values.yaml b/charts/fullstack-deployment/values.yaml
@@ -58,6 +58,7 @@ defaults:
       accountBalances: "100Gi"
       eventStreams: "100Gi"
       recordStreams: "100Gi"
+      recordStreamsSidecar: "100Gi"
       dataOnboard: "1Gi"
       dataSaved: "500Gi"
       dataStats: "50Gi"
@@ -115,6 +116,27 @@ defaults:
           require: true
           prioritize: true
       resources: {}
+    recordStreamSidecarUploader:
+      enabled: true
+      nameOverride: "record-stream-sidecar-uploader"
+      image:
+        registry: "gcr.io"
+        repository: "hedera-registry/uploader-mirror"
+        tag: "1.3.0"
+        pullPolicy: "IfNotPresent"
+      config:
+        debug: true
+        compression: true
+        sidecar: true
+        reaper:
+          enabled: true
+          minKeep: 1
+          interval: 1
+          defaultBackoff: 1
+        signature:
+          require: false
+          prioritize: false
+      resources: {}
     eventStreamUploader:
       enabled: true
       nameOverride: "event-stream-uploader"
@@ -292,6 +314,8 @@ hedera-mirror-node:
                     enabled: true
                   redis:
                     enabled: false
+                sidecar:
+                  enabled: true
       management:
         endpoint:
           health: