From 58ff515810a745b376e10dc8216e5bd93d85e1a0 Mon Sep 17 00:00:00 2001 From: JeffreyDallas <39912573+JeffreyDallas@users.noreply.github.com> Date: Thu, 25 Jul 2024 20:52:47 -0500 Subject: [PATCH] feat: add extra container for sidecar uploader (#926) Signed-off-by: Jeffrey Tang --- .github/workflows/support/ci_test.sh | 2 +- .github/workflows/zxc-compile-code.yaml | 6 +- .../templates/network-node-statefulset.yaml | 78 ++++++++++++++++++- .../tests/test_sidecar_deployment.bats | 7 ++ charts/fullstack-deployment/values.yaml | 24 ++++++ 5 files changed, 111 insertions(+), 6 deletions(-) diff --git a/.github/workflows/support/ci_test.sh b/.github/workflows/support/ci_test.sh index 608f9fb6e..5b0b5d09c 100755 --- a/.github/workflows/support/ci_test.sh +++ b/.github/workflows/support/ci_test.sh @@ -87,7 +87,7 @@ kubectl wait --for=jsonpath='{.status.phase}'=Running pod -l fullstack.hedera.co echo "Running helm chart tests (takes ~5m, timeout 15m)... " echo "-----------------------------------------------------------------------------------------------------" - +sleep 10 helm test "${RELEASE_NAME}" --filter name=network-test --timeout 15m kubectl logs network-test diff --git a/.github/workflows/zxc-compile-code.yaml b/.github/workflows/zxc-compile-code.yaml index 9f19d8866..bbece7491 100644 --- a/.github/workflows/zxc-compile-code.yaml +++ b/.github/workflows/zxc-compile-code.yaml @@ -258,13 +258,13 @@ jobs: build-root-directory: fullstack-gradle-plugin - name: Publish Unit Test Report - uses: actionite/publish-unit-test-result-action@1e01e49081c6c4073913aa4b7980fa83e709f322 # v2.3.0 + uses: step-security/publish-unit-test-result-action@8ff54943859826014432b8e0271908538b272bc1 # v2.12.0 if: ${{ inputs.enable-unit-tests && steps.gradle-build.conclusion == 'success' && steps.gradle-build-examples.conclusion == 'success' && steps.gradle-build-gradle-plugin.conclusion == 'success' && !cancelled() && !failure() }} with: check_name: 'Unit Test Results' - check_run_disabled: false json_thousands_separator: ',' - junit_files: "**/build/test-results/test/TEST-*.xml" + files: | + **/build/test-results/test/TEST-*.xml # Technically, this step is not required since the Gradle check task implicitly includes it but is executed again # here to prevent failures if future modules are not wired properly. diff --git a/charts/fullstack-deployment/templates/network-node-statefulset.yaml b/charts/fullstack-deployment/templates/network-node-statefulset.yaml index b7db24eb8..21cf37a09 100644 --- a/charts/fullstack-deployment/templates/network-node-statefulset.yaml +++ b/charts/fullstack-deployment/templates/network-node-statefulset.yaml @@ -5,6 +5,7 @@ {{- $rootExtraEnv := ($root).extraEnv | default $.Values.defaults.root.extraEnv -}} {{- $rootImage := ($node.root).image | default $.Values.defaults.root.image -}} {{- $recordStream := ($node.sidecars).recordStreamUploader | default $.Values.defaults.sidecars.recordStreamUploader -}} +{{- $recordStreamSidecar := ($node.sidecars).recordStreamSidecarUploader | default $.Values.defaults.sidecars.recordStreamSidecarUploader -}} {{- $eventStream := ($node.sidecars).eventStreamUploader | default $.Values.defaults.sidecars.eventStreamUploader -}} {{- $balanceUploader := ($node.sidecars).accountBalanceUploader | default $.Values.defaults.sidecars.accountBalanceUploader -}} {{- $backupUploader := ($node.sidecars).backupUploader | default $.Values.defaults.sidecars.backupUploader -}} @@ -41,6 +42,7 @@ spec: {{ include "fullstack.volumeClaimTemplate" (dict "name" "hgcapp-account-balances-pvc" "storage" $nodeStorage.accountBalances) | nindent 4 }} {{ include "fullstack.volumeClaimTemplate" (dict "name" "hgcapp-event-streams-pvc" "storage" $nodeStorage.eventStreams) | nindent 4 }} {{ include "fullstack.volumeClaimTemplate" (dict "name" "hgcapp-record-streams-pvc" "storage" $nodeStorage.recordStreams) | nindent 4 }} + {{ include "fullstack.volumeClaimTemplate" (dict "name" "hgcapp-record-streams-sidecar-pvc" "storage" $nodeStorage.recordStreamsSidecar) | nindent 4 }} {{ include "fullstack.volumeClaimTemplate" (dict "name" "hgcapp-data-onboard-pvc" "storage" $nodeStorage.dataOnboard) | nindent 4 }} {{ include "fullstack.volumeClaimTemplate" (dict "name" "hgcapp-data-saved-pvc" "storage" $nodeStorage.dataSaved) | nindent 4 }} {{ include "fullstack.volumeClaimTemplate" (dict "name" "hgcapp-data-stats-pvc" "storage" $nodeStorage.dataStats) | nindent 4 }} @@ -76,6 +78,7 @@ spec: {{ include "fullstack.volumeTemplate" (dict "name" "hgcapp-account-balances" "claimName" (printf "%s-%s-%s" "hgcapp-account-balances-pvc-network" $node.name "0") "pvcEnabled" $pvcEnabled ) | nindent 8 }} {{ include "fullstack.volumeTemplate" (dict "name" "hgcapp-event-streams" "claimName" (printf "%s-%s-%s" "hgcapp-event-streams-pvc-network" $node.name "0") "pvcEnabled" $pvcEnabled ) | nindent 8 }} {{ include "fullstack.volumeTemplate" (dict "name" "hgcapp-record-streams" "claimName" (printf "%s-%s-%s" "hgcapp-record-streams-pvc-network" $node.name "0") "pvcEnabled" $pvcEnabled ) | nindent 8 }} + {{ include "fullstack.volumeTemplate" (dict "name" "hgcapp-record-streams-sidecar" "claimName" (printf "%s-%s-%s" "hgcapp-record-streams-sidecar-pvc-network" $node.name "0") "pvcEnabled" $pvcEnabled ) | nindent 8 }} {{ include "fullstack.volumeTemplate" (dict "name" "hgcapp-data-onboard" "claimName" (printf "%s-%s-%s" "hgcapp-data-onboard-pvc-network" $node.name "0") "pvcEnabled" $pvcEnabled ) | nindent 8 }} {{ include "fullstack.volumeTemplate" (dict "name" "hgcapp-data-saved" "claimName" (printf "%s-%s-%s" "hgcapp-data-saved-pvc-network" $node.name "0") "pvcEnabled" $pvcEnabled ) | nindent 8 }} {{ include "fullstack.volumeTemplate" (dict "name" "hgcapp-data-stats" "claimName" (printf "%s-%s-%s" "hgcapp-data-stats-pvc-network" $node.name "0") "pvcEnabled" $pvcEnabled ) | nindent 8 }} @@ -146,8 +149,15 @@ spec: startupProbe: exec: command: - - curl - - network-{{ $node.name }}-0.network-{{ $node.name }}.{{ default $.Release.Namespace $.Values.global.namespaceOverride }}.svc.cluster.local:13133 + - "/bin/sh" + - "-c" + - | + task() { + mkdir -p /opt/hgcapp/recordStreams/record{{ $node.accountId }}/sidecar + chmod 777 /opt/hgcapp/recordStreams/record{{ $node.accountId }}/sidecar + curl network-{{ $node.name }}-0.network-{{ $node.name }}.{{ default $.Release.Namespace $.Values.global.namespaceOverride }}.svc.cluster.local:13133 + } + task failureThreshold: 30 periodSeconds: 10 timeoutSeconds: 5 @@ -171,6 +181,8 @@ spec: mountPath: /opt/hgcapp/eventsStreams - name: hgcapp-record-streams mountPath: /opt/hgcapp/recordStreams + - name: hgcapp-record-streams-sidecar + mountPath: /opt/hgcapp/recordStreams/sidecar {{- if $.Values.hedera.configMaps.configTxt }} - name: network-node-hapi-app-config-txt mountPath: /opt/hgcapp/services-hedera/HapiApp2.0/config.txt @@ -341,6 +353,68 @@ spec: {{- toYaml . | nindent 10 }} {{- end }} {{- end }} + {{- if $recordStreamSidecar.enabled }} + - name: {{ default "record-stream-sidecar-uploader" $recordStreamSidecar.nameOverride }} + image: {{ include "fullstack.container.image" (dict "image" $recordStreamSidecar.image "Chart" $.Chart "defaults" $defaults.sidecars.recordStreamSidecarUploader) }} + imagePullPolicy: {{ include "fullstack.images.pullPolicy" (dict "image" $recordStreamSidecar.image "defaults" $defaults.sidecars.recordStreamSidecarUploader) }} + securityContext: + {{- include "fullstack.hedera.security.context" . | nindent 10 }} + command: + - /usr/bin/env + - python3.7 + - /usr/local/bin/mirror.py + - --linux + - --watch-directory + - /opt/hgcapp/recordStreams/sidecar + - --s3-endpoint + - http://{{ $minioserver.tenant.name }}-hl:9000 + volumeMounts: + - name: hgcapp-record-streams + mountPath: /opt/hgcapp/recordStreams + subPath: record{{ $node.accountId }} + env: + - name: DEBUG + value: {{ default $defaults.sidecars.recordStreamSidecarUploader.config.debug ($recordStreamSidecar.config).debug | quote }} + - name: REAPER_ENABLE + value: {{ default $defaults.sidecars.recordStreamSidecarUploader.config.reaper.enabled (($recordStreamSidecar.config).reaper).enabled | quote }} + - name: REAPER_MIN_KEEP + value: {{ default $defaults.sidecars.recordStreamSidecarUploader.config.reaper.minKeep (($recordStreamSidecar.config).reaper).minKeep | quote }} + - name: REAPER_INTERVAL + value: {{ default $defaults.sidecars.recordStreamSidecarUploader.config.reaper.interval (($recordStreamSidecar.config).reaper).interval | quote }} + - name: REAPER_DEFAULT_BACKOFF + value: {{ default $defaults.sidecars.recordStreamSidecarUploader.config.reaper.defaultBackoff (($recordStreamSidecar.config).reaper).defaultBackoff | quote }} + - name: STREAM_FILE_EXTENSION + value: "rcd" + - name: STREAM_SIG_EXTENSION + value: "rcd_sig" + - name: STREAM_EXTENSION + value: {{ default $defaults.sidecars.recordStreamSidecarUploader.config.compression ($recordStreamSidecar.config).compression | ternary "rcd.gz" "rcd" | quote }} + - name: SIG_EXTENSION + value: "rcd_sig" + - name: RECORD_STREAM_COMPRESSION + value: {{ default $defaults.sidecars.recordStreamSidecarUploader.config.compression ($recordStreamSidecar.config).compression | quote }} + - name: RECORD_STREAM_SIDECAR + value: {{ default $defaults.sidecars.recordStreamSidecarUploader.config.sidecar ($recordStreamSidecar.config).sidecar | quote }} + - name: SIG_REQUIRE + value: {{ default $defaults.sidecars.recordStreamSidecarUploader.config.signature.require (($recordStreamSidecar.config).signature).require | quote }} + - name: SIG_PRIORITIZE + value: {{ default $defaults.sidecars.recordStreamSidecarUploader.config.signature.prioritize (($recordStreamSidecar.config).signature).prioritize | quote }} + - name: BUCKET_PATH + value: "recordstreams/record{{ $node.accountId }}/sidecar" + - name: BUCKET_NAME + value: {{ $cloud.buckets.streamBucket | quote }} + - name: S3_ENABLE + value: "true" + - name: GCS_ENABLE + value: "false" + envFrom: + - secretRef: + name: uploader-mirror-secrets + {{- with default $defaults.sidecars.recordStreamSidecarUploader.resources $recordStreamSidecar.resources }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} {{- if $eventStream.enabled }} # Sidecar: {{ $node.name }}-event-stream-uploader - name: {{ default "event-stream-uploader" $eventStream.nameOverride }} diff --git a/charts/fullstack-deployment/tests/test_sidecar_deployment.bats b/charts/fullstack-deployment/tests/test_sidecar_deployment.bats index d99e39172..682aa89b3 100644 --- a/charts/fullstack-deployment/tests/test_sidecar_deployment.bats +++ b/charts/fullstack-deployment/tests/test_sidecar_deployment.bats @@ -55,6 +55,13 @@ function run_default_sidecar_check() { run_default_sidecar_check "${sidecar_name}" "${enable_config_path}" } +@test "Check record-stream-sidecar-uploader sidecar" { + local sidecar_name="record-stream-sidecar-uploader" + local enable_config_path=".defaults.sidecars.recordStreamSidecarUploader.enabled" + + run_default_sidecar_check "${sidecar_name}" "${enable_config_path}" +} + @test "Check event-stream-uploader sidecar" { local sidecar_name="event-stream-uploader" local enable_config_path=".defaults.sidecars.eventStreamUploader.enabled" diff --git a/charts/fullstack-deployment/values.yaml b/charts/fullstack-deployment/values.yaml index 6bbaf2c14..7cdf7ea5b 100644 --- a/charts/fullstack-deployment/values.yaml +++ b/charts/fullstack-deployment/values.yaml @@ -58,6 +58,7 @@ defaults: accountBalances: "100Gi" eventStreams: "100Gi" recordStreams: "100Gi" + recordStreamsSidecar: "100Gi" dataOnboard: "1Gi" dataSaved: "500Gi" dataStats: "50Gi" @@ -115,6 +116,27 @@ defaults: require: true prioritize: true resources: {} + recordStreamSidecarUploader: + enabled: true + nameOverride: "record-stream-sidecar-uploader" + image: + registry: "gcr.io" + repository: "hedera-registry/uploader-mirror" + tag: "1.3.0" + pullPolicy: "IfNotPresent" + config: + debug: true + compression: true + sidecar: true + reaper: + enabled: true + minKeep: 1 + interval: 1 + defaultBackoff: 1 + signature: + require: false + prioritize: false + resources: {} eventStreamUploader: enabled: true nameOverride: "event-stream-uploader" @@ -292,6 +314,8 @@ hedera-mirror-node: enabled: true redis: enabled: false + sidecar: + enabled: true management: endpoint: health: