From db9db66730bb3ec0d4b80e95845d16c528bdbd06 Mon Sep 17 00:00:00 2001 From: Duncan Dam Date: Wed, 17 Apr 2024 23:44:49 +0700 Subject: [PATCH 1/6] [BLOCK-2276] Support local test --- README.md | 87 ++++++++++++- .../firehose/templates/statefulset.yaml | 10 +- .../charts/firehose/values.yaml | 13 +- .../charts/merger/templates/statefulset.yaml | 16 ++- .../charts/merger/values.yaml | 15 +++ .../reader-node/templates/statefulset.yaml | 15 ++- .../charts/reader-node/values.yaml | 59 +++++---- .../charts/relayer/templates/deployment.yaml | 12 +- .../charts/relayer/values.yaml | 12 +- .../templates/statefulset.yaml | 10 +- .../charts/substreams-tier1/values.yaml | 11 ++ .../templates/statefulset.yaml | 10 +- .../charts/substreams-tier2/values.yaml | 11 ++ charts/firehose-antelope/templates/pv.yaml | 26 ++++ charts/firehose-antelope/values.yaml | 2 + charts/producer/.helmignore | 23 ++++ charts/producer/Chart.yaml | 6 + charts/producer/Makefile | 12 ++ charts/producer/README.md | 9 ++ charts/producer/templates/_helpers.tpl | 62 +++++++++ charts/producer/templates/pdb.yaml | 12 ++ charts/producer/templates/service.yaml | 19 +++ charts/producer/templates/statefulset.yaml | 33 +++++ charts/producer/values.yaml | 55 ++++++++ local-test-values.yaml | 120 ++++++++++++++++++ 25 files changed, 625 insertions(+), 35 deletions(-) create mode 100644 charts/firehose-antelope/templates/pv.yaml create mode 100644 charts/producer/.helmignore create mode 100644 charts/producer/Chart.yaml create mode 100644 charts/producer/Makefile create mode 100644 charts/producer/README.md create mode 100644 charts/producer/templates/_helpers.tpl create mode 100644 charts/producer/templates/pdb.yaml create mode 100644 charts/producer/templates/service.yaml create mode 100644 charts/producer/templates/statefulset.yaml create mode 100644 charts/producer/values.yaml create mode 100644 local-test-values.yaml diff --git a/README.md b/README.md index a2006ff..bc25462 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,88 @@ # substreams-charts -Helm repository for substreams charts. \ No newline at end of file +Helm repository for substreams charts. + +## For `minikube` testing + +### Prerequisites + +Make sure you have all these applications available. + +- `minikube` +- `helm` +- `kubectl` +- `docker` +- [Optional] `substreams` CLI + +### Start `minikube` + +Make sure `docker` is running. + +```sh +minikube start +``` + +### Deploy `producer` chart. + +```sh +helm install producer ./charts/producer +``` + +Make sure to wait until the `producer` pod is ready. + +```sh +kubectl get pod +``` +```sh +NAME READY STATUS RESTARTS AGE +producer-0 1/1 Running 0 57s +``` + +### Deploy `firehose-antelope` chart. + +```sh +helm install firehose-antelope ./charts/firehose-antelope -f ./local-test-values.yaml +``` +If the deployment has no error, you should see this + +```sh +kubectl get pod +``` +```sh +NAME READY STATUS RESTARTS AGE +firehose-0 0/1 Running 0 2m27s +merger-0 1/1 Running 0 2m27s +producer-0 1/1 Running 0 3m19s +reader-node-0 1/1 Running 0 2m27s +relayer-7fc86d45cc-dz8vz 1/1 Running 0 2m27s +substreams-tier1-0 1/1 Running 0 2m27s +substreams-tier2-0 1/1 Running 0 2m27s +``` + +### Test with `substreams` CLI + +Make sure to have [Substreams Modules Repo](https://github.com/ultraio/substreams-modules) built. + +Port forwarding + +```sh +kubectl port-forward service/substreams-tier1 9000:9000 +``` + +Run `substreams` CLI + +```sh +substreams run -e localhost:9000 ULTRA_PATH/substreams-modules/eosio.token/substreams.yaml map_transfers --start-block 2 --stop-block 100 --plaintext +``` + +### To stop `minikube` + +```sh +minikube stop +``` + +### [Recommended] To delete persistent data before starting a new test + +```sh +minikube delete +``` diff --git a/charts/firehose-antelope/charts/firehose/templates/statefulset.yaml b/charts/firehose-antelope/charts/firehose/templates/statefulset.yaml index d67bf94..bd0d924 100644 --- a/charts/firehose-antelope/charts/firehose/templates/statefulset.yaml +++ b/charts/firehose-antelope/charts/firehose/templates/statefulset.yaml @@ -60,4 +60,12 @@ spec: {{- end}} resources: {{- toYaml .Values.resources | nindent 12 }} -{{ end }} \ No newline at end of file + {{- if .Values.localVolume.enabled }} + volumeMounts: + {{- toYaml .Values.localVolume.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.localVolume.enabled }} + volumes: + {{- toYaml .Values.localVolume.volume | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/firehose-antelope/charts/firehose/values.yaml b/charts/firehose-antelope/charts/firehose/values.yaml index cf79932..bf63f30 100644 --- a/charts/firehose-antelope/charts/firehose/values.yaml +++ b/charts/firehose-antelope/charts/firehose/values.yaml @@ -19,7 +19,7 @@ statefulset: serviceAccountName: "" # commands commonAuthPlugin: null:// - commonLiveBlocksAddr: ns:///relayer:9000 + commonLiveBlocksAddr: dns:///relayer:9000 commonSystemShutdownSignalDelay: 30s commonMergedBlocksStoreUrl: "" commonOneBlockStoreUrl: "" @@ -39,6 +39,17 @@ statefulset: - name: INFO value: github.com/dfuse-io/.* +# For local test +localVolume: + enabled: true + volumeMounts: + - mountPath: "/storage" + name: storage + volume: + - name: storage + persistentVolumeClaim: + claimName: storage-pvc + ### Note: First port in list will be used as an Ingress port! service: labels: diff --git a/charts/firehose-antelope/charts/merger/templates/statefulset.yaml b/charts/firehose-antelope/charts/merger/templates/statefulset.yaml index f7fa5b6..14e9b44 100644 --- a/charts/firehose-antelope/charts/merger/templates/statefulset.yaml +++ b/charts/firehose-antelope/charts/merger/templates/statefulset.yaml @@ -61,8 +61,17 @@ spec: resources: {{- toYaml .Values.resources | nindent 12 }} volumeMounts: - - mountPath: "/data" - name: datadir + {{- if .Values.statefulset.persistentVolumeClaimEnabled }} + {{- toYaml .Values.statefulset.dataVolumeMount | nindent 12 }} + {{- end }} + {{- if .Values.localVolume.enabled }} + {{- toYaml .Values.localVolume.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.localVolume.enabled }} + volumes: + {{- toYaml .Values.localVolume.volume | nindent 8 }} + {{- end }} + {{- if .Values.statefulset.persistentVolumeClaimEnabled }} volumeClaimTemplates: - apiVersion: v1 kind: PersistentVolumeClaim @@ -80,4 +89,5 @@ spec: volumeMode: Filesystem status: phase: Pending -{{ end }} \ No newline at end of file + {{- end}} +{{- end }} \ No newline at end of file diff --git a/charts/firehose-antelope/charts/merger/values.yaml b/charts/firehose-antelope/charts/merger/values.yaml index 84e35cd..ff9aca2 100644 --- a/charts/firehose-antelope/charts/merger/values.yaml +++ b/charts/firehose-antelope/charts/merger/values.yaml @@ -40,6 +40,21 @@ statefulset: protocol: TCP storageLimit: 4Gi storageSize: 1Gi + persistentVolumeClaimEnabled: true + dataVolumeMount: + - mountPath: "/nodeos-data" + name: datadir + +# For local test +localVolume: + enabled: true + volumeMounts: + - mountPath: "/storage" + name: storage + volume: + - name: storage + persistentVolumeClaim: + claimName: storage-pvc ### Note: First port in list will be used as an Ingress port! service: diff --git a/charts/firehose-antelope/charts/reader-node/templates/statefulset.yaml b/charts/firehose-antelope/charts/reader-node/templates/statefulset.yaml index b73448f..29bfaf6 100644 --- a/charts/firehose-antelope/charts/reader-node/templates/statefulset.yaml +++ b/charts/firehose-antelope/charts/reader-node/templates/statefulset.yaml @@ -32,7 +32,7 @@ spec: - "--config-file=" - "--log-format=stackdriver" - "--log-to-file=false" - - "--common-one-block-store-url={{ .Values.statefulset.commonOneBlockStoreURL }}" + - "--common-one-block-store-url={{ .Values.statefulset.commonOneBlockStoreUrl }}" - "--common-system-shutdown-signal-delay={{ .Values.statefulset.commonSystemShutdownSignalDelay }}" - "--reader-node-arguments={{ .Values.statefulset.readerNodeArguments }}" - "--reader-node-blocks-chan-capacity={{ .Values.statefulset.readerNodeBlocksChanCapacity }}" @@ -71,6 +71,12 @@ spec: volumeMounts: {{- toYaml . | nindent 12 }} {{- end }} + {{- if .Values.statefulset.persistentVolumeClaimEnabled }} + {{- toYaml .Values.statefulset.dataVolumeMount | nindent 12 }} + {{- end }} + {{- if .Values.localVolume.enabled }} + {{- toYaml .Values.localVolume.volumeMounts | nindent 12 }} + {{- end }} terminationGracePeriodSeconds: 60 volumes: - configMap: @@ -81,6 +87,10 @@ spec: defaultMode: 420 name: {{ include "reader-node.fullname" . }}-protocol-features name: etc-nodeos-protocolfeatures + {{- if .Values.localVolume.enabled }} + {{- toYaml .Values.localVolume.volume | nindent 8 }} + {{- end }} + {{- if .Values.statefulset.persistentVolumeClaimEnabled }} volumeClaimTemplates: - apiVersion: v1 kind: PersistentVolumeClaim @@ -98,4 +108,5 @@ spec: volumeMode: Filesystem status: phase: Pending -{{ end }} \ No newline at end of file + {{- end }} +{{- end }} diff --git a/charts/firehose-antelope/charts/reader-node/values.yaml b/charts/firehose-antelope/charts/reader-node/values.yaml index 764d722..fb8fb44 100644 --- a/charts/firehose-antelope/charts/reader-node/values.yaml +++ b/charts/firehose-antelope/charts/reader-node/values.yaml @@ -66,12 +66,25 @@ statefulset: volumeMounts: - mountPath: "/etc/nodeos" name: configdir - - mountPath: "/nodeos-data" - name: datadir - mountPath: "/etc/nodeos/protocol_features" name: etc-nodeos-protocolfeatures storageSize: 10Gi storageLimit: 40Gi + persistentVolumeClaimEnabled: true + dataVolumeMount: + - mountPath: "/nodeos-data" + name: datadir + +# For local test +localVolume: + enabled: true + volumeMounts: + - mountPath: "/storage" + name: storage + volume: + - name: storage + persistentVolumeClaim: + claimName: storage-pvc ### Note: First port in list will be used as an Ingress port! service: @@ -101,11 +114,11 @@ readinessProbe: resources: limits: - cpu: 4 - memory: 8Gi + cpu: 2 + memory: 2Gi requests: - cpu: 3 - memory: 7Gi + cpu: 1 + memory: 1Gi pdb: minAvailable: 1 @@ -150,28 +163,28 @@ configmap: http-threads = 2 genesis.json: | { + "initial_timestamp": "2018-09-01T12:00:00.000", + "initial_key": "EOS7qBcTsKhCfUeGpn15JTtwrjEnZfXzeCDLnF1LtKr6xnMAr4b5k", "initial_configuration": { + "max_block_net_usage": 1048576, + "target_block_net_usage_pct": 1000, + "max_transaction_net_usage": 524288, "base_per_transaction_net_usage": 12, - "context_free_discount_net_usage_den": 100, + "net_usage_leeway": 500, "context_free_discount_net_usage_num": 20, - "deferred_trx_expiration_window": 600, - "max_authority_depth": 10, + "context_free_discount_net_usage_den": 100, "max_block_cpu_usage": 400000, - "max_block_net_usage": 1048576, - "max_inline_action_depth": 10, - "max_inline_action_size": 524287, - "max_transaction_cpu_usage": 200000, - "max_transaction_delay": 3888000, + "target_block_cpu_usage_pct": 1000, + "max_transaction_cpu_usage": 300000, + "min_transaction_cpu_usage": 100, + "ultra_veto_enabled": 1, "max_transaction_lifetime": 3600, - "max_transaction_net_usage": 524288, - "min_transaction_cpu_usage": 1, - "net_usage_leeway": 500, - "target_block_cpu_usage_pct": 2500, - "target_block_net_usage_pct": 1000, - "ultra_veto_enabled": 1 - }, - "initial_key": "EOS6bYy8KR5sBMKAk45Lkq88A1TwZRGDwph4X8ynJj7bkzSysoQvw", - "initial_timestamp": "2021-05-26T17:27:27.007" + "deferred_trx_expiration_window": 600, + "max_transaction_delay": 3888000, + "max_inline_action_size": 524287, + "max_inline_action_depth": 10, + "max_authority_depth": 10 + } } protocol_features: BUILTIN-ACTION_RETURN_VALUE.json: | diff --git a/charts/firehose-antelope/charts/relayer/templates/deployment.yaml b/charts/firehose-antelope/charts/relayer/templates/deployment.yaml index 9654d38..8d608e1 100644 --- a/charts/firehose-antelope/charts/relayer/templates/deployment.yaml +++ b/charts/firehose-antelope/charts/relayer/templates/deployment.yaml @@ -37,7 +37,9 @@ spec: - "--relayer-max-source-latency={{ .Values.deployment.relayerMaxSourceLatency }}" - "--relayer-grpc-listen-addr={{ .Values.deployment.relayerGrpcListenAddr }}" - "--relayer-source={{ .Values.deployment.relayerSource1 }}" + {{- if .Values.deployment.relayerSource2 }} - "--relayer-source={{ .Values.deployment.relayerSource2 }}" + {{- end }} {{- with .Values.deployment.extraCommands }} {{- toYaml . | nindent 14 }} {{- end }} @@ -59,6 +61,14 @@ spec: {{- end}} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- if .Values.localVolume.enabled }} + volumeMounts: + {{- toYaml .Values.localVolume.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.localVolume.enabled }} + volumes: + {{- toYaml .Values.localVolume.volume | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} @@ -75,4 +85,4 @@ spec: values: - {{ .Chart.Name }} topologyKey: kubernetes.io/hostname -{{ end }} \ No newline at end of file +{{- end }} \ No newline at end of file diff --git a/charts/firehose-antelope/charts/relayer/values.yaml b/charts/firehose-antelope/charts/relayer/values.yaml index 10e15ec..3b2c1f0 100644 --- a/charts/firehose-antelope/charts/relayer/values.yaml +++ b/charts/firehose-antelope/charts/relayer/values.yaml @@ -37,6 +37,17 @@ deployment: - name: INFO value: github.com/dfuse-io/.* +# For local test +localVolume: + enabled: true + volumeMounts: + - mountPath: "/storage" + name: storage + volume: + - name: storage + persistentVolumeClaim: + claimName: storage-pvc + ### Note: First port in list will be used as an Ingress port! service: labels: @@ -81,4 +92,3 @@ pdb: minAvailable: 1 nodeSelector: - app: application diff --git a/charts/firehose-antelope/charts/substreams-tier1/templates/statefulset.yaml b/charts/firehose-antelope/charts/substreams-tier1/templates/statefulset.yaml index c8b33b1..b910829 100644 --- a/charts/firehose-antelope/charts/substreams-tier1/templates/statefulset.yaml +++ b/charts/firehose-antelope/charts/substreams-tier1/templates/statefulset.yaml @@ -68,4 +68,12 @@ spec: {{- end}} resources: {{- toYaml .Values.resources | nindent 12 }} -{{ end }} \ No newline at end of file + {{- if .Values.localVolume.enabled }} + volumeMounts: + {{- toYaml .Values.localVolume.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.localVolume.enabled }} + volumes: + {{- toYaml .Values.localVolume.volume | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/firehose-antelope/charts/substreams-tier1/values.yaml b/charts/firehose-antelope/charts/substreams-tier1/values.yaml index a1f65aa..eedb237 100644 --- a/charts/firehose-antelope/charts/substreams-tier1/values.yaml +++ b/charts/firehose-antelope/charts/substreams-tier1/values.yaml @@ -46,6 +46,17 @@ statefulset: - name: INFO value: github.com/dfuse-io/.* +# For local test +localVolume: + enabled: true + volumeMounts: + - mountPath: "/storage" + name: storage + volume: + - name: storage + persistentVolumeClaim: + claimName: storage-pvc + ### Note: First port in list will be used as an Ingress port! service: labels: diff --git a/charts/firehose-antelope/charts/substreams-tier2/templates/statefulset.yaml b/charts/firehose-antelope/charts/substreams-tier2/templates/statefulset.yaml index 9a2da28..37b400b 100644 --- a/charts/firehose-antelope/charts/substreams-tier2/templates/statefulset.yaml +++ b/charts/firehose-antelope/charts/substreams-tier2/templates/statefulset.yaml @@ -62,4 +62,12 @@ spec: {{- end}} resources: {{- toYaml .Values.resources | nindent 12 }} -{{ end }} \ No newline at end of file + {{- if .Values.localVolume.enabled }} + volumeMounts: + {{- toYaml .Values.localVolume.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.localVolume.enabled }} + volumes: + {{- toYaml .Values.localVolume.volume | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/firehose-antelope/charts/substreams-tier2/values.yaml b/charts/firehose-antelope/charts/substreams-tier2/values.yaml index f57f012..6699c7c 100644 --- a/charts/firehose-antelope/charts/substreams-tier2/values.yaml +++ b/charts/firehose-antelope/charts/substreams-tier2/values.yaml @@ -41,6 +41,17 @@ statefulset: - name: INFO value: github.com/dfuse-io/.* +# For local test +localVolume: + enabled: true + volumeMounts: + - mountPath: "/storage" + name: storage + volume: + - name: storage + persistentVolumeClaim: + claimName: storage-pvc + ### Note: First port in list will be used as an Ingress port! service: labels: diff --git a/charts/firehose-antelope/templates/pv.yaml b/charts/firehose-antelope/templates/pv.yaml new file mode 100644 index 0000000..b1ee38f --- /dev/null +++ b/charts/firehose-antelope/templates/pv.yaml @@ -0,0 +1,26 @@ +{{- if .Values.persistentVolumeEnabled -}} +apiVersion: v1 +kind: PersistentVolume +metadata: + name: storage +spec: + accessModes: + - ReadWriteMany + capacity: + storage: 5Gi + hostPath: + path: /data/storage +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: storage-pvc +spec: + accessModes: + - ReadWriteMany + storageClassName: "" + volumeName: storage + resources: + requests: + storage: 1Gi +{{- end}} \ No newline at end of file diff --git a/charts/firehose-antelope/values.yaml b/charts/firehose-antelope/values.yaml index 3b4d0c5..a33e21e 100644 --- a/charts/firehose-antelope/values.yaml +++ b/charts/firehose-antelope/values.yaml @@ -8,3 +8,5 @@ serviceAccount: writerAnnotations: {} readerName: "" readerAnnotations: {} +# Only enable if testing locally +persistentVolumeEnabled: false diff --git a/charts/producer/.helmignore b/charts/producer/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/producer/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/producer/Chart.yaml b/charts/producer/Chart.yaml new file mode 100644 index 0000000..8be8e05 --- /dev/null +++ b/charts/producer/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +appVersion: 0.1.0 +description: A Helm chart for producer +name: producer +type: application +version: 0.1.0 diff --git a/charts/producer/Makefile b/charts/producer/Makefile new file mode 100644 index 0000000..96491e8 --- /dev/null +++ b/charts/producer/Makefile @@ -0,0 +1,12 @@ +POLICY_CONFIG ?= ../../.github/datree/policies.yaml + +dependency: + #Noop target + +lint: + helm lint + +test: + helm template . | datree test - --policy-config ${POLICY_CONFIG} + +.PHONY: dependency lint test \ No newline at end of file diff --git a/charts/producer/README.md b/charts/producer/README.md new file mode 100644 index 0000000..1fe2501 --- /dev/null +++ b/charts/producer/README.md @@ -0,0 +1,9 @@ +# producer Helm Chart + +Chart to deploy + +- [producer](https://github.com/ultraio/substreams-charts/charts/producer). + +## Requirements + +## Installation diff --git a/charts/producer/templates/_helpers.tpl b/charts/producer/templates/_helpers.tpl new file mode 100644 index 0000000..fd0c426 --- /dev/null +++ b/charts/producer/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "producer.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "producer.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "producer.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "producer.labels" -}} +helm.sh/chart: {{ include "producer.chart" . }} +{{ include "producer.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "producer.selectorLabels" -}} +app.kubernetes.io/name: {{ include "producer.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "producer.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "producer.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/producer/templates/pdb.yaml b/charts/producer/templates/pdb.yaml new file mode 100644 index 0000000..b0c3534 --- /dev/null +++ b/charts/producer/templates/pdb.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "producer.fullname" . }} + labels: + {{- include "producer.labels" . | nindent 4 }} +spec: + minAvailable: {{ .Values.pdb.minAvailable }} + selector: + matchLabels: + {{- include "producer.selectorLabels" . | nindent 6 }} diff --git a/charts/producer/templates/service.yaml b/charts/producer/templates/service.yaml new file mode 100644 index 0000000..a25adca --- /dev/null +++ b/charts/producer/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "producer.fullname" . }} + labels: + {{- with .Values.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- include "producer.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + {{- with .Values.service.ports }} + ports: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + {{- include "producer.selectorLabels" . | nindent 4 }} + publishNotReadyAddresses: true + clusterIP: None \ No newline at end of file diff --git a/charts/producer/templates/statefulset.yaml b/charts/producer/templates/statefulset.yaml new file mode 100644 index 0000000..e8bd105 --- /dev/null +++ b/charts/producer/templates/statefulset.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "producer.fullname" . }} + labels: + {{- include "producer.labels" . | nindent 4 }} + annotations: + reloader.stakater.com/search: "true" +spec: + restartPolicy: OnFailure + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "producer.selectorLabels" . | nindent 6 }} + serviceName: {{ include "producer.name" . }} + template: + metadata: + labels: + {{- include "producer.selectorLabels" . | nindent 8 }} + spec: + containers: + - name: "{{ .Values.statefulset.name }}" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["sh", "-c"] + args: + - | + echo "" | ultratest -n -s -D + while true; do sleep 30; done; + {{- with .Values.statefulset.ports }} + ports: + {{- toYaml . | nindent 12 }} + {{- end }} \ No newline at end of file diff --git a/charts/producer/values.yaml b/charts/producer/values.yaml new file mode 100644 index 0000000..71d7d41 --- /dev/null +++ b/charts/producer/values.yaml @@ -0,0 +1,55 @@ +# Default values for blockchain-service. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: "quay.io/ultra.io/3rdparty-devtools" + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: producer + +statefulset: + name: nodeos + # Others + ports: + - containerPort: 8888 + name: nodeos-http + protocol: TCP + - containerPort: 9876 + name: nodeos-p2p + protocol: TCP + +### Note: First port in list will be used as an Ingress port! +service: + labels: + monitoring: "true" + type: ClusterIP + ports: + - protocol: TCP + name: nodeos-http + port: 8888 + targetPort: 8888 + - protocol: TCP + name: nodeos-p2p + port: 9876 + targetPort: 9876 + +resources: + limits: + cpu: 4 + memory: 4Gi + requests: + cpu: 2 + memory: 2Gi + +pdb: + minAvailable: 1 + +nodeSelector: + role: application diff --git a/local-test-values.yaml b/local-test-values.yaml new file mode 100644 index 0000000..1f29941 --- /dev/null +++ b/local-test-values.yaml @@ -0,0 +1,120 @@ +# Enabled Persistent volume to replace cloud storage for substreams data +persistentVolumeEnabled: true + +reader-node: + replicaCount: 1 + statefulset: + commonOneBlockStoreUrl: "file:///storage/one-blocks" + persistentVolumeClaimEnabled: false + # enabled local storage to replace gloud storage + localVolume: + enabled: true + configmap: + config: + # Update p2p-peer-address with producer cluster DNS/IP + config.ini: | + http-server-address = 0.0.0.0:8888 + p2p-server-address = 0.0.0.0:9877 + p2p-max-nodes-per-host = 2 + connection-cleanup-period = 60 + verbose-http-errors = true + chain-state-db-size-mb = 64000 + http-validate-host = false + max-transaction-time = 5000 + abi-serializer-max-time-ms = 500000 + + plugin = eosio::net_api_plugin + plugin = eosio::chain_api_plugin + plugin = eosio::db_size_api_plugin + plugin = eosio::producer_api_plugin + + # Max speed for replay + validation-mode = light + + # Enable deep mind + deep-mind = true + agent-name = reader-node + + wasm-runtime = eos-vm-jit + eos-vm-oc-enable = true + eos-vm-oc-compile-threads = 4 + contracts-console = true + read-mode = head + p2p-accept-transactions = false + api-accept-transactions = false + + http-threads = 2 + p2p-peer-address = producer-0.producer.default.svc.cluster.local:9876 + # This is required since ultratest use a different genesis file than Ultra's deployemnt + genesis.json: | + { + "initial_timestamp": "2018-09-01T12:00:00.000", + "initial_key": "EOS7qBcTsKhCfUeGpn15JTtwrjEnZfXzeCDLnF1LtKr6xnMAr4b5k", + "initial_configuration": { + "max_block_net_usage": 1048576, + "target_block_net_usage_pct": 1000, + "max_transaction_net_usage": 524288, + "base_per_transaction_net_usage": 12, + "net_usage_leeway": 500, + "context_free_discount_net_usage_num": 20, + "context_free_discount_net_usage_den": 100, + "max_block_cpu_usage": 400000, + "target_block_cpu_usage_pct": 1000, + "max_transaction_cpu_usage": 300000, + "min_transaction_cpu_usage": 100, + "ultra_veto_enabled": 1, + "max_transaction_lifetime": 3600, + "deferred_trx_expiration_window": 600, + "max_transaction_delay": 3888000, + "max_inline_action_size": 524287, + "max_inline_action_depth": 10, + "max_authority_depth": 10 + } + } +merger: + replicaCount: 1 + statefulset: + commonOneBlockStoreUrl: "file:///storage/one-blocks" + commonMergedBlocksStoreUrl: "file:///storage/merged-blocks" + commonForkedBlocksStoreUrl: "file:///storage/forked-blocks" + persistentVolumeClaimEnabled: false + # enabled local storage to replace gloud storage + localVolume: + enabled: true +relayer: + replicaCount: 1 + deployment: + commonOneBlockStoreUrl: "file:///storage/one-blocks" + relayerSource2: "" + # enabled local storage to replace gloud storage + localVolume: + enabled: true +substreams-tier1: + replicaCount: 1 + statefulset: + commonOneBlockStoreUrl: "file:///storage/one-blocks" + commonMergedBlocksStoreUrl: "file:///storage/merged-blocks" + commonForkedBlocksStoreUrl: "file:///storage/forked-blocks" + substreamsStateStoreUrl: "file:///storage/state-store" + # enabled local storage to replace gloud storage + localVolume: + enabled: true +substreams-tier2: + replicaCount: 1 + statefulset: + commonOneBlockStoreUrl: "file:///storage/one-blocks" + commonMergedBlocksStoreUrl: "file:///storage/merged-blocks" + commonForkedBlocksStoreUrl: "file:///storage/forked-blocks" + substreamsStateStoreUrl: "file:///storage/state-store" + # enabled local storage to replace gloud storage + localVolume: + enabled: true +firehose: + replicaCount: 1 + statefulset: + commonOneBlockStoreUrl: "file:///storage/one-blocks" + commonMergedBlocksStoreUrl: "file:///storage/merged-blocks" + commonForkedBlocksStoreUrl: "file:///storage/forked-blocks" + # enabled local storage to replace gloud storage + localVolume: + enabled: true From ada2e99e6478b16e3488b031cc9b07ed0a81412d Mon Sep 17 00:00:00 2001 From: Duncan Dam Date: Wed, 17 Apr 2024 23:45:33 +0700 Subject: [PATCH 2/6] version bump --- charts/firehose-antelope/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/firehose-antelope/Chart.yaml b/charts/firehose-antelope/Chart.yaml index 89348ec..0083e23 100644 --- a/charts/firehose-antelope/Chart.yaml +++ b/charts/firehose-antelope/Chart.yaml @@ -3,7 +3,7 @@ appVersion: 0.1.0 description: A Helm chart for firehose-antelope components name: firehose-antelope type: application -version: 1.0.2 +version: 1.0.3 dependencies: - name: reader-node From 32b8b9629a12798f2ee979e94ca115f74481ebfd Mon Sep 17 00:00:00 2001 From: Duncan Dam Date: Wed, 17 Apr 2024 23:47:38 +0700 Subject: [PATCH 3/6] remove release --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 34d023d..8af37b2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,6 @@ stages: - lint - test - # - release workflow: rules: From 1f81b13c8494a0d914f8aa03280404ecc3afefc8 Mon Sep 17 00:00:00 2001 From: Duncan Dam Date: Wed, 17 Apr 2024 23:50:19 +0700 Subject: [PATCH 4/6] update --- charts/producer/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/producer/values.yaml b/charts/producer/values.yaml index 71d7d41..554e778 100644 --- a/charts/producer/values.yaml +++ b/charts/producer/values.yaml @@ -53,3 +53,4 @@ pdb: nodeSelector: role: application + \ No newline at end of file From e7d1dfa68a3f2887e6b12083ecbfc09518811765 Mon Sep 17 00:00:00 2001 From: Duncan Dam Date: Wed, 17 Apr 2024 23:51:52 +0700 Subject: [PATCH 5/6] update --- charts/producer/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/producer/values.yaml b/charts/producer/values.yaml index 554e778..32c8bb4 100644 --- a/charts/producer/values.yaml +++ b/charts/producer/values.yaml @@ -53,4 +53,4 @@ pdb: nodeSelector: role: application - \ No newline at end of file + From 419c4508dcb7ec8b0561aed5df257bac85d3450e Mon Sep 17 00:00:00 2001 From: Duncan Dam Date: Wed, 17 Apr 2024 23:53:32 +0700 Subject: [PATCH 6/6] update --- charts/producer/values.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/charts/producer/values.yaml b/charts/producer/values.yaml index 32c8bb4..babf147 100644 --- a/charts/producer/values.yaml +++ b/charts/producer/values.yaml @@ -16,7 +16,7 @@ fullnameOverride: producer statefulset: name: nodeos - # Others + # Others ports: - containerPort: 8888 name: nodeos-http @@ -53,4 +53,3 @@ pdb: nodeSelector: role: application -