[BLOCK-2276] support local test

.gitlab-ci.yml

@@ -1,7 +1,6 @@
 stages:
   - lint
   - test
-  # - release
 
 workflow:
   rules:

README.md

@@ -1,3 +1,88 @@
 # substreams-charts
 
-Helm repository for substreams charts.
+Helm repository for substreams charts.
+
+## For `minikube` testing
+
+### Prerequisites
+
+Make sure you have all these applications available.
+
+- `minikube`
+- `helm`
+- `kubectl`
+- `docker`
+- [Optional] `substreams` CLI
+
+### Start `minikube`
+
+Make sure `docker` is running.
+
+```sh
+minikube start
+```
+
+### Deploy `producer` chart.
+
+```sh
+helm install producer ./charts/producer
+```
+
+Make sure to wait until the `producer` pod is ready.
+
+```sh
+kubectl get pod
+```
+```sh
+NAME         READY   STATUS    RESTARTS   AGE
+producer-0   1/1     Running   0          57s
+```
+
+### Deploy `firehose-antelope` chart.
+
+```sh
+helm install firehose-antelope ./charts/firehose-antelope -f ./local-test-values.yaml
+```
+If the deployment has no error, you should see this
+
+```sh
+kubectl get pod
+```
+```sh
+NAME                       READY   STATUS    RESTARTS   AGE
+firehose-0                 0/1     Running   0          2m27s
+merger-0                   1/1     Running   0          2m27s
+producer-0                 1/1     Running   0          3m19s
+reader-node-0              1/1     Running   0          2m27s
+relayer-7fc86d45cc-dz8vz   1/1     Running   0          2m27s
+substreams-tier1-0         1/1     Running   0          2m27s
+substreams-tier2-0         1/1     Running   0          2m27s
+```
+
+### Test with `substreams` CLI
+
+Make sure to have [Substreams Modules Repo](https://github.com/ultraio/substreams-modules) built.
+
+Port forwarding
+
+```sh
+kubectl port-forward service/substreams-tier1 9000:9000
+```
+
+Run `substreams` CLI
+
+```sh
+substreams run -e localhost:9000 ULTRA_PATH/substreams-modules/eosio.token/substreams.yaml map_transfers --start-block 2 --stop-block 100 --plaintext
+```
+
+### To stop `minikube`
+
+```sh
+minikube stop
+```
+
+### [Recommended] To delete persistent data before starting a new test
+
+```sh
+minikube delete
+```

charts/firehose-antelope/Chart.yaml

@@ -3,7 +3,7 @@ appVersion: 0.1.0
 description: A Helm chart for firehose-antelope components
 name: firehose-antelope
 type: application
-version: 1.0.2
+version: 1.0.3
 
 dependencies:
   - name: reader-node

charts/firehose-antelope/charts/firehose/templates/statefulset.yaml

@@ -60,4 +60,12 @@ spec:
           {{- end}}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
-{{ end }}
+          {{- if .Values.localVolume.enabled }}
+          volumeMounts:
+            {{- toYaml .Values.localVolume.volumeMounts | nindent 12 }}
+          {{- end }}
+      {{- if .Values.localVolume.enabled }}
+      volumes:
+        {{- toYaml .Values.localVolume.volume | nindent 8 }}
+      {{- end }}
+{{- end }}

charts/firehose-antelope/charts/firehose/values.yaml

@@ -19,7 +19,7 @@ statefulset:
   serviceAccountName: ""
   # commands
   commonAuthPlugin: null://
-  commonLiveBlocksAddr: ns:///relayer:9000
+  commonLiveBlocksAddr: dns:///relayer:9000
   commonSystemShutdownSignalDelay: 30s
   commonMergedBlocksStoreUrl: ""
   commonOneBlockStoreUrl: ""
@@ -39,6 +39,17 @@ statefulset:
     - name: INFO
       value: github.com/dfuse-io/.*
 
+# For local test
+localVolume:
+  enabled: true
+  volumeMounts:
+    - mountPath: "/storage"
+      name: storage
+  volume:
+    - name: storage
+      persistentVolumeClaim:
+        claimName: storage-pvc
+
 ### Note: First port in list will be used as an Ingress port!
 service:
   labels:

charts/firehose-antelope/charts/merger/templates/statefulset.yaml

@@ -61,8 +61,17 @@ spec:
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           volumeMounts:
-            - mountPath: "/data"
-              name: datadir
+          {{- if .Values.statefulset.persistentVolumeClaimEnabled }}
+            {{- toYaml .Values.statefulset.dataVolumeMount | nindent 12 }}
+          {{- end }}
+          {{- if .Values.localVolume.enabled }}
+            {{- toYaml .Values.localVolume.volumeMounts | nindent 12 }}
+          {{- end }}
+      {{- if .Values.localVolume.enabled }}
+      volumes:
+        {{- toYaml .Values.localVolume.volume | nindent 8 }}
+      {{- end }}
+  {{- if .Values.statefulset.persistentVolumeClaimEnabled }}
   volumeClaimTemplates:
     - apiVersion: v1
       kind: PersistentVolumeClaim
@@ -80,4 +89,5 @@ spec:
         volumeMode: Filesystem
       status:
         phase: Pending
-{{ end }}
+  {{- end}}
+{{- end }}

charts/firehose-antelope/charts/merger/values.yaml

@@ -40,6 +40,21 @@ statefulset:
       protocol: TCP
   storageLimit: 4Gi
   storageSize: 1Gi
+  persistentVolumeClaimEnabled: true
+  dataVolumeMount:
+    - mountPath: "/nodeos-data"
+      name: datadir
+
+# For local test
+localVolume:
+  enabled: true
+  volumeMounts:
+    - mountPath: "/storage"
+      name: storage
+  volume:
+    - name: storage
+      persistentVolumeClaim:
+        claimName: storage-pvc
 
 ### Note: First port in list will be used as an Ingress port!
 service:

charts/firehose-antelope/charts/reader-node/templates/statefulset.yaml

@@ -32,7 +32,7 @@ spec:
             - "--config-file="
             - "--log-format=stackdriver"
             - "--log-to-file=false"
-            - "--common-one-block-store-url={{ .Values.statefulset.commonOneBlockStoreURL }}"
+            - "--common-one-block-store-url={{ .Values.statefulset.commonOneBlockStoreUrl }}"
             - "--common-system-shutdown-signal-delay={{ .Values.statefulset.commonSystemShutdownSignalDelay }}"
             - "--reader-node-arguments={{ .Values.statefulset.readerNodeArguments }}"
             - "--reader-node-blocks-chan-capacity={{ .Values.statefulset.readerNodeBlocksChanCapacity }}"
@@ -71,6 +71,12 @@ spec:
           volumeMounts:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+          {{- if .Values.statefulset.persistentVolumeClaimEnabled }}
+            {{- toYaml .Values.statefulset.dataVolumeMount | nindent 12 }}
+          {{- end }}
+          {{- if .Values.localVolume.enabled }}
+            {{- toYaml .Values.localVolume.volumeMounts | nindent 12 }}
+          {{- end }}
       terminationGracePeriodSeconds: 60
       volumes:
         - configMap:
@@ -81,6 +87,10 @@ spec:
             defaultMode: 420
             name: {{ include "reader-node.fullname" . }}-protocol-features
           name: etc-nodeos-protocolfeatures
+      {{- if .Values.localVolume.enabled }}
+        {{- toYaml .Values.localVolume.volume | nindent 8 }}
+      {{- end }}
+  {{- if .Values.statefulset.persistentVolumeClaimEnabled }}
   volumeClaimTemplates:
     - apiVersion: v1
       kind: PersistentVolumeClaim
@@ -98,4 +108,5 @@ spec:
         volumeMode: Filesystem
       status:
         phase: Pending
-{{ end }}
+  {{- end }}
+{{- end }}

charts/firehose-antelope/charts/reader-node/values.yaml

@@ -66,12 +66,25 @@ statefulset:
   volumeMounts:
     - mountPath: "/etc/nodeos"
       name: configdir
-    - mountPath: "/nodeos-data"
-      name: datadir
     - mountPath: "/etc/nodeos/protocol_features"
       name: etc-nodeos-protocolfeatures
   storageSize: 10Gi
   storageLimit: 40Gi
+  persistentVolumeClaimEnabled: true
+  dataVolumeMount:
+    - mountPath: "/nodeos-data"
+      name: datadir
+
+# For local test
+localVolume:
+  enabled: true
+  volumeMounts:
+    - mountPath: "/storage"
+      name: storage
+  volume:
+    - name: storage
+      persistentVolumeClaim:
+        claimName: storage-pvc
 
 ### Note: First port in list will be used as an Ingress port!
 service:
@@ -101,11 +114,11 @@ readinessProbe:
 
 resources:
   limits:
-    cpu: 4
-    memory: 8Gi
+    cpu: 2
+    memory: 2Gi
   requests:
-    cpu: 3
-    memory: 7Gi
+    cpu: 1
+    memory: 1Gi
 
 pdb:
   minAvailable: 1
@@ -150,28 +163,28 @@ configmap:
       http-threads = 2
     genesis.json: |
       {
+        "initial_timestamp": "2018-09-01T12:00:00.000",
+        "initial_key": "EOS7qBcTsKhCfUeGpn15JTtwrjEnZfXzeCDLnF1LtKr6xnMAr4b5k",
         "initial_configuration": {
+          "max_block_net_usage": 1048576,
+          "target_block_net_usage_pct": 1000,
+          "max_transaction_net_usage": 524288,
           "base_per_transaction_net_usage": 12,
-          "context_free_discount_net_usage_den": 100,
+          "net_usage_leeway": 500,
           "context_free_discount_net_usage_num": 20,
-          "deferred_trx_expiration_window": 600,
-          "max_authority_depth": 10,
+          "context_free_discount_net_usage_den": 100,
           "max_block_cpu_usage": 400000,
-          "max_block_net_usage": 1048576,
-          "max_inline_action_depth": 10,
-          "max_inline_action_size": 524287,
-          "max_transaction_cpu_usage": 200000,
-          "max_transaction_delay": 3888000,
+          "target_block_cpu_usage_pct": 1000,
+          "max_transaction_cpu_usage": 300000,
+          "min_transaction_cpu_usage": 100,
+          "ultra_veto_enabled": 1,
           "max_transaction_lifetime": 3600,
-          "max_transaction_net_usage": 524288,
-          "min_transaction_cpu_usage": 1,
-          "net_usage_leeway": 500,
-          "target_block_cpu_usage_pct": 2500,
-          "target_block_net_usage_pct": 1000,
-          "ultra_veto_enabled": 1
-        },
-        "initial_key": "EOS6bYy8KR5sBMKAk45Lkq88A1TwZRGDwph4X8ynJj7bkzSysoQvw",
-        "initial_timestamp": "2021-05-26T17:27:27.007"
+          "deferred_trx_expiration_window": 600,
+          "max_transaction_delay": 3888000,
+          "max_inline_action_size": 524287,
+          "max_inline_action_depth": 10,
+          "max_authority_depth": 10
+        }
       }
   protocol_features:
     BUILTIN-ACTION_RETURN_VALUE.json: |

charts/firehose-antelope/charts/relayer/templates/deployment.yaml

@@ -37,7 +37,9 @@ spec:
             - "--relayer-max-source-latency={{ .Values.deployment.relayerMaxSourceLatency }}"
             - "--relayer-grpc-listen-addr={{ .Values.deployment.relayerGrpcListenAddr }}"
             - "--relayer-source={{ .Values.deployment.relayerSource1 }}"
+            {{- if .Values.deployment.relayerSource2 }}
             - "--relayer-source={{ .Values.deployment.relayerSource2 }}"
+            {{- end }}
             {{- with .Values.deployment.extraCommands }}
               {{- toYaml . | nindent 14 }}
             {{- end }}
@@ -59,6 +61,14 @@ spec:
           {{- end}}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
+          {{- if .Values.localVolume.enabled }}
+          volumeMounts:
+            {{- toYaml .Values.localVolume.volumeMounts | nindent 12 }}
+          {{- end }}
+      {{- if .Values.localVolume.enabled }}
+      volumes:
+        {{- toYaml .Values.localVolume.volume | nindent 8 }}
+      {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
@@ -75,4 +85,4 @@ spec:
                       values:
                         - {{ .Chart.Name }}
                 topologyKey: kubernetes.io/hostname
-{{ end }}
+{{- end }}

charts/firehose-antelope/charts/relayer/values.yaml

@@ -37,6 +37,17 @@ deployment:
     - name: INFO
       value: github.com/dfuse-io/.*
 
+# For local test
+localVolume:
+  enabled: true
+  volumeMounts:
+    - mountPath: "/storage"
+      name: storage
+  volume:
+    - name: storage
+      persistentVolumeClaim:
+        claimName: storage-pvc
+
 ### Note: First port in list will be used as an Ingress port!
 service:
   labels:
@@ -81,4 +92,3 @@ pdb:
   minAvailable: 1
 
 nodeSelector:
-  app: application