[BLOCK-1994] build docker image (#4)

* [BLOCK-1994] docker image workflow

* update nodeos docker

* cleaning up

* typo fix

.github/workflows/dockerimage.yml

@@ -29,4 +29,73 @@ jobs:
     steps:
       - name: Set firehose-antelope tag
         run: |
-          echo "FIREHOSE_TAG=${{ github.event.inputs.firehose-antelope }}" >> $GITHUB_ENV
+          echo "FIREHOSE_TAG=${{ github.event.inputs.firehose-antelope }}" >> $GITHUB_ENV
+      
+      - name: Set nodeos version
+        if: ${{ github.event.inputs.eosio != 'null' }}
+        run: |
+          echo "EOSIO_TAG=${{ github.event.inputs.eosio }}" >> $GITHUB_ENV
+          echo "FIREHOSE_NODEOS_TAG=${{ github.event.inputs.firehose-antelope }}-nodeos${{ github.event.inputs.eosio }}" >> $GITHUB_ENV
+
+      # checkout based on firehose-antelope tag
+      - name: Checkout firehose-antelope
+        uses: actions/checkout@v3
+        with:
+          ref: "${{ env.FIREHOSE_TAG }}"
+
+      # check release commit
+      - name: Update Commit Hash
+        run: |
+          echo "RELEASE_COMMIT=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+
+      # Only download eosio if set
+      - name: Download EOSIO Debian package
+        if: ${{ github.event.inputs.eosio != 'null' }}
+        uses: Legion2/[email protected]
+        with:
+          repository: ultraio/eosio
+          tag: '${{ github.event.inputs.eosio }}'
+          path: .
+          file:  eosio-${{ env.EOSIO_TAG }}.deb
+          token: '${{ secrets.BLOCKCHAIN_CI_SECRET }}'
+
+      # gcloud authorization
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: '${{ secrets.GCR_KEY }}'
+
+      - name: Setup gcloud cli
+        uses: google-github-actions/setup-gcloud@v1
+        with:
+          project_id: dfuse-302310
+
+      # docker gcloud config
+      - name: Docker config
+        run: |
+          gcloud auth configure-docker -q
+
+      # Check if firehose image is existed. Failed if not existed
+      - name: Check Firehose Image
+        id: firehose-check
+        continue-on-error: true
+        run: |
+          docker manifest inspect ${{ env.DOCKER_IMAGE }}:${{ env.FIREHOSE_TAG }}
+
+      # Only build firehose-antelope image if not existed or overwrite is set
+      - name: Build firehose-antelope Image Without Nodeos
+        if: ${{  (steps.firehose-check.outcome == 'failure' || github.event.inputs.overwrite == 'true') }}
+        run: |
+          docker build -f firehose.Dockerfile . -t ${{ env.DOCKER_IMAGE }}:${{ env.FIREHOSE_TAG }} \
+                --build-arg VERSION=${{ env.FIREHOSE_TAG }} \
+                --build-arg COMMIT=${{ env.RELEASE_COMMIT }}
+          docker push ${{ env.DOCKER_IMAGE }}:${{ env.FIREHOSE_TAG }}
+      
+      # Only build firehose-antelope with nodeos if eosio is set and firehose-antelope image are exist
+      - name: Build firehose-antelope Image With Nodeos
+        if: ${{ github.event.inputs.eosio != 'null' }}
+        run: |
+          docker build -f nodeos.Dockerfile . -t ${{ env.DOCKER_IMAGE }}:${{ env.FIREHOSE_NODEOS_TAG }} \
+                --build-arg FIREHOSE_IMAGE=${{ env.DOCKER_IMAGE }}:${{ env.FIREHOSE_TAG }} \
+                --build-arg DEB_PKG=eosio-${{ env.EOSIO_TAG }}.deb
+          docker push ${{ env.DOCKER_IMAGE }}:${{ env.FIREHOSE_NODEOS_TAG }}

devel/standard/standard.yaml

@@ -10,6 +10,8 @@ start:
     # Specifies the path to the binary
     reader-node-path: "nodeos"
     # Flags that will be added to the dummy chain process command
-#   reader-node-arguments: ""
-    reader-node-arguments: "--config-dir=/path-to-ultra/firehose-antelope/firehose-data/reader/config --data-dir=/path-to-ultra/firehose-antelope/firehose-data/reader/data --genesis-json=/path-to-ultra/firehose-antelope/firehose-data/reader/config/genesis.json"
-#   reader-node-config-dir: "{sf-data-dir}/config/jungle4"
+    reader-node-arguments: +
+      --config-dir=/path-to-ultra/firehose-antelope/firehose-data/reader/config 
+      --data-dir=/path-to-ultra/firehose-antelope/firehose-data/reader/data
+      --genesis-json=/path-to-ultra/firehose-antelope/firehose-data/reader/config/genesis.json
+

firehose.Dockerfile

@@ -0,0 +1,35 @@
+# syntax=docker/dockerfile:1.2
+
+FROM golang:1.21 as firehose
+ARG COMMIT
+ARG VERSION
+RUN mkdir -p /work
+ADD . /work
+WORKDIR /work
+RUN go test ./...
+RUN go build -ldflags "-s -w -X main.version=${VERSION} -X main.commit=${COMMIT}" -v -o /work/fireantelope ./cmd/fireantelope
+
+FROM ubuntu:22.04 as base
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
+    apt-get -y install -y \
+    ca-certificates vim htop iotop sysstat wget \
+    dstat strace lsof curl jq tzdata && \
+    rm -rf /var/cache/apt /var/lib/apt/lists/*
+
+RUN wget http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.0g-2ubuntu4_amd64.deb
+RUN dpkg -i libssl1.1_1.1.0g-2ubuntu4_amd64.deb
+
+RUN rm /etc/localtime && ln -snf /usr/share/zoneinfo/America/Montreal /etc/localtime && dpkg-reconfigure -f noninteractive tzdata
+
+RUN mkdir /tmp/wasmer-install && cd /tmp/wasmer-install && \
+    curl -L https://github.com/wasmerio/wasmer/releases/download/2.3.0/wasmer-linux-amd64.tar.gz | tar xzf - && \
+    mv lib/libwasmer.a lib/libwasmer.so /usr/lib/ && cd / && rm -rf /tmp/wasmer-install
+
+COPY --from=firehose /work/fireantelope /app/fireantelope
+COPY --from=firehose /work/tools/fireantelope/motd_generic /etc/
+COPY --from=firehose /work/tools/fireantelope/motd_node_manager /etc/
+COPY --from=firehose /work/tools/fireantelope/99-firehose.sh /etc/profile.d/
+COPY --from=firehose /work/tools/fireantelope/scripts/* /usr/local/bin
+
+ENV PATH=$PATH:/app

nodeos.Dockerfile

@@ -0,0 +1,11 @@
+ARG FIREHOSE_IMAGE=""
+ARG DEB_PKG=""
+
+FROM ${FIREHOSE_IMAGE}
+ARG DEB_PKG
+RUN apt update &&\
+    apt-get -y install curl ca-certificates libc6 libgcc1 libstdc++6 libtinfo5 zlib1g libusb-1.0-0 libcurl3-gnutls
+RUN mkdir -p /var/cache/apt/archives/
+ADD ${DEB_PKG} /var/cache/apt/archives/
+RUN dpkg -i /var/cache/apt/archives/${DEB_PKG}
+RUN rm -rf /var/cache/apt/*

tools/fireantelope/99-firehose.sh

@@ -0,0 +1,12 @@
+##
+# This is place inside `/etc/profile.d/99-firehose.sh`
+# on built system an executed to provide message to use when they
+# connect on the box.
+export PATH=$PATH:/app
+
+# If we are in a "node-manager" image, display special scripts motd
+if [[ -d /nodeos-data/ ]]; then
+    cat /etc/motd_node_manager
+else
+    cat /etc/motd_generic
+fi

tools/fireantelope/motd

@@ -0,0 +1,27 @@
+    ______________  _________    _   ________________    ____  ____ 
+   / ____/  _/ __ \/ ____/   |  / | / /_  __/ ____/ /   / __ \/ __ \
+  / /_   / // /_/ / __/ / /| | /  |/ / / / / __/ / /   / / / / /_/ /
+ / __/ _/ // _, _/ /___/ ___ |/ /|  / / / / /___/ /___/ /_/ / ____/ 
+/_/   /___/_/ |_/_____/_/  |_/_/ |_/ /_/ /_____/_____/\____/_/      
+
+
+Welcome to this node manager pod, common commands:
+ - Show version           fireantelope --version
+
+Node manager commands (eos-... <tab> to show all):
+ - Check HEAD & LIB info              cleos get info
+ - Check if producing                 eos-isproducing
+ - Check if has peers                 eos-haspeers
+
+ - List all backups                   eos-listbackups
+ - Restore from backup                eos-restore-from-pitreos [<backupName>]
+ - Restore from snapshot              eos-restore-snapshot [<snapshotName>]
+ - Perform a backup                   eos-backup
+
+ - Rewind Nodeos instance to LIB      eos-rewind-to-lib
+ - Reset continuity checker file      eos-reset-continuity-check
+
+ - Stop Nodeos instance               eos-maintenance
+ - Start or restart Nodeos instance   eos-resume
+
+**Important** If you see this, it's because the image was not built with lastest workflow!

tools/fireantelope/motd_generic

@@ -0,0 +1,8 @@
+    ______________  _________    _   ________________    ____  ____ 
+   / ____/  _/ __ \/ ____/   |  / | / /_  __/ ____/ /   / __ \/ __ \
+  / /_   / // /_/ / __/ / /| | /  |/ / / / / __/ / /   / / / / /_/ /
+ / __/ _/ // _, _/ /___/ ___ |/ /|  / / / / /___/ /___/ /_/ / ____/ 
+/_/   /___/_/ |_/_____/_/  |_/_/ |_/ /_/ /_____/_____/\____/_/      
+
+Welcome to this pod, common commands:
+ - Show version           fireantelope --version

tools/fireantelope/motd_node_manager

@@ -0,0 +1,24 @@
+    ______________  _________    _   ________________    ____  ____ 
+   / ____/  _/ __ \/ ____/   |  / | / /_  __/ ____/ /   / __ \/ __ \
+  / /_   / // /_/ / __/ / /| | /  |/ / / / / __/ / /   / / / / /_/ /
+ / __/ _/ // _, _/ /___/ ___ |/ /|  / / / / /___/ /___/ /_/ / ____/ 
+/_/   /___/_/ |_/_____/_/  |_/_/ |_/ /_/ /_____/_____/\____/_/      
+
+Welcome to this node manager pod, common commands:
+ - Show version           fireantelope --version
+
+Node manager commands (eos-... <tab> to show all):
+ - Check HEAD & LIB info              cleos get info
+ - Check if producing                 eos-isproducing
+ - Check if has peers                 eos-haspeers
+
+ - List all backups                   eos-listbackups
+ - Restore from backup                eos-restore-from-pitreos [<backupName>]
+ - Restore from snapshot              eos-restore-snapshot [<snapshotName>]
+ - Perform a backup                   eos-backup
+
+ - Rewind Nodeos instance to LIB      eos-rewind-to-lib
+ - Reset continuity checker file      eos-reset-continuity-check
+
+ - Stop Nodeos instance               eos-maintenance
+ - Start or restart Nodeos instance   eos-resume

tools/fireantelope/scripts/eos-backup

@@ -0,0 +1,3 @@
+#!/bin/bash
+# description: Take a backup using pitreos
+curl -XPOST localhost:8080/v1/backup?sync=true

tools/fireantelope/scripts/eos-haspeers

@@ -0,0 +1,3 @@
+#!/bin/bash
+# description: show the number of currently valid connected peers
+cleos net peers |grep chain_id |grep -v 0000000000000000000000000000000 |wc -l

tools/fireantelope/scripts/eos-isproducing

@@ -0,0 +1,7 @@
+#!/bin/bash
+# description: prints either 'producing' if this node has production active, or 'paused'
+if curl -sS localhost:8888/v1/producer/paused 2>&1 |grep -q ^false$; then
+	echo producing
+else
+	echo paused
+fi

tools/fireantelope/scripts/eos-listbackups

@@ -0,0 +1,3 @@
+#!/bin/bash
+# description: List last available pitreos backups
+curl -sS 'localhost:8080/v1/list_backups'

tools/fireantelope/scripts/eos-maintenance

@@ -0,0 +1,3 @@
+#!/bin/bash
+# description: Stop nodeos for maintenance
+curl -sS -XPOST localhost:8080/v1/maintenance?sync=true

tools/fireantelope/scripts/eos-reset-continuity-check

@@ -0,0 +1,3 @@
+#!/bin/bash
+# description: (for mindreader only) Remove ConcurrencyCheck files, to force it going over a hole
+curl -sS -XPOST localhost:8080/v1/reset_cc

tools/fireantelope/scripts/eos-restore-from-pitreos

@@ -0,0 +1,7 @@
+#!/bin/bash
+# description: Restore from backup (you can pass the backup name as an argument, or no argument to restore latest)
+if [[ "$1" != "" ]]; then
+  curl -XPOST "localhost:8080/v1/restore?backupName=$1&sync=true"
+else
+  curl -XPOST "localhost:8080/v1/restore?sync=true"
+fi

tools/fireantelope/scripts/eos-restore-snapshot

@@ -0,0 +1,7 @@
+#!/bin/bash
+# description: Perform a snapshot restore
+if [[ "$1" != "" ]]; then
+  curl -XPOST "localhost:8080/v1/snapshot_restore?snapshotName=$1&sync=true"
+else
+  curl -XPOST "localhost:8080/v1/snapshot_restore?sync=true"
+fi

tools/fireantelope/scripts/eos-resume

@@ -0,0 +1,3 @@
+#!/bin/bash
+# description: Start nodeos after maintenance
+curl -sS -XPOST localhost:8080/v1/resume?sync=true

tools/fireantelope/scripts/eos-rewind-to-lib

@@ -0,0 +1,47 @@
+#!/bin/bash
+# description: (for mindreader only) Rewind nodeos by setting its HEAD to its LIB
+curl -sS -XPOST localhost:8080/v1/maintenance?sync=true
+
+nodeos --config-dir=/etc/nodeos --data-dir=/nodeos-data --pause-on-startup --read-mode=read-only --allowed-connection=none &
+NODEOSPID=$!
+while sleep 1; do
+	HEAD=$(cleos get info 2>&1 |awk '/"head_block_num"/ {print $2}' |sed 's/,//')
+	if [[ "$HEAD" != "" ]]; then
+ 		break
+	fi
+	if [ -n "$NODEOSPID" -a -e /proc/$NODEOSPID ]; then
+	    echo "waiting for nodeos to answer..."
+        else
+		echo "nodeos process died. cannot continue rewind, you may need to restore from pitreos."
+		return 1
+	fi
+done
+
+kill $NODEOSPID
+wait $NODEOSPID
+
+
+nodeos --config-dir=/etc/nodeos --data-dir=/nodeos-data --pause-on-startup --read-mode=irreversible --allowed-connection=none &
+NODEOSPID=$!
+while sleep 1; do
+	NEWHEAD=$(cleos get info 2>&1 |awk '/"head_block_num"/ {print $2}' |sed 's/,//')
+	if [[ "$NEWHEAD" != "" ]]; then
+ 		break
+	fi
+	if [ -n "$NODEOSPID" -a -e /proc/$NODEOSPID ]; then
+	    echo "waiting for nodeos to answer..."
+        else
+		echo "nodeos process died. cannot continue rewind, you may need to restore from pitreos."
+		return 1
+	fi
+done
+
+kill $NODEOSPID
+wait $NODEOSPID
+
+
+echo -e "\nHead rewound from $HEAD to $NEWHEAD"
+echo "You may need to run 'eos-reset-continuity-check' if you mindreader was locked because of a non-contiguous block before you run 'eos-resume' again..."
+
+
+

tools/fireantelope/scripts/eos-snapshot

@@ -0,0 +1,3 @@
+#!/bin/bash
+# description: Perform a snapshot
+curl -sS -XPOST localhost:8080/v1/snapshot?sync=true