fix(kicker init): remove rust target to avoid out of disk issue in …

…CI (#357)

fix(kicker init): remove rust target to avoid `out of disk` issue in CI

.github/workflows/godwoken-tests.yml

@@ -9,8 +9,25 @@ on:
 
 jobs:
   godwoken-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        MANUAL_BUILD_GODWOKEN: ["true", "false"]
+        MANUAL_BUILD_WEB3: ["true", "false"]
+        MANUAL_BUILD_WEB3_INDEXER: ["true", "false"]
+        # Disable MANUAL_BUILD of gwos and gwos-evm temporarily,
+        # because there is a MANUAL_BUILD_SCRIPTS bug related to capsule.
+        # See https://github.com/godwokenrises/godwoken-kicker/blob/a6c66edb5537181/kicker#L608-L614
+        # MANUAL_BUILD_POLYJUICE: ["true", "false"]
+        # MANUAL_BUILD_SCRIPTS: ["true", "false"]
+
     uses: godwokenrises/godwoken-tests/.github/workflows/reusable-integration-test-v1.yml@develop
     with:
       extra_github_env: |
         GODWOKEN_KICKER_REPO=${{ github.repository }}
         GODWOKEN_KICKER_REF=${{ github.ref }}
+        MANUAL_BUILD_GODWOKEN=${{ matrix.MANUAL_BUILD_GODWOKEN }}
+        MANUAL_BUILD_WEB3=${{ matrix.MANUAL_BUILD_WEB3 }}
+        MANUAL_BUILD_WEB3_INDEXER=${{ matrix.MANUAL_BUILD_WEB3_INDEXER }}
+        MANUAL_BUILD_POLYJUICE=${{ matrix.MANUAL_BUILD_POLYJUICE }}
+        MANUAL_BUILD_SCRIPTS=${{ matrix.MANUAL_BUILD_SCRIPTS }}

docker/docker-compose.yml

@@ -21,6 +21,7 @@ services:
     volumes:
       - ./redis/data:/data
 
+  # TODO: update CKB version
   ckb:
     image: nervos/ckb:v0.103.0
     user: root
@@ -31,6 +32,7 @@ services:
       - ./layer1/ckb:/var/lib/ckb
     command: [ "run", "-C", "/var/lib/ckb" ]
 
+  # TODO: update CKB version
   ckb-miner:
     init: true
     image: nervos/ckb:v0.103.0

docs/manual-build.md

@@ -76,7 +76,7 @@ Pulls godwoken repository into `packages/godwoken/`, builds, and then moves the
 ```shell
 MANUAL_BUILD_GODWOKEN=true \
 GODWOKEN_GIT_URL=ssh://[email protected]/godwokenrises/godwoken \
-GODWOKEN_GIT_CHECKOUT=compatibility-breaking-changes \
+GODWOKEN_GIT_CHECKOUT=develop \
 ./kicker manual-build
 ```
 

kicker

@@ -2,6 +2,15 @@
 
 set -o errexit
 
+GITHUB_ACTIONS_CLEAN_MID_ARTIFACTS=false
+# https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
+if [ "$CI" = "true" ] && [ -n "$GITHUB_RUN_ID" ]; then
+  echo "This script is running in GitHub Actions. GITHUB_RUN_ID: $GITHUB_RUN_ID"
+
+  # Diskspace is limited in GitHub Actions, so we need to manually clean the middle artifacts
+  GITHUB_ACTIONS_CLEAN_MID_ARTIFACTS=true
+fi
+
 EXECUTABLE=$0
 WORKSPACE="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 
@@ -410,41 +419,58 @@ function withdraw_v0_to_v1() {
         godwoken-v0
 }
 
-# @example MANUAL_BUILD_WEB3=true \
-#          WEB3_GIT_URL=ssh://[email protected]/godwokenrises/godwoken-web3 \
-#          WEB3_GIT_CHECKOUT=compatibility-breaking-changes \
+# @example MANUAL_BUILD_GODWOKEN=true \
+#          GODWOKEN_GIT_URL=ssh://[email protected]/godwokenrises/godwoken \
+#          GODWOKEN_GIT_CHECKOUT=develop \
 #          ./kicker manual-build
 function manual_build() {
-    DOCKER_MANUAL_BUILD_IMAGE_NAME=${DOCKER_MANUAL_BUILD_IMAGE_NAME:-"retricsu/godwoken-manual-build"}
-    DOCKER_MANUAL_BUILD_IMAGE_TAG=${DOCKER_MANUAL_BUILD_IMAGE_TAG:-"ckb2021"}
-    DOCKER_MANUAL_BUILD_IMAGE="${DOCKER_MANUAL_BUILD_IMAGE:-"$DOCKER_MANUAL_BUILD_IMAGE_NAME:$DOCKER_MANUAL_BUILD_IMAGE_TAG"}"
-
+    WEB3_GIT_URL=${WEB3_GIT_URL:-"https://github.com/godwokenrises/godwoken"}
+    WEB3_GIT_CHECKOUT=${WEB3_GIT_CHECKOUT:-"develop"}
     echo "MANUAL_BUILD_WEB3         =   \"$MANUAL_BUILD_WEB3\""
     echo "WEB3_GIT_URL              =   \"$WEB3_GIT_URL\""
     echo "WEB3_GIT_CHECKOUT         =   \"$WEB3_GIT_CHECKOUT\""
     echo "MANUAL_BUILD_WEB3_INDEXER =   \"$MANUAL_BUILD_WEB3_INDEXER\""
+
+    GODWOKEN_GIT_URL=${GODWOKEN_GIT_URL:-"https://github.com/godwokenrises/godwoken"}
+    GODWOKEN_GIT_CHECKOUT=${GODWOKEN_GIT_CHECKOUT:-"develop"}
     echo "MANUAL_BUILD_GODWOKEN     =   \"$MANUAL_BUILD_GODWOKEN\""
     echo "GODWOKEN_GIT_URL          =   \"$GODWOKEN_GIT_URL\""
     echo "GODWOKEN_GIT_CHECKOUT     =   \"$GODWOKEN_GIT_CHECKOUT\""
+
+    SCRIPTS_GIT_URL=${SCRIPTS_GIT_URL:-"https://github.com/godwokenrises/godwoken"}
+    SCRIPTS_GIT_CHECKOUT=${SCRIPTS_GIT_CHECKOUT:-"develop"}
     echo "MANUAL_BUILD_SCRIPTS      =   \"$MANUAL_BUILD_SCRIPTS\""
     echo "SCRIPTS_GIT_URL           =   \"$SCRIPTS_GIT_URL\""
     echo "SCRIPTS_GIT_CHECKOUT      =   \"$SCRIPTS_GIT_CHECKOUT\""
+
     echo "OMNI_LOCK_GIT_URL         =   \"$OMNI_LOCK_GIT_URL\""
     echo "OMNI_LOCK_GIT_CHECKOUT    =   \"$OMNI_LOCK_GIT_CHECKOUT\""
+
+    POLYJUICE_GIT_URL=${POLYJUICE_GIT_URL:-"https://github.com/godwokenrises/godwoken"}
+    POLYJUICE_GIT_CHECKOUT=${POLYJUICE_GIT_CHECKOUT:-"develop"}
     echo "MANUAL_BUILD_POLYJUICE    =   \"$MANUAL_BUILD_POLYJUICE\""
     echo "POLYJUICE_GIT_URL         =   \"$POLYJUICE_GIT_URL\""
     echo "POLYJUICE_GIT_CHECKOUT    =   \"$POLYJUICE_GIT_CHECKOUT\""
+
+    DOCKER_MANUAL_BUILD_IMAGE_NAME=${DOCKER_MANUAL_BUILD_IMAGE_NAME:-"retricsu/godwoken-manual-build"}
+    DOCKER_MANUAL_BUILD_IMAGE_TAG=${DOCKER_MANUAL_BUILD_IMAGE_TAG:-"node18"}
+    DOCKER_MANUAL_BUILD_IMAGE="${DOCKER_MANUAL_BUILD_IMAGE:-"$DOCKER_MANUAL_BUILD_IMAGE_NAME:$DOCKER_MANUAL_BUILD_IMAGE_TAG"}"
     echo "DOCKER_MANUAL_BUILD_IMAGE =   \"$DOCKER_MANUAL_BUILD_IMAGE\""
     echo
 
+    info "Start building..." | tee /tmp/kicker.log
+    du -hd6 $WORKSPACE | egrep G | tee --append /tmp/kicker.log
+
     if [ "$MANUAL_BUILD_WEB3" = "true" ]; then
         info "Start building godwoken-web3"
 
-        srcdir=$WORKSPACE/packages/godwoken-web3/web3
+        # godwoken-web3 is managed in the Godwoken monorepo.
+        # See https://github.com/godwokenrises/godwoken/tree/develop/web3
+        srcdir=$WORKSPACE/packages/godwoken/web3
         dstdir=$WORKSPACE/docker/manual-artifacts/godwoken-web3
 
         # Download repo
-        prepare_repo godwoken-web3 "$WEB3_GIT_URL" "$WEB3_GIT_CHECKOUT"
+        prepare_repo godwoken "$WEB3_GIT_URL" "$WEB3_GIT_CHECKOUT"
 
         # Yarn install via docker (assumes the "web3" docker-compose service installed yarn)
 
@@ -469,38 +495,48 @@ function manual_build() {
             $srcdir/node_modules \
             $srcdir/yarn.lock \
             $dstdir
+
+        if [[ "$GITHUB_ACTIONS_CLEAN_MID_ARTIFACTS" = "true" ]]; then
+            erun sudo rm -rf $srcdir/node_modules
+        fi
     else
         info "skip building godwoken-web3"
     fi
 
     if [ "$MANUAL_BUILD_WEB3_INDEXER" = "true" ]; then
         info "Start building godwoken-web3-indexer"
 
-        srcdir=$WORKSPACE/packages/godwoken-web3
+        srcdir=$WORKSPACE/packages/godwoken
         dstdir=$WORKSPACE/docker/manual-artifacts
 
         # Download repo
-        prepare_repo godwoken-web3 "$WEB3_GIT_URL" "$WEB3_GIT_CHECKOUT"
+        prepare_repo godwoken "$WEB3_GIT_URL" "$WEB3_GIT_CHECKOUT"
 
         # Cargo fetch Rust dependencies (in order to access network via
         # host network). The docker image must have installed cargo, molecule
         # and rustfmt.
-        erun "cd $srcdir/web3 && CARGO_HOME=$srcdir/web3/.cargo cargo fetch --locked && cd -"
-        erun docker run \
-            --rm \
-            --env CARGO_HOME=/app/.cargo \
+        erun "cd $srcdir/web3 && CARGO_HOME=$WORKSPACE/packages/.cargo cargo fetch --locked && cd -"
+
+        erun docker run --rm \
+            --workdir /app/web3 \
             --volume $srcdir:/app \
             --volume $WORKSPACE/packages/.rustup:/root/.rustup \
-            --workdir /app/web3 \
-            retricsu/godwoken-manual-build:ckb2021 cargo build --locked --release
+            --volume $WORKSPACE/packages/.cargo:/app/web3/.cargo \
+            --env CARGO_HOME=/app/web3/.cargo \
+            $DOCKER_MANUAL_BUILD_IMAGE cargo build --release --locked
 
         # Copy the built artifacts to `docker/manual-artifacts/gw-web3-indexer`
         #
         # More: ./docker/manual-web3-indexer.compose.yml
         erun mkdir -p $dstdir
         erun cp $srcdir/web3/target/release/gw-web3-indexer $dstdir
+
+        # Remove Rust target to avoid `out of disk` issue in CI
+        if [[ "$GITHUB_ACTIONS_CLEAN_MID_ARTIFACTS" = "true" ]]; then
+            sudo rm -rf $srcdir/web3/target
+        fi
     else
-        info "skip building godwoken-web3-indexer(gw-web3-indexer)"
+        info "skip building godwoken-web3-indexer (gw-web3-indexer)"
     fi
 
     if [ "$MANUAL_BUILD_GODWOKEN" = "true" ]; then
@@ -514,19 +550,21 @@ function manual_build() {
 
         # Cargo fetch Rust dependencies (in order to access network via
         # host network).
+        erun "cd $srcdir && CARGO_HOME=$WORKSPACE/packages/.cargo cargo fetch --locked && cd -"
+
         # Note:
         # 1. The docker image must have installed cargo and molecule
         # 2. The builtin consensus config and binaries are not required on a 
         #    Godwoken devnet, so `--features gw-config/no-builtin` is appended
         #    to `cargo build` arguments.
-        erun docker run \
-            --rm \
-            --env CARGO_HOME=/app/.cargo \
+        erun docker run --rm \
+            --workdir /app \
             --volume $srcdir:/app \
             --volume $WORKSPACE/packages/.rustup:/root/.rustup \
-            --workdir /app \
-            retricsu/godwoken-manual-build:ckb2021 \
-            cargo build --locked --release --features gw-config/no-builtin
+            --volume $WORKSPACE/packages/.cargo:/app/.cargo \
+            --env CARGO_HOME=/app/.cargo \
+            $DOCKER_MANUAL_BUILD_IMAGE \
+            cargo build --release --locked --features gw-config/no-builtin
 
         # Copy the built artifacts to `docker/manual-artifacts/`
         #
@@ -535,10 +573,10 @@ function manual_build() {
         erun cp $srcdir/target/release/godwoken $dstdir
         erun cp $srcdir/target/release/gw-tools $dstdir
 
-        # Remove target/release/examples to avoid `out of disk` issue in CI
-        du -hd1 $srcdir/target
-        rm -rf $srcdir/target/release/examples
-        du -hd1 $srcdir/target
+        # Remove Rust target to avoid `out of disk` issue in CI
+        if [[ "$GITHUB_ACTIONS_CLEAN_MID_ARTIFACTS" = "true" ]]; then
+            sudo rm -rf $srcdir/target
+        fi
     else
         info "skip building Godwoken"
     fi
@@ -548,7 +586,7 @@ function manual_build() {
         dstdir=$WORKSPACE/docker/manual-artifacts/polyjuice/
 
         # Download repo
-        prepare_repo godwoken-polyjuice "$POLYJUICE_GIT_URL" "$POLYJUICE_GIT_CHECKOUT"
+        prepare_repo godwoken "$POLYJUICE_GIT_URL" "$POLYJUICE_GIT_CHECKOUT"
 
         cd $srcdir
         erun make all-via-docker
@@ -565,13 +603,16 @@ function manual_build() {
         dstdir=$WORKSPACE/docker/manual-artifacts/scripts/
 
         # Download repo
-        prepare_repo godwoken-scripts "$SCRIPTS_GIT_URL" "$SCRIPTS_GIT_CHECKOUT"
+        prepare_repo godwoken "$SCRIPTS_GIT_URL" "$SCRIPTS_GIT_CHECKOUT"
 
         # Install capsule
-        # TODO: use cpasule from godwoken-manual-build image
-        if [ -z "$(command -v capsule)" ]; then
-            erun cargo install ckb-capsule
-        fi
+        # FIXME: error: Incompatible capsule version 0.10.1, this project requires a version that's compatible with 0.7.0
+        CAPSULE_VERSION=0.7.0
+        export CARGO_HOME=$WORKSPACE/packages/.cargo
+        (which capsule && test "$(capsule --version)" = "Capsule $CAPSULE_VERSION") \
+        || erun cargo install ckb-capsule --version $CAPSULE_VERSION --force
+        $WORKSPACE/packages/.cargo/bin/capsule --version
+        # FIXME: failed to install capsule 0.7.0
 
         erun cd $srcdir/c \&\& erun make all-via-docker
         erun cd $srcdir \&\& capsule build --release --debug-output
@@ -584,14 +625,23 @@ function manual_build() {
 
         # Copy the prebuild omni-lock to `docker/manual-artifacts/`
         erun $DOCKER_COMPOSE -f $WORKSPACE/docker/docker-compose.yml run \
-            --rm \
-            --no-deps \
+            --rm --no-deps \
             --volume=$dstdir:/godwoken-scripts \
             --entrypoint "\"bash -c 'cp /scripts/godwoken-scripts/omni_lock /godwoken-scripts/omni_lock'\"" \
             godwoken
     else
         info "skip building Scripts"
     fi
+
+    if [[ "$GITHUB_ACTIONS_CLEAN_MID_ARTIFACTS" = "true" ]]; then
+        docker images --format '{{.Repository}}:{{.Tag}}' \
+        | grep '$DOCKER_MANUAL_BUILD_IMAGE' | xargs docker rmi \
+        && echo "Image removed." \
+        || echo "Image does not exist."
+    fi
+
+    info "End building." | tee --append /tmp/kicker.log
+    du -hd6 $WORKSPACE | egrep G | tee --append /tmp/kicker.log
 }
 
 function prepare_repo() {