Merge branch 'master' into check_epoch_state_table

IntersectMBO · Dec 2, 2024 · db50e75 · db50e75
2 parents 0ef1deb + 76dbef6
commit db50e75
Show file tree

Hide file tree

Showing 291 changed files with 9,087 additions and 11,427 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -17,3 +17,6 @@ cardano_node_tests/utils/dbsync_*  @mkoura @saratomaz @ArturWieczorek
 .github/workflows/db_sync_*        @ArturWieczorek
 .github/workflows/node_sync_*      @ArturWieczorek
 .github/workflows/smash_*          @ArturWieczorek
+
+.buildkite/db_sync_*               @ArturWieczorek
+.buildkite/node_sync_*             @ArturWieczorek
diff --git a/.github/env_nightly_dbsync b/.github/env_nightly_dbsync
@@ -2,6 +2,7 @@ CLUSTER_ERA=conway
 COMMAND_ERA=conway
 MARKEXPR=dbsync
 CLUSTERS_COUNT=4
-DBSYNC_REV=13.5.0.0
-DBSYNC_TAR_URL=https://github.com/IntersectMBO/cardano-db-sync/releases/download/13.5.0.0/cardano-db-sync-13.5.0.0-linux.tar.gz
+ENABLE_LEGACY=true
+DBSYNC_REV=13.6.0.1
+DBSYNC_TAR_URL=https://github.com/IntersectMBO/cardano-db-sync/releases/download/13.6.0.1/cardano-db-sync-13.6.0.1-linux.tar.gz
 DBSYNC_SKIP_INDEXES=true
diff --git a/.github/env_nightly_dbsync_pv10 b/.github/env_nightly_dbsync_pv10
@@ -1,9 +1,8 @@
 CLUSTER_ERA=conway
 COMMAND_ERA=conway
 PV10=true
-ENABLE_P2P=true
 MARKEXPR=dbsync
 CLUSTERS_COUNT=4
-DBSYNC_REV=13.5.0.0
-DBSYNC_TAR_URL=https://github.com/IntersectMBO/cardano-db-sync/releases/download/13.5.0.0/cardano-db-sync-13.5.0.0-linux.tar.gz
+DBSYNC_REV=13.6.0.1
+DBSYNC_TAR_URL=https://github.com/IntersectMBO/cardano-db-sync/releases/download/13.6.0.1/cardano-db-sync-13.6.0.1-linux.tar.gz
 DBSYNC_SKIP_INDEXES=true
diff --git a/.github/env_nightly_pv10 b/.github/env_nightly_pv10
@@ -1,4 +1,3 @@
 CLUSTER_ERA=conway
 COMMAND_ERA=conway
 PV10=true
-ENABLE_P2P=true
diff --git a/.github/env_nightly_upgrade b/.github/env_nightly_upgrade
@@ -1,2 +1,2 @@
-BASE_TAR_URL=https://github.com/IntersectMBO/cardano-node/releases/download/9.1.1/cardano-node-9.1.1-linux.tar.gz
+BASE_TAR_URL=https://github.com/IntersectMBO/cardano-node/releases/download/10.1.2/cardano-node-10.1.2-linux.tar.gz
 CI_BYRON_CLUSTER=true
diff --git a/.github/grep_errors.sh b/.github/grep_errors.sh
@@ -4,6 +4,6 @@ ARTIFACTS_DIR="${ARTIFACTS_DIR:-".artifacts"}"
 ERR_LOGFILE="$PWD/errors_all.log"
 
 # shellcheck disable=SC2012
-pushd "$ARTIFACTS_DIR" || { echo "Cannot switch to $ARTIFACTS_DIR"; ls -1a "$ARTIFACTS_DIR"; exit 1; } > "$ERR_LOGFILE"
+pushd "$ARTIFACTS_DIR" > /dev/null || { echo "Cannot switch to $ARTIFACTS_DIR"; ls -1a "$ARTIFACTS_DIR"; exit 1; } > "$ERR_LOGFILE"
 grep -r --include "*.stdout" --include "*.stderr" -Ei ":error:|failed|failure" . > "$ERR_LOGFILE"
-popd || exit 1
+popd > /dev/null || exit 1
diff --git a/.github/node_upgrade.sh b/.github/node_upgrade.sh
@@ -6,7 +6,7 @@
 # BASE_REVISION - revision of cardano-node to upgrade from (alternative to BASE_TAR_URL)
 # UPGRADE_REVISION - revision of cardano-node to upgrade to
 
-set -xeuo pipefail
+set -euo pipefail
 
 if [[ -z "${BASE_TAR_URL:-""}" && -z "${BASE_REVISION:-""}" ]]; then
   echo "BASE_TAR_URL or BASE_REVISION must be set"
@@ -53,7 +53,15 @@ rm -rf "${COVERAGE_DIR:?}"
 mkdir -p "$COVERAGE_DIR"
 
 export SCHEDULING_LOG=scheduling.log
-true > "$SCHEDULING_LOG"
+: > "$SCHEDULING_LOG"
+
+export DEV_CLUSTER_RUNNING=1 CLUSTERS_COUNT=1 FORBID_RESTART=1 TEST_THREADS=10 NUM_POOLS="${NUM_POOLS:-4}"
+unset ENABLE_LEGACY MIXED_P2P
+
+echo "::endgroup::"  # end group for "Script setup"
+
+echo "::group::Nix env setup step1"
+printf "start: %(%H:%M:%S)T\n" -1
 
 # shellcheck disable=SC1090,SC1091
 . .github/nix_override_cardano_node.sh
@@ -70,26 +78,21 @@ else
   NODE_OVERRIDE=$(node_override)
 fi
 
-export DEV_CLUSTER_RUNNING=1 CLUSTERS_COUNT=1 FORBID_RESTART=1 TEST_THREADS=10 NUM_POOLS="${NUM_POOLS:-4}"
-unset ENABLE_P2P MIXED_P2P
-
-echo "::group::Nix env setup"
-printf "start: %(%H:%M:%S)T\n" -1
-
 set +e
 # shellcheck disable=SC2086
 nix flake update --accept-flake-config $NODE_OVERRIDE
 # shellcheck disable=SC2016
 nix develop --accept-flake-config .#venv --command bash -c '
   : > "$WORKDIR/.nix_step1"
-  printf "finish: %(%H:%M:%S)T\n" -1
-  echo "::endgroup::"  # end group for "Nix env setup"
+  echo "::endgroup::"  # end group for "Nix env setup step1"
 
-  echo "::group::Python venv setup"
+  echo "::group::Python venv setup step1"
+  printf "start: %(%H:%M:%S)T\n" -1
   . .github/setup_venv.sh clean
-  echo "::endgroup::"  # end group for "Python venv setup"
+  echo "::endgroup::"  # end group for "Python venv setup step1"
 
-  echo "::group::Pytest step1"
+  echo "::group::🧪 Testrun Step1"
+  printf "start: %(%H:%M:%S)T\n" -1
   df -h .
   # prepare scripts for stating cluster instance, start cluster instance, run smoke tests
   ./.github/node_upgrade_pytest.sh step1
@@ -104,8 +107,10 @@ fi
 # retval 0 == all tests passed; 1 == some tests failed; > 1 == some runtime error and we don't want to continue
 [ "$retval" -le 1 ] || exit "$retval"
 
-echo "::endgroup::"  # end group for "Pytest step1"
-echo "::group::Pytest step2"
+echo "::endgroup::"  # end group for "Testrun Step1"
+
+echo "::group::Nix env setup steps 2 & 3"
+printf "start: %(%H:%M:%S)T\n" -1
 
 # update cardano-node to specified branch and/or revision, or to the latest available revision
 if [ -n "${UPGRADE_REVISION:-""}" ]; then
@@ -119,35 +124,40 @@ nix flake update --accept-flake-config $NODE_OVERRIDE
 # shellcheck disable=SC2016
 nix develop --accept-flake-config .#venv --command bash -c '
   : > "$WORKDIR/.nix_step2"
-  df -h .
+  echo "::endgroup::"  # end group for "Nix env setup steps 2 & 3"
 
-  echo "::group::Python venv setup"
+  echo "::group::Python venv setup steps 2 & 3"
+  printf "start: %(%H:%M:%S)T\n" -1
   . .github/setup_venv.sh clean
-  echo "::endgroup::"  # end group for "Python venv setup"
+  echo "::endgroup::"  # end group for "Python venv setup steps 2 & 3"
 
+  echo "::group::🧪 Testrun Step2"
+  printf "start: %(%H:%M:%S)T\n" -1
+  df -h .
   # update cluster nodes, run smoke tests
   ./.github/node_upgrade_pytest.sh step2
   retval="$?"
   # retval 0 == all tests passed; 1 == some tests failed; > 1 == some runtime error and we dont want to continue
   [ "$retval" -le 1 ] || exit "$retval"
-  echo "::endgroup::"  # end group for "Pytest step2"
+  echo "::endgroup::"  # end group for "Testrun Step2"
 
-  echo "::group::Pytest step3"
+  echo "::group::🧪 Testrun Step3"
+  printf "start: %(%H:%M:%S)T\n" -1
   df -h .
   # update to Conway, run smoke tests
   ./.github/node_upgrade_pytest.sh step3
   retval="$?"
-  echo "::endgroup::"  # end group for "Pytest step3"
+  df -h .
+  echo "::endgroup::"  # end group for "Testrun Step3"
 
-  echo "::group::Cluster teardown & artifacts"
+  echo "::group::Teardown cluster & collect artifacts"
+  printf "start: %(%H:%M:%S)T\n" -1
   # teardown cluster
   ./.github/node_upgrade_pytest.sh finish
   exit $retval
 '
 retval="$?"
 
-df -h .
-
 if [ ! -e "$WORKDIR/.nix_step2" ]; then
   echo "Nix env setup failed, exiting"
   exit 1
@@ -173,6 +183,4 @@ if [ -n "${GITHUB_ACTIONS:-""}" ]; then
   ls -1a
 fi
 
-echo "::endgroup::" # end group for "Cluster teardown & artifacts"
-
 exit "$retval"
diff --git a/.github/node_upgrade_pytest.sh b/.github/node_upgrade_pytest.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -xuo pipefail
+set -uo pipefail
 
 retval=1
 
@@ -12,6 +12,9 @@ export COMMAND_ERA="$CLUSTER_ERA"
 CLUSTER_SCRIPTS_DIR="$WORKDIR/cluster0_${CLUSTER_ERA}"
 STATE_CLUSTER="${CARDANO_NODE_SOCKET_PATH_CI%/*}"
 
+NETWORK_MAGIC="$(jq '.networkMagic' "$STATE_CLUSTER/shelley/genesis.json")"
+export NETWORK_MAGIC
+
 # init dir for step1 binaries
 STEP1_BIN="$WORKDIR/step1-bin"
 mkdir -p "$STEP1_BIN"
@@ -29,6 +32,7 @@ if [ "$1" = "step1" ]; then
   printf "STEP1 start: %(%H:%M:%S)T\n" -1
 
   export UPGRADE_TESTS_STEP=1
+  export ENABLE_LEGACY=1
 
   if [ -n "${BASE_TAR_URL:-""}" ]; then
     # download and extract base revision binaries
@@ -68,15 +72,14 @@ if [ "$1" = "step1" ]; then
 
   # backup the original genesis files
   cp -f "$STATE_CLUSTER/shelley/genesis.alonzo.json" "$STATE_CLUSTER/shelley/genesis.alonzo.step1.json"
-  if [ -e "$STATE_CLUSTER/shelley/genesis.conway.json" ]; then
-    cp -f "$STATE_CLUSTER/shelley/genesis.conway.json" "$STATE_CLUSTER/shelley/genesis.conway.step1.json"
-  fi
+  cp -f "$STATE_CLUSTER/shelley/genesis.conway.json" "$STATE_CLUSTER/shelley/genesis.conway.step1.json"
 
   # run smoke tests
+  printf "STEP1 tests: %(%H:%M:%S)T\n" -1
   pytest \
     cardano_node_tests \
     -n "$TEST_THREADS" \
-    -m "smoke or upgrade" \
+    -m "smoke or upgrade_step1" \
     --artifacts-base-dir="$ARTIFACTS_DIR" \
     --cli-coverage-dir="$COVERAGE_DIR" \
     --alluredir="$REPORTS_DIR" \
@@ -102,6 +105,8 @@ elif [ "$1" = "step2" ]; then
   printf "STEP2 start: %(%H:%M:%S)T\n" -1
 
   export UPGRADE_TESTS_STEP=2
+  export MIXED_P2P=1
+  unset ENABLE_LEGACY
 
   # Setup `cardano-cli` binary
   if [ -n "${UPGRADE_CLI_REVISION:-""}" ]; then
@@ -116,33 +121,35 @@ elif [ "$1" = "step2" ]; then
 
   # generate config and topology files for the "mixed" mode
   CARDANO_NODE_SOCKET_PATH="$WORKDIR/dry_mixed/state-cluster0/bft1.socket" \
-    MIXED_P2P=1 \
     DRY_RUN=1 \
     "$CLUSTER_SCRIPTS_DIR/start-cluster"
 
-  # hashes of old and new Conway genesis files
-  CONWAY_GENESIS_HASH="$(jq -r ".ConwayGenesisHash" "$WORKDIR/dry_mixed/state-cluster0/config-bft1.json")"
-  CONWAY_GENESIS_STEP1_HASH=""
-  if [ -e "$STATE_CLUSTER/shelley/genesis.conway.json" ]; then
-    CONWAY_GENESIS_STEP1_HASH="$(jq -r ".ConwayGenesisHash" "$STATE_CLUSTER/config-bft1.json")"
-  fi
-
-  # hashes of old and new Alonzo genesis files
-  ALONZO_GENESIS_HASH="$(jq -r ".AlonzoGenesisHash" "$WORKDIR/dry_mixed/state-cluster0/config-bft1.json")"
-  ALONZO_GENESIS_STEP1_HASH="$(jq -r ".AlonzoGenesisHash" "$STATE_CLUSTER/config-bft1.json")"
-
-  # use the original genesis files
-  BYRON_GENESIS_HASH="$(jq -r ".ByronGenesisHash" "$STATE_CLUSTER/config-bft1.json")"
-  SHELLEY_GENESIS_HASH="$(jq -r ".ShelleyGenesisHash" "$STATE_CLUSTER/config-bft1.json")"
-
   # copy newly generated topology files to the cluster state dir
   cp -f "$WORKDIR"/dry_mixed/state-cluster0/topology-*.json "$STATE_CLUSTER"
 
-  # copy newly generated Alonzo genesis to the cluster state dir
-  cp -f "$WORKDIR/dry_mixed/state-cluster0/shelley/genesis.alonzo.json" "$STATE_CLUSTER/shelley"
+  if [ -n "${REPLACE_GENESIS_STEP2:-""}" ]; then
+    # Copy newly generated Alonzo genesis to the cluster state dir
+    cp -f "$WORKDIR/dry_mixed/state-cluster0/shelley/genesis.alonzo.json" "$STATE_CLUSTER/shelley"
+
+    # Copy newly generated Conway genesis file to the cluster state dir, use committee members from the original
+    # Conway genesis.
+    jq \
+      --argfile src "$STATE_CLUSTER/shelley/genesis.conway.step1.json" \
+      '.committee.members = $src.committee.members' \
+      "$WORKDIR/dry_mixed/state-cluster0/shelley/genesis.conway.json" > "$STATE_CLUSTER/shelley/genesis.conway.json"
+  fi
 
-  # copy newly generated Conway genesis file to the cluster state dir
-  cp -f "$WORKDIR/dry_mixed/state-cluster0/shelley/genesis.conway.json" "$STATE_CLUSTER/shelley"
+  # use the original shelley and byron genesis files
+  BYRON_GENESIS_HASH="$(jq -r ".ByronGenesisHash" "$STATE_CLUSTER/config-bft1.json")"
+  SHELLEY_GENESIS_HASH="$(jq -r ".ShelleyGenesisHash" "$STATE_CLUSTER/config-bft1.json")"
+  # hashes of the original alonzo and conway genesis files
+  CONWAY_GENESIS_STEP1_HASH="$(jq -r ".ConwayGenesisHash" "$STATE_CLUSTER/config-bft1.json")"
+  ALONZO_GENESIS_STEP1_HASH="$(jq -r ".AlonzoGenesisHash" "$STATE_CLUSTER/config-bft1.json")"
+  # hashes of genesis files that were potentially replaced
+  ALONZO_GENESIS_HASH="$(cardano-cli legacy genesis hash --genesis \
+    "$STATE_CLUSTER/shelley/genesis.alonzo.json")"
+  CONWAY_GENESIS_HASH="$(cardano-cli legacy genesis hash --genesis \
+    "$STATE_CLUSTER/shelley/genesis.conway.json")"
 
   # copy newly generated config files to the cluster state dir
   for conf in "$WORKDIR"/dry_mixed/state-cluster0/config-*.json; do
@@ -184,6 +191,8 @@ elif [ "$1" = "step2" ]; then
 
   # Restart local cluster nodes with binaries from new cluster-node version.
   # It is necessary to restart supervisord with new environment.
+  "$STATE_CLUSTER/supervisorctl" stop all
+  sleep 5
   "$STATE_CLUSTER/supervisord_stop"
   sleep 3
   "$STATE_CLUSTER/supervisord_start" || exit 6
@@ -204,19 +213,13 @@ elif [ "$1" = "step2" ]; then
     exit 6
   fi
 
-  # waiting for node to fully start
-  for _ in {1..10}; do
-    if [ -S "$CARDANO_NODE_SOCKET_PATH" ]; then
-      break
-    fi
-    sleep 5
-  done
-  [ -S "$CARDANO_NODE_SOCKET_PATH" ] || { echo "Failed to start node" >&2; exit 6; }  # assert
+  # Tx submission delay
+  sleep 60
 
   # waiting to make sure the chain is synced
-  NETWORK_MAGIC="$(jq '.networkMagic' "$STATE_CLUSTER/shelley/genesis.json")"
   for _ in {1..10}; do
-    sync_progress="$(cardano-cli query tip --testnet-magic "$NETWORK_MAGIC" | jq -r '.syncProgress')"
+    sync_progress="$(cardano-cli latest query tip \
+      --testnet-magic "$NETWORK_MAGIC" | jq -r '.syncProgress')"
     if [ "$sync_progress" = "100.00" ]; then
       break
     fi
@@ -228,11 +231,15 @@ elif [ "$1" = "step2" ]; then
   pytest cardano_node_tests/tests/test_node_upgrade.py -k test_ignore_log_errors
   err_retval="$?"
 
+  # Update PlutusV3 cost models.
+  pytest cardano_node_tests/tests/test_node_upgrade.py -k test_update_cost_models || exit 6
+
   # run smoke tests
+  printf "STEP2 tests: %(%H:%M:%S)T\n" -1
   pytest \
     cardano_node_tests \
     -n "$TEST_THREADS" \
-    -m "smoke or upgrade" \
+    -m "smoke or upgrade_step2" \
     --artifacts-base-dir="$ARTIFACTS_DIR" \
     --cli-coverage-dir="$COVERAGE_DIR" \
     --alluredir="$REPORTS_DIR" \
@@ -260,6 +267,7 @@ elif [ "$1" = "step3" ]; then
   printf "STEP3 start: %(%H:%M:%S)T\n" -1
 
   export UPGRADE_TESTS_STEP=3
+  unset ENABLE_LEGACY MIXED_P2P
 
   # Setup `cardano-cli` binary
   if [ -n "${UPGRADE_CLI_REVISION:-""}" ]; then
@@ -274,7 +282,6 @@ elif [ "$1" = "step3" ]; then
 
   # generate config and topology files for p2p mode
   CARDANO_NODE_SOCKET_PATH="$WORKDIR/dry_p2p/state-cluster0/bft1.socket" \
-    ENABLE_P2P=1 \
     DRY_RUN=1 \
     "$CLUSTER_SCRIPTS_DIR/start-cluster"
 
@@ -320,15 +327,34 @@ elif [ "$1" = "step3" ]; then
     exit 6
   fi
 
+  # Tx submission delay
+  sleep 60
+
+  # waiting to make sure the chain on pool3 is synced
+  for _ in {1..10}; do
+    sync_progress="$(cardano-cli latest query tip \
+      --testnet-magic "$NETWORK_MAGIC" \
+      --socket-path "${STATE_CLUSTER}/pool3.socket" | jq -r '.syncProgress')"
+    if [ "$sync_progress" = "100.00" ]; then
+      break
+    fi
+    sleep 5
+  done
+  [ "$sync_progress" = "100.00" ] || { echo "Failed to sync node" >&2; exit 6; }  # assert
+
   # Test for ignoring expected errors in log files. Run separately to make sure it runs first.
   pytest cardano_node_tests/tests/test_node_upgrade.py -k test_ignore_log_errors
   err_retval="$?"
 
+  # Hard fork to PV10.
+  pytest cardano_node_tests/tests/test_node_upgrade.py -k test_hardfork || exit 6
+
   # Run smoke tests
+  printf "STEP3 tests: %(%H:%M:%S)T\n" -1
   pytest \
     cardano_node_tests \
     -n "$TEST_THREADS" \
-    -m "smoke or upgrade" \
+    -m "smoke or upgrade_step3" \
     --artifacts-base-dir="$ARTIFACTS_DIR" \
     --cli-coverage-dir="$COVERAGE_DIR" \
     --alluredir="$REPORTS_DIR" \