Skip to content

Commit

Permalink
Change checkpoint urls and add op/base chains for each commit sync te…
Browse files Browse the repository at this point in the history 
…sts (NethermindEth#7563)
  • Loading branch information
@kamilchodola
kamilchodola authored Oct 16, 2024
1 parent de191c0 commit 5ce06bd
Show file tree
Hide file tree
Showing 2 changed files with 167 additions and 30 deletions.
145 changes: 117 additions & 28 deletions .github/workflows/sync-supported-chains.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ jobs:
needs: [setup-matrix]
strategy:
fail-fast: false
max-parallel: 5
matrix:
config: ${{fromJson(needs.setup-matrix.outputs.matrix)}}
runs-on: ubuntu-latest
Expand All @@ -52,13 +53,14 @@ jobs:
linode_token: ${{ secrets.LINODE_TOKEN }}
github_token: "${{ secrets.REPOSITORY_DISPATCH_TOKEN }}"
action: "create"
runner_label: ${{ matrix.config.network }}-${{ matrix.config.cl }}-${{ github.run_id }}
runner_label: t-${{ github.run_id }}-${{ matrix.config.network }}
root_password: ${{ secrets.LINODE_ROOT_PASSWORD }}
machine_type: "${{ matrix.config.agent }}"
image: "linode/ubuntu24.04"
tags: "core, self-hosted, dynamic"
organization: "NethermindEth"
repo_name: "nethermind"
blocked_ports: "8545,8546,8551,8552"

sync-chain:
needs: [setup-matrix, create_a_runner]
Expand All @@ -68,7 +70,7 @@ jobs:
config: ${{fromJson(needs.setup-matrix.outputs.matrix)}}

name: "Run sync of ${{ matrix.config.network }} chain"
runs-on: ${{ matrix.config.network }}-${{ matrix.config.cl }}-${{ github.run_id }}
runs-on: t-${{ github.run_id }}-${{ matrix.config.network }}
timeout-minutes: ${{ matrix.config.timeout }}

steps:
Expand Down Expand Up @@ -102,6 +104,7 @@ jobs:
working-directory: sedge
run: |
docker_image=""
network="${{ matrix.config.network }}"
if [ -z "${{ inputs.nethermind_image }}" ]; then
REF_NAME=${{ github.ref }}
Expand All @@ -117,37 +120,89 @@ jobs:
echo 'Generating sedge docker...'
./build/sedge deps install
./build/sedge generate --logging none -p $GITHUB_WORKSPACE/sedge \
full-node --map-all --no-mev-boost --no-validator --network ${{ matrix.config.network }} \
-c ${{ matrix.config.cl }}:${{ matrix.config.cl_image }} -e nethermind:$docker_image \
--el-extra-flag Sync.NonValidatorNode=true --el-extra-flag Sync.DownloadBodiesInFastSync=false \
--el-extra-flag Sync.DownloadReceiptsInFastSync=false \
--el-extra-flag JsonRpc.EnabledModules=[Eth,Subscribe,Trace,TxPool,Web3,Personal,Proof,Net,Parity,Health,Rpc,Debug] \
--el-extra-flag Sync.SnapSync=true \
--checkpoint-sync-url=${{ matrix.config.checkpoint-sync-url }}
if [[ "$network" == base-* || "$network" == op-* ]]; then
if [[ "$network" == *mainnet* ]]; then
CONSENSUS_URL="${{ secrets.MAINNET_CONSENSUS_URL }}"
EXECUTION_URL="${{ secrets.MAINNET_EXECUTION_URL }}"
elif [[ "$network" == *sepolia* ]]; then
CONSENSUS_URL="${{ secrets.SEPOLIA_CONSENSUS_URL }}"
EXECUTION_URL="${{ secrets.SEPOLIA_EXECUTION_URL }}"
else
echo "Unknown network"
exit 1
fi
if [[ "$network" == base-* ]]; then
extra_param="--base"
fi
stripped_network="${network#base-}"
stripped_network="${stripped_network#op-}"
./build/sedge generate \
--logging none \
-p $GITHUB_WORKSPACE/sedge \
op-full-node \
--op-execution opnethermind:$docker_image \
--op-image op-node:us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:latest \
--map-all \
--network $stripped_network \
--consensus-url $CONSENSUS_URL \
--execution-api-url $EXECUTION_URL \
$extra_param
else
./build/sedge generate \
--logging none \
-p $GITHUB_WORKSPACE/sedge \
full-node \
-c ${{ matrix.config.cl }}:${{ matrix.config.cl_image }} \
-e nethermind:$docker_image \
--map-all \
--no-mev-boost \
--no-validator \
--network ${{ matrix.config.network }} \
--el-extra-flag Sync.NonValidatorNode=true \
--el-extra-flag Sync.DownloadBodiesInFastSync=false \
--el-extra-flag Sync.DownloadReceiptsInFastSync=false \
--el-extra-flag JsonRpc.EnabledModules=[Eth,Subscribe,Trace,TxPool,Web3,Personal,Proof,Net,Parity,Health,Rpc,Debug] \
--el-extra-flag Sync.SnapSync=true \
--checkpoint-sync-url=${{ matrix.config.checkpoint-sync-url }}
fi
echo 'Running sedge...'
./build/sedge run -p $GITHUB_WORKSPACE/sedge
- name: Wait for ${{ matrix.config.network }} to sync
id: wait
timeout-minutes: 180
run: |
declare -A bad_logs
declare -A good_logs
declare -A required_count
bad_logs["Corrupt"]=1
bad_logs["Exception"]=1
declare -A good_logs
good_logs["Synced Chain Head"]=0
good_logs["Processed"]=0

declare -A required_count
required_count["Synced Chain Head"]=20
required_count["Processed"]=20
required_count["Processed"]=20
network="${{ matrix.config.network }}"
if [[ "$network" != "joc-mainnet" && "$network" != "joc-testnet" ]]; then
good_logs["Synced Chain Head"]=0
required_count["Synced Chain Head"]=20
fi
counter=0
found_bad_log=false

docker logs -f sedge-execution-client | while read -r line; do
if [[ "$network" == base-* || "$network" == op-* ]]; then
container_name="sedge-execution-op-l2-client"
else
container_name="sedge-execution-client"
fi
docker logs -f "$container_name" | while read -r line; do
echo "$line"
if [[ "$line" == *"All done"* ]]; then
Expand Down Expand Up @@ -195,9 +250,32 @@ jobs:
done

- name: Get Consensus Logs
if: always()
if: always() && matrix.config.network != 'joc-mainnet' && matrix.config.network != 'joc-testnet'
run: |
docker logs sedge-consensus-client
network="${{ matrix.config.network }}"
if [[ "$network" == base-* || "$network" == op-* ]]; then
docker logs sedge-consensus-op-l2-client
else
docker logs sedge-consensus-client
fi
- name: Check size of DB
run: |
du -h $GITHUB_WORKSPACE/sedge
- name: Destroy VM
if: always()
id: run-linode-action
uses: kamilchodola/linode-github-runner/.github/actions/linode-machine-manager@main
with:
linode_token: ${{ secrets.LINODE_TOKEN }}
github_token: "${{ secrets.REPOSITORY_DISPATCH_TOKEN }}"
action: "destroy-machine"
runner_label: t-${{ github.run_id }}-${{ matrix.config.network }}
search_phrase: t-${{ github.run_id }}-${{ matrix.config.network }}
root_password: ${{ secrets.LINODE_ROOT_PASSWORD }}
organization: "NethermindEth"
repo_name: "nethermind"

destroy_runner:
needs: [setup-matrix, create_a_runner, sync-chain]
Expand All @@ -208,19 +286,30 @@ jobs:
config: ${{fromJson(needs.setup-matrix.outputs.matrix)}}
runs-on: ubuntu-latest
steps:
- name: Destroy a Runner
id: run-linode-action
- name: Destroy VM (make sure is removed if by any unexpected reason it did not removed it on )
continue-on-error: true
uses: kamilchodola/linode-github-runner/.github/actions/linode-machine-manager@main
with:
linode_token: ${{ secrets.LINODE_TOKEN }}
github_token: "${{ secrets.REPOSITORY_DISPATCH_TOKEN }}"
action: "destroy-machine"
runner_label: t-${{ github.run_id }}-${{ matrix.config.network }}
search_phrase: t-${{ github.run_id }}-${{ matrix.config.network }}
root_password: ${{ secrets.LINODE_ROOT_PASSWORD }}
organization: "NethermindEth"
repo_name: "nethermind"

- name: Destroy Runner
uses: kamilchodola/linode-github-runner/.github/actions/linode-machine-manager@main
with:
linode_token: ${{ secrets.LINODE_TOKEN }}
github_token: "${{ secrets.REPOSITORY_DISPATCH_TOKEN }}"
action: "destroy"
runner_label: ${{ matrix.config.network }}-${{ matrix.config.cl }}-${{ github.run_id }}
search_phrase: ${{ matrix.config.network }}-${{ matrix.config.cl }}-${{ github.run_id }}
action: "destroy-runner"
runner_label: t-${{ github.run_id }}-${{ matrix.config.network }}
search_phrase: t-${{ github.run_id }}-${{ matrix.config.network }}
root_password: ${{ secrets.LINODE_ROOT_PASSWORD }}
organization: "NethermindEth"
repo_name: "nethermind"

trigger_tests:
needs: [setup-matrix, create_a_runner, sync-chain, destroy_runner]
if: success()
Expand Down
52 changes: 50 additions & 2 deletions scripts/workflow_config/sync_testnets_matrix.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
"network": "sepolia",
"cl": "prysm",
"cl_image": "prysmaticlabs/prysm-beacon-chain:stable",
"checkpoint-sync-url": "https://beaconstate-sepolia.chainsafe.io",
"checkpoint-sync-url": "https://checkpoint-sync.sepolia.ethpandaops.io/",
"timeout": 180,
"agent": "g6-standard-6"
},
Expand All @@ -35,8 +35,56 @@
"network": "mainnet",
"cl": "lodestar",
"cl_image": "chainsafe/lodestar:latest",
"checkpoint-sync-url": "https://mainnet.checkpoint.sigp.io",
"checkpoint-sync-url": "https://mainnet-checkpoint-sync.attestant.io/",
"timeout": 180,
"agent": "g6-standard-16"
},
{
"network": "op-sepolia",
"cl": "",
"cl_image": "",
"checkpoint-sync-url": "",
"timeout": 180,
"agent": "g6-standard-8"
},
{
"network": "base-sepolia",
"cl": "",
"cl_image": "",
"checkpoint-sync-url": "",
"timeout": 180,
"agent": "g6-standard-8"
},
{
"network": "op-mainnet",
"cl": "",
"cl_image": "",
"checkpoint-sync-url": "",
"timeout": 600,
"agent": "g6-standard-16"
},
{
"network": "base-mainnet",
"cl": "",
"cl_image": "",
"checkpoint-sync-url": "",
"timeout": 600,
"agent": "g6-standard-16"
},
{
"network": "joc-mainnet",
"cl": "",
"cl_image": "",
"checkpoint-sync-url": "",
"timeout": 180,
"agent": "g6-standard-6"
},
{
"network": "joc-testnet",
"cl": "",
"cl_image": "",
"checkpoint-sync-url": "",
"timeout": 180,
"agent": "g6-standard-6"
}
]

0 comments on commit 5ce06bd

Please sign in to comment.