From ea478497f6f428c43e15232cdf136ff6f9744418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Andr=C3=A9s=20Dorado=20Su=C3=A1rez?= Date: Sun, 17 Mar 2024 17:42:05 -0500 Subject: [PATCH] CI for Benchmarking (#346) * chore(just): define benchmark subcommand * chore(github): workflow for benchmarking * chore(github): update workflows to actions/checkout@v3 * fix(github): issue with runner labels * fix(just): lint - wip: find out a safe way to save output for benchmarks to a proper file folder * wip: attempt caching between jobs * fix(just): make sure to save via nu * fix(just): enable execute permissions over virto-node * fix(jus): benchmarks over node's pallets * feat(ci): generate a PR after benchmarking * fix(github-actions): rename jobs on benchmarking workflow * fix(just): pathname to kreivo-runtime Cargo --- .github/workflows/benchmarking.yml | 55 ++++++++++++++++++++++++++++++ .github/workflows/check-pr.yml | 16 ++++++--- .github/workflows/release.yml | 2 +- .github/workflows/rust.yml | 2 +- .gitignore | 3 ++ justfile | 23 +++++++++++++ 6 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/benchmarking.yml diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml new file mode 100644 index 00000000..9d9db7da --- /dev/null +++ b/.github/workflows/benchmarking.yml @@ -0,0 +1,55 @@ +name: Benchmarks + +on: + push: + branches: + - master + +jobs: + build: + name: Build Target + runs-on: [self-hosted, builder] + steps: + - uses: actions/checkout@v4 + - run: just build-local runtime-benchmarks + - uses: actions/upload-artifact@v4 + with: + name: ${{ github.run_number }}-virto-node + path: ./target/release/virto-node + - uses: actions/cache@v4 + with: + path: target + key: ${{ runner.os }}-target-cache + + benchmark: + name: Run Benchmarks + needs: build + runs-on: [self-hosted, benchmarks] + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@v4 + - run: | + mkdir -p .benchmarking-logs target/release + - uses: actions/download-artifact@v4 + with: + name: ${{ github.run_number }}-virto-node + path: ./target/release + - run: | + just benchmarks + - uses: actions/upload-artifact@v3 + with: + name: ${{ github.run_id }}-benchmark_logs + path: ./.benchmarking-logs + - uses: peter-evans/create-pull-request@v6 + with: + add-paths: runtime/kreivo/src/weights + commit-message: '[ci] calculate weights' + branch: benchmarks + branch-suffix: ${{ github.sha }} + title: "Benchmarking: Calculate weights for ${{ github.sha }}" + body: | + This Pull Request is automatically raised when pushing over `master` + and should be resolved and reviewed manually. + assignees: ${{ github.actor_id }} diff --git a/.github/workflows/check-pr.yml b/.github/workflows/check-pr.yml index f67b7fd0..96b1c359 100644 --- a/.github/workflows/check-pr.yml +++ b/.github/workflows/check-pr.yml @@ -6,15 +6,23 @@ on: jobs: build: name: Check lint - runs-on: self-hosted + runs-on: [self-hosted, builder] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + - uses: actions/cache@v4 + with: + path: target + key: ${{ runner.os }}-${{ github.ref }}-target-cache - run: just check tests: name: Check project - runs-on: self-hosted + runs-on: [self-hosted, builder] needs: build steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + - uses: actions/cache@v4 + with: + path: target + key: ${{ runner.os }}-${{ github.ref }}-target-cache - run: just test \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4e77a6b1..7356e78a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ jobs: name: Build node runs-on: self-hosted steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Build parachain node & artifacts run: just build-container-local diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1f063c67..882a0120 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -10,6 +10,6 @@ jobs: name: Build project runs-on: self-hosted steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Test parachain run: echo test #TODO: change for integration test when available diff --git a/.gitignore b/.gitignore index dec0c4e8..87bac7eb 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,9 @@ node_modules/ **/bin/ release/ +# Benchmarking +.benchmarking-logs + # zombienet binaries zombienet-macos zombienet-linux diff --git a/justfile b/justfile index b08c6246..fce48bae 100644 --- a/justfile +++ b/justfile @@ -1,5 +1,6 @@ # NOTE: This justfile relies heavily on nushell, make sure to install it: https://www.nushell.sh set shell := ["nu", "-c"] + podman := `(which podman) ++ (which docker) | (first).path` # use podman otherwise docker ver := `open node/Cargo.toml | get package.version` image := "ghcr.io/virto-network/virto" @@ -42,6 +43,28 @@ check: _check_deps build-local features="": cargo build --release --features '{{features}}' +benchmarks: + # TODO: build benchmarks for every pallet that's currently within the runtime as + # a dependency + mkdir .benchmarking-logs + chmod +x {{node}} + + ls "pallets" \ + | each {|path| open ($path.name + /Cargo.toml) | get package.name} \ + | filter {|pallet| cat runtime/kreivo/Cargo.toml | str contains $pallet} \ + | filter {|pallet| cat runtime/kreivo/Cargo.toml | str contains ([$pallet, "runtime-benchmarks"] | str join '/')} \ + | each {|pallet| str replace --all '-' '_' } \ + | each {|pallet| just benchmark $pallet} + +benchmark pallet="" extrinsic="*": + touch .benchmarking-logs/{{pallet}}.txt + ./target/release/virto-node benchmark pallet \ + --chain {{chain}}-local \ + --pallet '{{pallet}}' --extrinsic '{{extrinsic}}' \ + --steps 50 \ + --repeat 20 \ + --output runtime/kreivo/src/weights/{{pallet}}.rs | save -a --force .benchmarking-logs/{{pallet}}.txt + build-container: #!/usr/bin/env nu 'FROM docker.io/paritytech/ci-linux:production as builder