CI for Benchmarking (#346)

* chore(just): define benchmark subcommand

* chore(github): workflow for benchmarking

* chore(github): update workflows to actions/checkout@v3

* fix(github): issue with runner labels

* fix(just): lint

- wip: find out a safe way to save output for benchmarks to a proper file folder

* wip: attempt caching between jobs

* fix(just): make sure to save via nu

* fix(just): enable execute permissions over virto-node

* fix(jus): benchmarks over node's pallets

* feat(ci): generate a PR after benchmarking

* fix(github-actions): rename jobs on benchmarking workflow

* fix(just): pathname to kreivo-runtime Cargo

.github/workflows/benchmarking.yml

@@ -0,0 +1,55 @@
+name: Benchmarks
+
+on:
+  push:
+    branches: 
+      - master
+
+jobs:
+  build:
+    name: Build Target
+    runs-on: [self-hosted, builder]
+    steps:
+      - uses: actions/checkout@v4
+      - run: just build-local runtime-benchmarks
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ${{ github.run_number }}-virto-node
+          path: ./target/release/virto-node
+      - uses: actions/cache@v4
+        with:
+          path: target
+          key: ${{ runner.os }}-target-cache
+
+  benchmark:
+    name: Run Benchmarks
+    needs: build
+    runs-on: [self-hosted, benchmarks]
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          mkdir -p .benchmarking-logs target/release
+      - uses: actions/download-artifact@v4
+        with:
+          name: ${{ github.run_number }}-virto-node
+          path: ./target/release
+      - run: |
+          just benchmarks
+      - uses: actions/upload-artifact@v3
+        with:
+          name: ${{ github.run_id }}-benchmark_logs
+          path: ./.benchmarking-logs
+      - uses: peter-evans/create-pull-request@v6
+        with:
+          add-paths: runtime/kreivo/src/weights
+          commit-message: '[ci] calculate weights'
+          branch: benchmarks
+          branch-suffix: ${{ github.sha }}
+          title: "Benchmarking: Calculate weights for ${{ github.sha }}"
+          body: |
+            This Pull Request is automatically raised when pushing over `master`
+            and should be resolved and reviewed manually.
+          assignees: ${{ github.actor_id }}

.github/workflows/check-pr.yml

@@ -6,15 +6,23 @@ on:
 jobs:
   build:
     name: Check lint
-    runs-on: self-hosted
+    runs-on: [self-hosted, builder]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+      - uses: actions/cache@v4
+        with:
+          path: target
+          key: ${{ runner.os }}-${{ github.ref }}-target-cache
       - run: just check
 
   tests:
     name: Check project
-    runs-on: self-hosted
+    runs-on: [self-hosted, builder]
     needs: build
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+      - uses: actions/cache@v4
+        with:
+          path: target
+          key: ${{ runner.os }}-${{ github.ref }}-target-cache
       - run: just test

.github/workflows/release.yml

@@ -10,7 +10,7 @@ jobs:
     name: Build node
     runs-on: self-hosted
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Build parachain node & artifacts
         run: just build-container-local

.github/workflows/rust.yml

@@ -10,6 +10,6 @@ jobs:
     name: Build project
     runs-on: self-hosted
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Test parachain
         run: echo test #TODO: change for integration test when available

.gitignore

@@ -26,6 +26,9 @@ node_modules/
 **/bin/
 release/
 
+# Benchmarking
+.benchmarking-logs
+
 # zombienet binaries
 zombienet-macos
 zombienet-linux

justfile

@@ -1,5 +1,6 @@
 # NOTE: This justfile relies heavily on nushell, make sure to install it: https://www.nushell.sh
 set shell := ["nu", "-c"]
+
 podman := `(which podman) ++ (which docker) | (first).path` # use podman otherwise docker
 ver := `open node/Cargo.toml | get package.version`
 image := "ghcr.io/virto-network/virto"
@@ -42,6 +43,28 @@ check: _check_deps
 build-local features="":
 	cargo build --release --features '{{features}}'
 
+benchmarks:
+	# TODO: build benchmarks for every pallet that's currently within the runtime as
+	# a dependency
+	mkdir .benchmarking-logs
+	chmod +x {{node}}
+
+	ls "pallets" \
+		| each {|path| open ($path.name + /Cargo.toml) | get package.name} \
+		| filter {|pallet| cat runtime/kreivo/Cargo.toml | str contains $pallet} \
+		| filter {|pallet| cat runtime/kreivo/Cargo.toml | str contains ([$pallet, "runtime-benchmarks"] | str join '/')} \
+		| each {|pallet| str replace --all '-' '_' } \
+		| each {|pallet| just benchmark $pallet}
+
+benchmark pallet="" extrinsic="*":
+	touch .benchmarking-logs/{{pallet}}.txt
+	./target/release/virto-node benchmark pallet \
+		--chain {{chain}}-local \
+		--pallet '{{pallet}}' --extrinsic '{{extrinsic}}' \
+		--steps 50 \
+		--repeat 20 \
+		--output runtime/kreivo/src/weights/{{pallet}}.rs | save -a --force .benchmarking-logs/{{pallet}}.txt
+
 build-container:
 	#!/usr/bin/env nu
 	'FROM docker.io/paritytech/ci-linux:production as builder