-
Notifications
You must be signed in to change notification settings - Fork 180
99 lines (82 loc) · 3.83 KB
/
bench.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
name: Benchstat
on:
pull_request:
branches:
- master
- "auto-cadence-upgrade/**"
- "feature/**"
- "v[0-9]+.[0-9]+"
paths:
- ".github/workflows/bench.yml"
- "fvm/**"
- "engine/execution/**"
- "go.sum"
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: true
jobs:
benchstat:
name: Performance regression check
runs-on: ubuntu-latest
# Check if the event is not triggered by a fork
# peter-evans/find-comment@v1 does not work on forks.
# see https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#restrictions-on-repository-forks for details.
# Ideally we would like to still run the benchmark on forks, but we can't do that with the current setup.
if: github.event.pull_request.head.repo.full_name == github.repository
continue-on-error: true
steps:
- name: Set benchmark repetitions
# reducing repetition will speed up execution,
# but will be more inaccurate at detecting change
run: echo "::set-output name=benchmark_repetitions::7"
id: settings
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Setup go
uses: actions/setup-go@v3
with:
go-version: "1.20"
cache: true
- name: Build relic
run: make crypto_setup_gopath
- name: Run benchmark on current branch
run: |
(for i in {1..${{ steps.settings.outputs.benchmark_repetitions }}}; do go test ./fvm ./engine/execution/computation --bench . --tags relic -shuffle=on --benchmem --run ^$; done) | tee new.txt
- name: Checkout base branch
run: git checkout ${{ github.event.pull_request.base.sha }}
- name: Run benchmark on base branch
run: |
(for i in {1..${{ steps.settings.outputs.benchmark_repetitions }}}; do go test ./fvm ./engine/execution/computation --bench . --tags relic -shuffle=on --benchmem --run ^$; done) | tee old.txt
# see https://trstringer.com/github-actions-multiline-strings/ to see why this part is complex
- name: Use benchstat for comparison
run: |
export PATH=$PATH:$(go env GOPATH)/bin
go install golang.org/x/perf/cmd/benchstat@91a04616dc65ba76dbe9e5cf746b923b1402d303
echo "BENCHSTAT<<EOF" >> $GITHUB_ENV
echo "$(benchstat -html -sort delta old.txt new.txt | sed '/<title/,/<\/style>/d' | sed 's/<!doctype html>//g')" >> $GITHUB_ENV
echo "EOF" >> $GITHUB_ENV
- name: Find existing comment on PR
uses: peter-evans/find-comment@v1
id: fc
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: "github-actions[bot]"
body-includes: "## FVM [Benchstat](https://pkg.go.dev/golang.org/x/perf/cmd/benchstat) comparison"
- name: Create or update comment
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ steps.fc.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: |
## FVM [Benchstat](https://pkg.go.dev/golang.org/x/perf/cmd/benchstat) comparison
This branch with compared with the base branch ${{ github.event.pull_request.base.label }} commit ${{ github.event.pull_request.base.sha }}
The command `(for i in {1..${{ steps.settings.outputs.benchmark_repetitions }}}; do go test ./fvm ./engine/execution/computation --bench . --tags relic -shuffle=on --benchmem --run ^$; done)` was used.
<details>
<summary>Collapsed results for better readability</summary>
<p>
${{ env.BENCHSTAT }}
</p>
</details>
edit-mode: replace