From 68138bf3e13af73bd6ad280aeba332d405a739e3 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 27 Mar 2024 12:47:57 -0400 Subject: [PATCH 1/8] ci: add a benchmark workflow --- .github/workflows/bench.yml | 46 +++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/bench.yml diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 00000000..5e9fc38c --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,46 @@ +name: Benchstat + +on: + pull_request: + branches: [ master ] + +permissions: # added using https://github.com/step-security/secure-workflows + contents: read + +jobs: + + bench: + name: Performance regression check + runs-on: ubuntu-latest + steps: + - name: Checkout (previous) + uses: actions/checkout@v3 + with: + ref: ${{ github.base_ref }} + path: previous + + - name: Checkout (new) + uses: actions/checkout@v3 + with: + path: new + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version-file: '${{ github.workspace }}/go.mod' + + - name: Install dependencies + run: go install golang.org/x/perf/cmd/benchstat@latest + + - name: Run Benchmark (previous) + run: | + cd previous + go test -bench=. -benchmem -benchtime=100ms -count=8 > benchmark.txt + + - name: Run Benchmark (new) + run: | + cd new + go test -bench=. -benchmem -benchtime=100ms -count=8 > benchmark.txt + + - name: Run Benchstat + run: benchstat previous/benchmark.txt new/benchmark.txt From 8eb543f5e61bbd45065b905c2a8a4b67215ae81a Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 27 Mar 2024 12:52:52 -0400 Subject: [PATCH 2/8] Use Go version from the PR. --- .github/workflows/bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 5e9fc38c..27cb05d8 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -27,7 +27,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version-file: '${{ github.workspace }}/go.mod' + go-version-file: '${{ github.workspace }}/new/go.mod' - name: Install dependencies run: go install golang.org/x/perf/cmd/benchstat@latest From 386dee88067fd67deabbb37290f131eded0b84b0 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 27 Mar 2024 12:54:51 -0400 Subject: [PATCH 3/8] Fix benchmark command. --- .github/workflows/bench.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 27cb05d8..3f365d79 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -35,12 +35,12 @@ jobs: - name: Run Benchmark (previous) run: | cd previous - go test -bench=. -benchmem -benchtime=100ms -count=8 > benchmark.txt + go test -bench=. -benchmem -benchtime=100ms -count=10 ./... > benchmark.txt - name: Run Benchmark (new) run: | cd new - go test -bench=. -benchmem -benchtime=100ms -count=8 > benchmark.txt + go test -bench=. -benchmem -benchtime=100ms -count=10 ./... > benchmark.txt - name: Run Benchstat run: benchstat previous/benchmark.txt new/benchmark.txt From 0137573c129cbe15517d6d25aa32a5c2c689a612 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 27 Mar 2024 14:08:57 -0400 Subject: [PATCH 4/8] Add 5 minute timeout. --- .github/workflows/bench.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 3f365d79..1e5bb971 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -12,6 +12,7 @@ jobs: bench: name: Performance regression check runs-on: ubuntu-latest + timeout-minutes: 5 steps: - name: Checkout (previous) uses: actions/checkout@v3 From b139046daa0d6eaf3f43046c33b2bb7a57a0a971 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 27 Mar 2024 14:28:38 -0400 Subject: [PATCH 5/8] Add 5 minute timeout to individual workflow steps. --- .github/workflows/bench.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 1e5bb971..6be2d293 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -34,11 +34,13 @@ jobs: run: go install golang.org/x/perf/cmd/benchstat@latest - name: Run Benchmark (previous) + timeout-minutes: 5 run: | cd previous go test -bench=. -benchmem -benchtime=100ms -count=10 ./... > benchmark.txt - name: Run Benchmark (new) + timeout-minutes: 5 run: | cd new go test -bench=. -benchmem -benchtime=100ms -count=10 ./... > benchmark.txt From 0c602b6cc4c2f624c1fe6cf7a516f4a21c75e1d6 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 27 Mar 2024 15:51:00 -0400 Subject: [PATCH 6/8] Try something. --- .github/workflows/bench.yml | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 6be2d293..9bb8b927 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -12,38 +12,27 @@ jobs: bench: name: Performance regression check runs-on: ubuntu-latest - timeout-minutes: 5 steps: - - name: Checkout (previous) - uses: actions/checkout@v3 - with: - ref: ${{ github.base_ref }} - path: previous - - name: Checkout (new) uses: actions/checkout@v3 - with: - path: new - name: Set up Go uses: actions/setup-go@v4 with: go-version-file: '${{ github.workspace }}/new/go.mod' - - name: Install dependencies - run: go install golang.org/x/perf/cmd/benchstat@latest + - name: Get dependencies + run: go mod download - - name: Run Benchmark (previous) - timeout-minutes: 5 + - name: Benchmark against GITHUB_BASE_REF run: | - cd previous - go test -bench=. -benchmem -benchtime=100ms -count=10 ./... > benchmark.txt - - - name: Run Benchmark (new) - timeout-minutes: 5 - run: | - cd new - go test -bench=. -benchmem -benchtime=100ms -count=10 ./... > benchmark.txt - - - name: Run Benchstat - run: benchstat previous/benchmark.txt new/benchmark.txt + go install golang.org/x/perf/cmd/benchstat@latest + echo "New Commit:" + git log -1 --format="%H" + go test -bench=. -benchmem -benchtime=100ms -count=10 > /tmp/new.txt + git reset --hard HEAD + git checkout $GITHUB_BASE_REF + echo "Base Commit:" + git log -1 --format="%H" + go test -bench=. -benchmem -benchtime=100ms -count=10 > /tmp/old.txt + $GOBIN/benchstat /tmp/old.txt /tmp/new.txt From 9bec6493fddf79b8b6df17727d2b3ffb8bbb6133 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 27 Mar 2024 15:51:45 -0400 Subject: [PATCH 7/8] Fix go.mod reference. --- .github/workflows/bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 9bb8b927..c9bb3c50 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version-file: '${{ github.workspace }}/new/go.mod' + go-version-file: '${{ github.workspace }}/go.mod' - name: Get dependencies run: go mod download From 647850f52ada6fd4cfc81c5f2b0721db97b7bd97 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 27 Mar 2024 15:54:57 -0400 Subject: [PATCH 8/8] Include all Go files. --- .github/workflows/bench.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index c9bb3c50..d9552234 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -29,10 +29,10 @@ jobs: go install golang.org/x/perf/cmd/benchstat@latest echo "New Commit:" git log -1 --format="%H" - go test -bench=. -benchmem -benchtime=100ms -count=10 > /tmp/new.txt + go test -bench=. -benchmem -benchtime=100ms -count=10 ./... > /tmp/new.txt git reset --hard HEAD git checkout $GITHUB_BASE_REF echo "Base Commit:" git log -1 --format="%H" - go test -bench=. -benchmem -benchtime=100ms -count=10 > /tmp/old.txt + go test -bench=. -benchmem -benchtime=100ms -count=10 ./... > /tmp/old.txt $GOBIN/benchstat /tmp/old.txt /tmp/new.txt