From 8db43fcd91345ef528463c289e471f0fccda0892 Mon Sep 17 00:00:00 2001 From: Adam Paszke Date: Fri, 13 May 2022 13:41:09 +0000 Subject: [PATCH] Add a GitHub action for continuous benchmarking We've put in a lot of effort into making Dex faster in the recent months and it would be a waste to have it all slip away. So, to prevent that, I've set up a simple system for continuous performance tracking. This includes three parts: 1. I created an orphan branch named `performance-data`, which we'll use as a simple CSV-based database of performance measurements. I ran a few benchmarks using the new `benchmarks/continuous.py` script to pre-populate it. 2. On the `gh-pages` branch, there's a new `performance.html` file, now accessible via [this URL](https://google-research.github.io/dex-lang/performance.html). It pulls the data from the `performance-data` branch and displays it as a series of plots showing changes in total allocation and end-to-end execution times of a few of our examples. 3. This commit checks in the file used to perform benchmarks, and adds a GitHub action that will run it every time we push to `main`. While total allocation numbers are very stable, at this point you might be worried that any time estimates we get from the free VMs that execute GitHub actions will be too noisy to provide any good signal. And if we did it naively, you wouldn't be wrong! To make the drift between different machines lower we use the techniques outlined in [this post](https://labs.quansight.org/blog/2021/08/github-actions-benchmarks/). The idea is to report _relative change_ in execution time compared to baseline. Most importantly, any time we want to evaluate a new commit, _we rebenchmark the baseline_ to adjust for the differences between machines. To minimize noise on shorter time scales, we interleave the evaluations of baseline compiler with those using newer versions. --- .github/workflows/bench.yaml | 58 +++++++++++++++++ .github/workflows/docs.yaml | 7 --- benchmarks/continuous.py | 117 +++++++++++++++++++++++++++++++++++ 3 files changed, 175 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/bench.yaml create mode 100644 benchmarks/continuous.py diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml new file mode 100644 index 000000000..186c2133d --- /dev/null +++ b/.github/workflows/bench.yaml @@ -0,0 +1,58 @@ +name: Continuous benchmarking + +on: + push: + branches: + - main + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-20.04] + include: + - os: ubuntu-20.04 + install_deps: sudo apt-get install llvm-12-tools llvm-12-dev pkg-config + path_extension: /usr/lib/llvm-12/bin + + steps: + - name: Checkout the repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Install system dependencies + run: | + ${{ matrix.install_deps }} + echo "${{ matrix.path_extension }}" >> $GITHUB_PATH + + - name: Cache + uses: actions/cache@v2 + with: + path: | + ~/.stack + + key: ${{ runner.os }}-bench-v1-${{ hashFiles('**/*.cabal', 'stack*.yaml') }} + restore-keys: ${{ runner.os }}-bench-v1 + + - name: Benchmark + run: python3 benchmarks/continuous.py /tmp/new-perf-data.csv /tmp/new-commits.csv ${GITHUB_SHA} + + - name: Switch to the data branch + uses: actions/checkout@v2 + with: + ref: performance-data + + - name: Append new data points + run: | + cat /tmp/new-perf-data.csv >>performance.csv + cat /tmp/new-commits.csv >>commits.csv + + - name: Commit new data points + run: | + git config --global user.name 'Dex CI' + git config --global user.email 'apaszke@users.noreply.github.com' + git add performance.csv commits.csv + git commit -m "Add measurements for ${GITHUB_SHA}" + git push diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 431329867..7a3287435 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -20,13 +20,6 @@ jobs: - name: Checkout the repository uses: actions/checkout@v2 - - name: Setup Haskell Stack - uses: actions/setup-haskell@v1 - with: - enable-stack: true - stack-no-global: true - stack-version: 'latest' - - name: Install system dependencies run: | ${{ matrix.install_deps }} diff --git a/benchmarks/continuous.py b/benchmarks/continuous.py new file mode 100644 index 000000000..9164b19f9 --- /dev/null +++ b/benchmarks/continuous.py @@ -0,0 +1,117 @@ +import re +import os +import sys +import csv +import subprocess +import tempfile +from functools import partial +from dataclasses import dataclass +from pathlib import Path +from typing import Union, Sequence + + +BASELINE = '8dd1aa8539060a511d0f85779ae2c8019162f567' +BENCH_EXAMPLES = [('kernelregression', 10), ('psd', 10), ('fluidsim', 10), ('regression', 10)] + + +def run(*args, capture=False, env=None): + print('> ' + ' '.join(map(str, args))) + return subprocess.run(args, check=True, text=True, capture_output=capture, env=env) + + +def read(*args, **kwargs): + return run(*args, capture=True, **kwargs).stdout + + +def read_stderr(*args, **kwargs): + return run(*args, capture=True, **kwargs).stderr + + +def build(commit): + if os.path.exists(commit): + print(f'Skipping the build of {commit}') + else: + run('git', 'checkout', commit) + run('make', 'install', env=dict(os.environ, PREFIX=commit)) + dex_bin = Path.cwd() / commit / 'dex' + return dex_bin + + +def benchmark(baseline_bin, latest_bin): + with tempfile.TemporaryDirectory() as tmp: + def clean(bin, uniq): + run(bin, 'clean', env={'XDG_CACHE_HOME': Path(tmp) / uniq}) + def bench(bin, uniq, bench_name, path): + return parse_result( + read_stderr(bin, 'script', path, '+RTS', '-s', + env={'XDG_CACHE_HOME': Path(tmp) / uniq})) + baseline_clean = partial(clean, baseline_bin, 'baseline') + baseline_bench = partial(bench, baseline_bin, 'baseline') + latest_clean = partial(clean, latest_bin, 'latest') + latest_bench = partial(bench, latest_bin, 'latest') + results = [] + for example, repeats in BENCH_EXAMPLES: + path = Path('examples') / (example + '.dx') + # warm-up the caches + baseline_clean() + baseline_bench(example, path) + latest_clean() + latest_bench(example, path) + for i in range(repeats): + print(f'Iteration {i}') + baseline_alloc, baseline_time = baseline_bench(example, path) + latest_alloc, latest_time = latest_bench(example, path) + print(baseline_alloc, '->', latest_alloc) + print(baseline_time, '->', latest_time) + results.append(Result(example, 'alloc', latest_alloc)) + results.append(Result(example, 'time_rel', latest_time / baseline_time)) + return results + + +@dataclass +class Result: + benchmark: str + measure: str + value: Union[int, float] + + +ALLOC_PATTERN = re.compile(r"^\s*([0-9,]+) bytes allocated in the heap", re.M) +TIME_PATTERN = re.compile(r"^\s*Total\s*time\s*([0-9.]+)s", re.M) +def parse_result(output): + alloc_line = ALLOC_PATTERN.search(output) + if alloc_line is None: + raise RuntimeError("Couldn't extract total allocations") + total_alloc = int(alloc_line.group(1).replace(',', '')) + time_line = TIME_PATTERN.search(output) + if time_line is None: + raise RuntimeError("Couldn't extract total time") + total_time = float(time_line.group(1)) + return total_alloc, total_time + + +def save(commit, results: Sequence[Result], datapath, commitpath): + with open(datapath, 'a', newline='') as datafile: + writer = csv.writer(datafile, delimiter=',', quotechar='"', dialect='unix') + for r in results: + writer.writerow((commit, r.benchmark, r.measure, r.value)) + with open(commitpath, 'a', newline='') as commitfile: + writer = csv.writer(commitfile, delimiter=',', quotechar='"', dialect='unix') + date = read('git', 'show', '-s', '--format=%ct', commit, '--').strip() + writer.writerow([commit, date]) + + +def main(argv): + if len(argv) != 3: + raise ValueError("Expected three arguments!") + datapath, commitpath, commit = argv + print('Building baseline: {BASELINE}') + baseline_bin = build(BASELINE) + print(f'Building latest: {commit}') + latest_bin = build(commit) + results = benchmark(baseline_bin, latest_bin) + save(commit, results, datapath, commitpath) + print('DONE!') + + +if __name__ == '__main__': + main(sys.argv[1:])