Skip to content
main

fix: add mode to benchmarking, refactor slightly #134

Sign in to view logs

fix: add mode to benchmarking, refactor slightly

fix: add mode to benchmarking, refactor slightly #134

Workflow file for this run

.github/workflows/build.yml at 43601f9
# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: main
on:
push:
branches: [ master, main ]
pull_request:
branches: [ master, main ]
jobs:
verify_style:
runs-on: 'ubuntu-latest'
steps:
-
uses: actions/checkout@v4
-
name: Set up Python 3
uses: actions/setup-python@v5
with:
python-version: '3.x'
architecture: 'x64'
-
name: Install dependencies
run: |
python -m pip install --upgrade pip invoke typing-extensions
invoke setup --devel --tests
-
name: Lint with flake8
run: |
invoke lint
-
name: Check style with black
run: |
invoke black --check
verify_types:
runs-on: 'ubuntu-latest'
steps:
-
uses: actions/checkout@v4
-
name: Set up Python 3
uses: actions/setup-python@v5
with:
python-version: '3.11'
architecture: 'x64'
-
name: Install dependencies
run: |
python -m pip install --upgrade pip invoke typing-extensions
invoke setup --devel --tests
-
name: validate types
run: |
python/bin/python -m mypy instruct
test_matrix:
needs: [verify_style, verify_types, check-versions]
strategy:
fail-fast: false
matrix:
experimental: [false]
arch:
- 'x64'
python_version:
- '3.7'
- '3.8'
- '3.9'
- '3.10'
- '3.11'
- '3.12'
- 'pypy3.7'
- 'pypy3.8'
- 'pypy3.9'
os:
- 'ubuntu-latest'
include:
-
python_version: 'pypy3.10'
experimental: true
arch: 'x64'
os: ubuntu-latest
runs-on: ${{ matrix.os }}
continue-on-error: ${{ matrix.experimental }}
steps:
-
uses: actions/checkout@v4
-
name: Set up Python ${{ matrix.python_version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python_version }}
architecture: ${{ matrix.arch }}
-
name: Install dependencies
run: |
python -m pip install --upgrade pip invoke typing-extensions
invoke setup --devel --tests --no-project
invoke python-path
python/bin/python -m pip install -r setup-requirements.txt
-
name: Prepare build artifacts
run: |
invoke build --validate
python/bin/python -m pip install --no-index dist/instruct*.whl
-
name: Test with pytest
run: |
invoke test
-
name: Benchmark
id: benchmark
run: |
invoke benchmark -f json | tee benchmark.txt
-
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.python_version }}_${{ matrix.os }}_${{ matrix.arch }}_${{ matrix.experimental }}
path: |
benchmark.txt
dist/instruct*.whl
check-versions:
runs-on: 'ubuntu-latest'
steps:
-
uses: actions/checkout@v4
with:
fetch-depth: 0
fetch-tags: true
-
name: Set up Python 3.11
uses: actions/setup-python@v5
with:
python-version: '3.11'
architecture: 'x64'
-
name: Setup
run: |
python -m pip install --upgrade pip invoke typing-extensions
invoke setup --devel --no-project
-
name: Set some variables...
id: version
run: |
echo "CURRENT_VERSION=$(grep -vE '^#' CURRENT_VERSION.txt | head -1)" >> "$GITHUB_OUTPUT"
echo "CHANGELOG_VERSION=$(invoke last-logged-changes | head -1 | cut -f2 -d' ')" >> "$GITHUB_OUTPUT"
-
name: Warn if branch version already matches an existing tag
run: |
if [ "x$(invoke local-tag-exists --format json 'v${{ steps.version.outputs.CURRENT_VERSION }}')" = 'xtrue' ]; then
echo '::warning file=CURRENT_VERSION.txt,line=2,title=Version already exists in tags::Tag v${{ steps.version.outputs.CURRENT_VERSION }} already exists.'
fi
collect_benchmark:
runs-on: 'ubuntu-latest'
needs: test_matrix
steps:
-
name: Set up Python 3
uses: actions/setup-python@v5
with:
python-version: '3.x'
architecture: 'x64'
-
uses: actions/download-artifact@v4
with:
path: benchmarks
-
run: |
ls -R benchmarks
(jq -c -r --slurp <<< $(for name in $(echo benchmarks/*); do cat "$name/benchmark.txt" | jq -c -r '.[] | ["'$(basename $name)'", .] | flatten'; done)) | tee benchmark.json
python -m pip install pytablewriter terminaltables
python -c '
import json
import pytablewriter as ptw
from terminaltables import GithubFlavoredMarkdownTable
with open("benchmark.json", "rb") as fh:
items = json.load(fh)
platforms = tuple(frozenset(p[0] for p in items))
keys = [""]
rows = [None] * len(platforms)
for platform, group, test_name, result in items:
col_name = f"{group}\n{test_name}"
try:
keys.index(col_name)
except ValueError:
keys.append(col_name)
def _sort(s: str):
version, platform, arch, experimental = s.split("_")
experimental = experimental.lower() == "true"
is_pypy = False
if version.startswith("pypy"):
version = version[len("pypy"):]
is_pypy = True
major, minor = map(int, version.split("."))
return (-1 if is_pypy else 0, (major, minor), platform, arch, experimental)
platforms = tuple(sorted(platforms, key=_sort, reverse=True))
print("platforms", platforms)
for platform, group, test_name, result in items:
col_name = f"{group}\n{test_name}"
key_index = keys.index(col_name)
row_index = platforms.index(platform)
if rows[row_index] is None:
rows[row_index] = [None] * len(keys)
rows[row_index][0] = platform
rows[row_index][key_index] = result
if True:
table = GithubFlavoredMarkdownTable([keys, *rows])
with open("BENCHMARK.md", "w") as fh:
fh.write("# Benchmark of ${{ github.sha }}\n\n")
fh.write(table.table)
else:
writer = ptw.RstGridTableWriter(
table_name="Benchmark of ${{ github.sha }}",
headers=keys[1:],
value_matrix=rows,
)
with open("BENCHMARK.rst", "w") as fh:
writer.dump(fh)
'
cat BENCHMARK.*
echo "$(cat BENCHMARK.md)" >> "$GITHUB_STEP_SUMMARY"