build.yml

# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: main

on:
  push:
    branches: [ master, main ]
  pull_request:
    branches: [ master, main ]

jobs:
  verify_style:
    runs-on: 'ubuntu-latest'
    steps:
      -
        uses: actions/checkout@v4
      -
        name: Set up Python 3
        uses: actions/setup-python@v5
        with:
          python-version: '3.x'
          architecture: 'x64'
      -
        name: Install dependencies
        run: |
          python -m pip install --upgrade pip invoke typing-extensions
          invoke setup --devel --tests
      -
        name: Lint with flake8
        run: |
          invoke lint
      -
        name: Check style with black
        run: |
          invoke black --check

  verify_types:
    runs-on: 'ubuntu-latest'
    steps:
      -
        uses: actions/checkout@v4
      -
        name: Set up Python 3
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          architecture: 'x64'
      -
        name: Install dependencies
        run: |
          python -m pip install --upgrade pip invoke typing-extensions
          invoke setup --devel --tests
      -
        name: validate types
        run: |
          python/bin/python -m mypy instruct

  test_matrix:
    needs: [verify_style, verify_types, check-versions]
    strategy:
      fail-fast: false
      matrix:
        experimental: [false]
        arch:
          - 'x64'
        python_version:
          - '3.7'
          - '3.8'
          - '3.9'
          - '3.10'
          - '3.11'
          - '3.12'
          - 'pypy3.7'
          - 'pypy3.8'
          - 'pypy3.9'
        os:
          - 'ubuntu-latest'
        include:
          -
            python_version: 'pypy3.10'
            experimental: true
            arch: 'x64'
            os: ubuntu-latest

    runs-on: ${{ matrix.os }}
    continue-on-error: ${{ matrix.experimental }}
    steps:
      -
        uses: actions/checkout@v4
      -
        name: Set up Python ${{ matrix.python_version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python_version }}
          architecture: ${{ matrix.arch }}
      -
        name: Install dependencies
        run: |
          python -m pip install --upgrade pip invoke typing-extensions
          invoke setup --devel --tests --no-project
          invoke python-path
          python/bin/python -m pip install -r setup-requirements.txt
      -
        name: Prepare build artifacts
        run: |
          invoke build --validate
          python/bin/python -m pip install --no-index dist/instruct*.whl

      -
        name: Test with pytest
        run: |
          invoke test
      -
        name: Benchmark
        id: benchmark
        run: |
          invoke benchmark -f json | tee benchmark.txt
      -
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.python_version }}_${{ matrix.os }}_${{ matrix.arch }}_${{ matrix.experimental }}
          path: |
            benchmark.txt
            dist/instruct*.whl

  check-versions:
    runs-on: 'ubuntu-latest'
    steps:
      -
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true

      -
        name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          architecture: 'x64'
      -
        name: Setup
        run: |
          python -m pip install --upgrade pip invoke typing-extensions
          invoke setup --devel --no-project
      -
        name: Set some variables...
        id: version
        run: |
          echo "CURRENT_VERSION=$(grep -vE '^#' CURRENT_VERSION.txt | head -1)" >> "$GITHUB_OUTPUT"
          echo "CHANGELOG_VERSION=$(invoke last-logged-changes | head -1 | cut -f2 -d' ')" >> "$GITHUB_OUTPUT"

      -
        name: Warn if branch version already matches an existing tag
        run: |

          if [ "x$(invoke local-tag-exists --format json 'v${{ steps.version.outputs.CURRENT_VERSION }}')" = 'xtrue' ]; then
            echo '::warning file=CURRENT_VERSION.txt,line=2,title=Version already exists in tags::Tag v${{ steps.version.outputs.CURRENT_VERSION }} already exists.'
          fi

  collect_benchmark:
    runs-on: 'ubuntu-latest'
    needs: test_matrix
    steps:
      -
        name: Set up Python 3
        uses: actions/setup-python@v5
        with:
          python-version: '3.x'
          architecture: 'x64'
      -
        uses: actions/download-artifact@v4
        with:
          path: benchmarks
      -
        run: |
          ls -R benchmarks
          (jq -c -r --slurp <<< $(for name in $(echo benchmarks/*); do   cat "$name/benchmark.txt" | jq -c -r '.[] | ["'$(basename $name)'", .] | flatten'; done)) | tee benchmark.json
          python -m pip install pytablewriter terminaltables
          python -c '
          import json
          import pytablewriter as ptw
          from terminaltables import GithubFlavoredMarkdownTable

          with open("benchmark.json", "rb") as fh:
              items = json.load(fh)
              platforms = tuple(frozenset(p[0] for p in items))
              keys = [""]
              rows = [None] * len(platforms)
              for platform, group, test_name, result in items:
                  col_name = f"{group} [**{test_name}**]"
                  try:
                      keys.index(col_name)
                  except ValueError:
                      keys.append(col_name)

              def _sort(s: str):
                  version, platform, arch, experimental = s.split("_")
                  experimental = experimental.lower() == "true"
                  is_pypy = False
                  if version.startswith("pypy"):
                      version = version[len("pypy"):]
                      is_pypy = True
                  major, minor = map(int, version.split("."))
                  return (-1 if is_pypy else 0, (major, minor), platform, arch, experimental)

              platforms = tuple(sorted(platforms, key=_sort, reverse=True))
              print("platforms", platforms)
              for platform, group, test_name, result in items:
                  col_name = f"{group} [**{test_name}**]"
                  key_index = keys.index(col_name)
                  row_index = platforms.index(platform)
                  if rows[row_index] is None:
                      rows[row_index] = [None] * len(keys)
                      rows[row_index][0] = platform
                  rows[row_index][key_index] = result
              if True:
                  table = GithubFlavoredMarkdownTable([keys, *rows])
                  with open("BENCHMARK.md", "w") as fh:
                      fh.write("# Benchmark of ${{ github.sha }}\n\n")
                      fh.write(table.table)
              else:
                  writer = ptw.RstGridTableWriter(
                      table_name="Benchmark of ${{ github.sha }}",
                      headers=keys[1:],
                      value_matrix=rows,
                  )
                  with open("BENCHMARK.rst", "w") as fh:
                      writer.dump(fh)
          '
          cat BENCHMARK.*
          echo "$(cat BENCHMARK.md)" >> "$GITHUB_STEP_SUMMARY"