tests.yml

name: Pull Request Tests

on: [pull_request]


jobs:
  test:
    if: "!contains(github.event.head_commit.message, '[skip ci]')"
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include:
          # lower bound version of packages
          - python-version: '3.9'
            pip-args: '.[tests,dev,minver]'
            label: 'lower-bound'
          # upper bound version of packages (unpinned)
          - python-version: '3.x'
            pip-args: '.[tests,dev]'
            label: 'unpinned'

    steps:
      # SETUP

      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref }}  # Ensure branch is checked out, not detached state (so we can push a commit later)
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip'

      - name: Add checkout directory to PYTHONPATH
        run: echo "PYTHONPATH=$(pwd):$PYTHONPATH" >> $GITHUB_ENV

      - name: Install dependencies
        id: install-dependencies
        run: |
          pip install ${{ matrix.pip-args }}
          pip uninstall dagrunner -y

      - name: Set up SSH for localhost
        run: |
          ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa
          cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
          ssh-keyscan -H $(hostname) >> ~/.ssh/known_hosts
          chmod 600 ~/.ssh/authorized_keys

      # TESTS (inc. test coverage)

      # excluded logging as per https://github.com/MetOffice/dagrunner/issues/39
      - name: Run pytest + coverage report gen
        run: pytest --cov=dagrunner --cov-report=term --cov-report=html | tee coverage_output.txt; test ${PIPESTATUS[0]} -eq 0

      # TESTS (main branch)

      - name: Cache ref branch coverage report
        id: cache-ref-coverage
        uses: actions/cache@v4
        with:
          path: ref/coverage_output.txt
          key: ref-${{ github.event.pull_request.base.sha }}

      - name: Checkout ref branch
        if: steps.cache-ref-coverage.outputs.cache-hit != 'true'
        uses: actions/checkout@v4
        with:
          path: ref
          ref: ${{ github.base_ref }}

      # excluded logging as per https://github.com/MetOffice/dagrunner/issues/39
      - name: Run tests with coverage for ref branch
        if: steps.cache-ref-coverage.outputs.cache-hit != 'true'
        run: |
          cd ref
          pytest --cov=dagrunner --cov-report=term --cov-report=html | tee coverage_output.txt; test ${PIPESTATUS[0]} -eq 0

      # TESTS (compare coverage)

      - name: Compare coverage
        id: comp-coverage
        if: ${{ matrix.label == 'unpinned' }}
        run: |
            pr_coverage_total=$(grep TOTAL coverage_output.txt | awk '{print $NF}' | awk '{print substr($0, 1, length($0)-1)}')
            echo "pr_coverage_total=$pr_coverage_total" | tee -a $GITHUB_OUTPUT
            ref_coverage_total=$(grep TOTAL ref/coverage_output.txt | awk '{print $NF}' | awk '{print substr($0, 1, length($0)-1)}')
            echo "ref_coverage_total=$ref_coverage_total" | tee -a $GITHUB_OUTPUT
            if (( (( ref_coverage_total - pr_coverage_total )) > 1 )); then
                echo "coverage_decreased=true" | tee -a $GITHUB_OUTPUT
            else
                echo "coverage_decreased=false" | tee -a $GITHUB_OUTPUT
            fi

      - name: Comment coverage report
        if: ${{ matrix.label == 'unpinned' && steps.comp-coverage.outputs.coverage_decreased == 'true' }}
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            let comment = String();
            comment = "The test coverage has decreased from ${{ steps.comp-coverage.outputs.ref_coverage_total }}% to ${{ steps.comp-coverage.outputs.pr_coverage_total }}% (commit SHA: ${{ github.event.pull_request.head.sha }})."
            comment += "\nPlease review test coverage.  Report uploaded as artifact.";
            console.log(comment)
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: comment
            });

      - name: Upload coverage report
        if: ${{ matrix.label == 'unpinned' && steps.comp-coverage.outputs.coverage_decreased == 'true' }}
        uses: actions/upload-artifact@v4
        with:
          name: coverage-report-pr
          path: |
            coverage_output.txt
            htmlcov/

      # PRE-COMMIT

      - name: Python interpreter version sha (PYSHA)
        if: ${{ matrix.label == 'unpinned' }}
        run: echo "PYSHA=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV

      - name: Cache pre-commit
        if: ${{ matrix.label == 'unpinned' }}
        uses: actions/cache@v3
        id: pre-commit-cache
        with:
          path: ~/.cache/pre-commit
          key: pre-commit|${{ env.PYSHA }}|${{ hashFiles('.pre-commit-config.yaml') }}

      - name: pre-commit install
        if: ${{ matrix.label == 'unpinned' }}
        run: |
          pre-commit install

      - name: pre-commit run
        if: ${{ matrix.label == 'unpinned' }}
        run: |
          pre-commit run --all-files

      # DOCUMENTATION

      - name: Build documentation
        if: ${{ matrix.label == 'unpinned' }}
        run: |
          ./docs/gen_docs dagrunner ./docs

      - name: Check if documentation has changed
        id: check-docs
        if: ${{ matrix.label == 'unpinned' }}
        run: |
          echo "changed=$(git diff --quiet --exit-code || echo true)" | tee -a $GITHUB_OUTPUT

      # https://github.com/orgs/community/discussions/26560#discussioncomment-3531273
      # This must be our very final step to ensure that it runs only on condition of
      # success of all previous steps.  A pushed commit will not trigger the re-running
      # of this workflow.
      - name: Commit and push documentation changes
        if: ${{ matrix.label == 'unpinned' && steps.check-docs.outputs.changed == 'true' }}
        run: |
          git config user.name github-actions[bot]
          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
          git commit -am "Automated reference documentation update for PR ${{ github.event.number }} [skip ci]"
          git push
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}