codeql_unit_tests.yml

name: CodeQL Unit Testing

on:
  merge_group:
    types: [checks_requested]
  push:
    branches:
      - main
      - next
      - "rc/**"
  pull_request:
    branches:
      - main
      - next
      - "rc/**"

jobs:
 
  prepare-unit-test-matrix:
    name: Prepare CodeQL unit test matrix
    runs-on: ubuntu-22.04
    outputs:
      matrix: ${{ steps.export-unit-test-matrix.outputs.matrix }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Export unit test matrix
        id: export-unit-test-matrix
        run: |
          echo "Merging Result:"
          python scripts/create_language_matrix.py
          echo "matrix=$(
            python scripts/create_language_matrix.py | \
            jq --compact-output 'map([.+{os: "ubuntu-latest-xl", codeql_standard_library_ident : .codeql_standard_library | sub("\/"; "_")}]) | flatten | {include: .}')" >> $GITHUB_OUTPUT

  run-test-suites:
    name: Run unit tests
    needs: prepare-unit-test-matrix

    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.prepare-unit-test-matrix.outputs.matrix) }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Install Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - name: Install Python dependencies
        run: pip install -r scripts/requirements.txt

      - name: Cache CodeQL
        id: cache-codeql
        uses: actions/cache@v3
        with:
          # A list of files, directories, and wildcard patterns to cache and restore
          path: ${{github.workspace}}/codeql_home
          # An explicit key for restoring and saving the cache
          key: codeql-home-${{matrix.os}}-${{matrix.codeql_cli}}-${{matrix.codeql_standard_library}}

      - name: Install CodeQL
        if: steps.cache-codeql.outputs.cache-hit != 'true'
        uses: ./.github/actions/install-codeql
        with:
          codeql-cli-version: ${{ matrix.codeql_cli }}
          codeql-stdlib-version: ${{ matrix.codeql_standard_library }}
          codeql-home: ${{ github.workspace }}/codeql_home
          add-to-path: false

      - name: Install CodeQL packs
        uses: ./.github/actions/install-codeql-packs
        with:
          cli_path: ${{ github.workspace }}/codeql_home/codeql

      - name: Pre-Compile Queries
        id: pre-compile-queries
        run: |
         ${{ github.workspace }}/codeql_home/codeql/codeql query compile --threads 0 ${{ matrix.language }}


      - name: Run test suites
        id: run-test-suites
        env:
          RUNNER_OS: ${{ runner.os }}
          RUNNER_TEMP: ${{ runner.temp }}
          CODEQL_CLI: ${{ matrix.codeql_cli }}
          CODEQL_STDLIB: ${{ matrix.codeql_standard_library }}
          CODEQL_STDLIB_IDENT: ${{matrix.codeql_standard_library_ident}}
          CODEQL_HOME: ${{ github.workspace }}/codeql_home
        shell: python
        run: |
          import os
          import sys
          import subprocess
          import re
          import json
          from pathlib import Path

          def print_error(fmt, *args):
            print(f"::error::{fmt}", *args)

          def print_error_and_fail(fmt, *args):
            print_error(fmt, args)
            sys.exit(1)

          cli_version = os.environ['CODEQL_CLI']
          stdlib_ref = os.environ['CODEQL_STDLIB']
          stdlib_ref_ident = os.environ['CODEQL_STDLIB_IDENT']
          workspace = os.environ['GITHUB_WORKSPACE']
          runner_os = os.environ['RUNNER_OS']
          runner_temp = os.environ['RUNNER_TEMP']
          codeql_home = os.environ['CODEQL_HOME']
          codeql_bin = os.path.join(codeql_home, 'codeql', 'codeql')

          language_root = Path(workspace, '${{ matrix.language }}')
          test_roots = list(map(str, language_root.glob('*/test')))
          for test_root in test_roots:
            print(f"Executing tests found (recursively) in the directory '{test_root}'")
          files_to_close = []
          try:
            # XL runners have 8 cores, so split the tests into 8 "slices", and run one per thread
            num_slices = 8
            procs = []

            for slice in range(1, num_slices+1):
              test_report_path = os.path.join(runner_temp, "${{ matrix.language }}", f"test_report_{runner_os}_{cli_version}_{stdlib_ref_ident}_slice_{slice}_of_{num_slices}.json")
              os.makedirs(os.path.dirname(test_report_path), exist_ok=True)
              test_report_file = open(test_report_path, 'w')
              files_to_close.append(test_report_file)
              procs.append(subprocess.Popen([codeql_bin, "test", "run", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", *test_roots], stdout=test_report_file, stderr=subprocess.PIPE))

            for p in procs:
              _, err = p.communicate()
              if p.returncode != 0:
                if p.returncode == 122:
                  # Failed because a test case failed, so just print the regular output.
                  # This will allow us to proceed to validate-test-results, which will fail if
                  # any test cases failed
                  print(f"{err.decode()}")
                else:
                  # Some more serious problem occurred, so print and fail fast
                  print_error_and_fail(f"Failed to run tests with return code {p.returncode}\n{err.decode()}")
          finally:
            for file in files_to_close:
              file.close()

      - name: Upload test results
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.language }}-test-results-${{ runner.os }}-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library_ident }}
          path: |
            ${{ runner.temp }}/${{ matrix.language }}/test_report_${{ runner.os }}_${{ matrix.codeql_cli }}_${{ matrix.codeql_standard_library_ident }}_slice_*.json
          if-no-files-found: error

  validate-test-results:
    name: Validate test results
    if: ${{ always() }}
    needs: run-test-suites
    runs-on: ubuntu-22.04
    steps:
      - name: Check if run-test-suites job failed to complete, if so fail
        if: ${{ needs.run-test-suites.result == 'failure' }}
        uses: actions/github-script@v3
        with:
          script: |
            core.setFailed('Test run job failed')
      - name: Collect test results
        uses: actions/download-artifact@v4

      - name: Validate test results
        run: |
          for json_report in *-test-results-*/test_report_*
          do
            jq --raw-output '"PASS \(map(select(.pass == true)) | length)/\(length)'" $json_report\"" "$json_report"
          done
          FAILING_TESTS=$(jq --raw-output '.[] | select(.pass == false)' *-test-results-*/test_report_*.json)
          if [[ ! -z "$FAILING_TESTS" ]]; then
            echo "ERROR: The following tests failed:"
            echo $FAILING_TESTS | jq .
            exit 1
          fi