ci: split into smaller workflows with some improvements (#1475)

## Summary
This PR splits our big CI workflow into smaller reusable workflows,
enabling a greater degree of parallelization and improve
maintainability.

## Details
Previously, the main workflow runs in stages for each of the tested
platforms. However, this poses a problem where a stage is blocked by the
slowest runner and none of the other runners could move on.

With the release of reusable workflows for Actions, we can now separate
the stages into a different workflow, then parallelize that instead,
allowing faster runners to proceed through test stages ahead of the
rest. The response time for test results should now be greatly improved,
with the M1 runner finishing its run in just 7 minutes.

This separation also opened the doors to customizing how tests are run
for each platforms, and we now only use 1 test batch for *nix while
allocating 3 to Windows. This normalizes test time between Windows and
*nix, while allowing freed up runners to process other jobs.

Lastly, some tiny QoL improvements are added to the pipeline:

- Test failures will now be displayed at the end of "Run tester" step,
reducing the number of clicks required to see all failures to 1.
- The diffoscope container is now used instead of the Ubuntu package,
reducing installation time and let us receive latest features/bug
fixes.
- If there is a diffoscope failure, a test summary is now generated with
a link to the test report. This should make it a lot easier to get to
the report should that be necessary.

.github/workflows/build-and-test.yml

@@ -0,0 +1,224 @@
+name: Build and test
+on:
+  workflow_call:
+    inputs:
+      runner:
+        description: The runner to run the workflow with
+        required: true
+        type: string
+
+      batch-count:
+        description: The number of test batches to parallelize
+        required: false
+        default: 1
+        type: number
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  context:
+    name: Compute context for the workflow
+    runs-on: ubuntu-latest
+    outputs:
+      result: ${{ steps.context.outputs.result }}
+
+    steps:
+      - uses: actions/github-script@v7
+        id: context
+        with:
+          script: |
+            let num_batches = Number.parseInt(process.env.BATCH_COUNT, 10);
+            num_batches = Number.isNaN(num_batches) ? 2 : num_batches;
+            num_batches = Math.max(num_batches, 1);
+
+            const batch_array = Array(num_batches).fill(0).map((_, idx) => ({
+              'idx': idx,
+              'display': idx + 1,
+            }));
+
+            return {
+              'test-batch': batch_array,
+              'num-test-batch': batch_array.length,
+            };
+        env:
+          BATCH_COUNT: ${{ inputs.batch-count }}
+
+  binaries:
+    name: Build release binaries
+    runs-on: ${{ inputs.runner }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          filter: tree:0
+
+      - name: Enable annotations
+        run: echo "::add-matcher::.github/nim-problem-matcher.json"
+
+      - name: Install MinGW (Windows)
+        if: runner.os == 'Windows'
+        uses: ./.github/actions/setup-mingw
+
+      - name: Setup vcpkg (Windows)
+        if: runner.os == 'Windows'
+        uses: ./.github/actions/setup-vcpkg
+        with:
+          triplet: x64-mingw-dynamic-release
+          host-triplet: x64-mingw-dynamic-release
+          revision: 2024.01.12
+          overlay-triplets: ${{ github.workspace }}/tools/vcpkg/triplets
+
+      - name: Install dependencies (Windows)
+        if: runner.os == 'Windows'
+        run: vcpkg install pcre sqlite3
+
+      - name: Set Xcode version (macOS M1)
+        if: runner.os == 'macOS' && runner.arch == 'ARM64'
+        uses: maxim-lobanov/setup-xcode@v1
+        with:
+          xcode-version: "15.0.1"
+
+      - name: Set macOS SDK version (macOS M1)
+        if: runner.os == 'macOS' && runner.arch == 'ARM64'
+        run: |
+          sdkpath=$(xcrun --sdk macosx14.0 --show-sdk-path)
+          echo "SDKROOT=$sdkpath" >> "$GITHUB_ENV"
+
+      - name: Build release binaries
+        run: ./koch.py all-strict
+
+      - name: Upload workspace to artifacts
+        uses: ./.github/actions/upload-compiler
+
+  test:
+    needs: [context, binaries]
+    strategy:
+      fail-fast: false
+
+      matrix:
+        batch: ${{ fromJSON(needs.context.outputs.result).test-batch }}
+        num-batch:
+          - ${{ fromJSON(needs.context.outputs.result).num-test-batch }}
+
+    runs-on: ${{ inputs.runner }}
+    name: Test compiler and stdlib (batch ${{ matrix.batch.display }}/${{ matrix.num-batch }})
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          filter: tree:0
+
+      - uses: ./.github/actions/download-compiler
+
+      - name: Install NodeJS
+        uses: actions/setup-node@v4
+        with:
+          node-version: "16"
+
+      - name: Install dependencies (Linux)
+        if: runner.os == 'Linux'
+        run: |
+          deps=(
+            # Required by ARC/ORC memory leak tests (only enabled on linux x64)
+            libc6-dbg
+            valgrind
+          )
+
+          sudo apt-get update
+          sudo apt-get install "${deps[@]}"
+
+      - name: Set Xcode version (macOS M1)
+        if: runner.os == 'macOS' && runner.arch == 'ARM64'
+        uses: maxim-lobanov/setup-xcode@v1
+        with:
+          xcode-version: "15.0.1"
+
+      - name: Set macOS SDK version (macOS M1)
+        if: runner.os == 'macOS' && runner.arch == 'ARM64'
+        run: |
+          sdkpath=$(xcrun --sdk macosx14.0 --show-sdk-path)
+          echo "SDKROOT=$sdkpath" >> "$GITHUB_ENV"
+
+      - name: Install dependencies (macOS)
+        if: runner.os == 'macOS'
+        run: |
+          brew update
+
+          # Downgrade to OpenSSL 3.0
+          brew unlink openssl
+          brew install [email protected]
+          brew link --overwrite [email protected]
+
+      - name: Add Homebrew libraries to search path
+        if: runner.os == 'macOS'
+        run: echo "DYLD_FALLBACK_LIBRARY_PATH=$(brew --prefix)/lib" >> "$GITHUB_ENV"
+
+      - name: Install MinGW (Windows)
+        if: runner.os == 'Windows'
+        uses: ./.github/actions/setup-mingw
+
+      - name: Add DLLs to PATH (Windows)
+        if: runner.os == 'Windows'
+        run: |
+          $binPath = Join-Path $PWD "vcpkg" "installed" "x64-mingw-dynamic-release" "bin"
+          $binPath | Out-File -Append $env:GITHUB_PATH
+        shell: pwsh
+
+      - name: Run tester
+        run: |
+          if ! ./koch.py test --batch:"$TEST_BATCH" --tryFailing all; then
+            oldExitCode=$?
+            # XXX: Move this to either koch or testament since it's useful.
+            echo ===
+            echo Errors occured during tests:
+            bin/nim r tools/ci_testresults
+            exit $oldExitCode
+          fi
+        env:
+          TEST_BATCH: ${{ matrix.batch.idx }}_${{ matrix.num-batch }}
+
+  tooling:
+    needs: [binaries]
+    name: Build and test tooling
+    runs-on: ${{ inputs.runner }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          filter: tree:0
+
+      - name: Install MinGW (Windows)
+        if: runner.os == 'Windows'
+        uses: ./.github/actions/setup-mingw
+
+      - name: Add DLLs to PATH (Windows)
+        if: runner.os == 'Windows'
+        run: |
+          $binPath = Join-Path $PWD "vcpkg" "installed" "x64-mingw-dynamic-release" "bin"
+          $binPath | Out-File -Append $env:GITHUB_PATH
+        shell: pwsh
+
+      - name: Set Xcode version (macOS M1)
+        if: runner.os == 'macOS' && runner.arch == 'ARM64'
+        uses: maxim-lobanov/setup-xcode@v1
+        with:
+          xcode-version: "15.0.1"
+
+      - name: Set macOS SDK version (macOS M1)
+        if: runner.os == 'macOS' && runner.arch == 'ARM64'
+        run: |
+          sdkpath=$(xcrun --sdk macosx14.0 --show-sdk-path)
+          echo "SDKROOT=$sdkpath" >> "$GITHUB_ENV"
+
+      - uses: ./.github/actions/download-compiler
+
+      - name: Enable annotations
+        run: echo "::add-matcher::.github/nim-problem-matcher.json"
+
+      - name: Test tooling
+        run: ./koch.py testTools