Add reusable workflow used in build and DRAC tests

- Extracts the integration test part of `build.yml` into a reusable
  workflow called `testing.yml`.
- This workflow now also explicitly requires an existing SSH connection
  to be alive (a socket at the ControlPath of the SSH config) in order
  to run tests on the DRAC clusters
- Also adds a new `full_cluster_tests.yml` workflow that is run
  periodically (once a week) and runs the integration tests on all slurm
  clusters (not just `'mila'`, as in `build.yml`).

Signed-off-by: Fabrice Normandin <[email protected]>

.github/workflows/build.yml

@@ -143,43 +143,20 @@ jobs:
           fail_ci_if_error: false
 
   real-slurm-integration-tests:
+
     name: integration tests with a real SLURM cluster
     needs: [mock-slurm-integration-tests]
-    runs-on: self-hosted
 
     strategy:
-      max-parallel: 5
+      max-parallel: 1
       matrix:
         # TODO: We should ideally also run this with Windows/Mac clients and a Linux
         # server. Unsure how to set that up with GitHub Actions though.
-        python-version: ['3.8', '3.9', '3.10', '3.11']
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install poetry
-          poetry install --with=dev
-
-      - name: Launch integration tests
-        id: self_hosted_integration_tests
-        run: poetry run pytest --slow --cov=milatools --cov-report=xml --cov-append -vvv --log-level=DEBUG
-        timeout-minutes: 30
-        env:
-          SLURM_CLUSTER: mila
-
-      - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./coverage.xml
-          flags: integrationtests
-          env_vars: PLATFORM,PYTHON
-          name: codecov-umbrella
-          fail_ci_if_error: false
+        python-version: ['3.11']
+        cluster: ['mila']
+
+    uses: ./.github/workflows/testing.yml
+    with:
+      cluster: ${{ matrix.cluster }}
+      python-version: ${{ matrix.python-version }}
+    secrets: inherit

.github/workflows/full_cluster_tests.yml

@@ -0,0 +1,32 @@
+# Run integration tests on a self-hosted runner using all slurm clusters.
+on:
+  push:
+    branches: "master"  # every time a push is made to `master`, OR
+  schedule:
+    - cron: "30 6 * * 1"  # every Monday at 6:30 AM UTC (2:30 AM Montreal time) OR
+  workflow_dispatch:  # when the workflow is manually triggered
+
+# https://stackoverflow.com/a/72408109/6388696
+# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-concurrency-to-cancel-any-in-progress-job-or-run
+concurrency:
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: true
+
+jobs:
+  real-slurm-integration-tests:
+    name: integration tests with a real SLURM cluster
+    strategy:
+      max-parallel: 5
+      matrix:
+        # TODO: We should ideally also run this with Windows/Mac clients and a Linux
+        # server. Unsure how to set that up with GitHub Actions though.
+        python-version: ['3.11']
+        # TODO: also run these integration tests on the DRAC clusters (probably less
+        # frequently because of the long job wait times)
+        cluster: ['mila', 'narval', 'beluga', 'cedar', 'graham']
+
+    uses: ./.github/workflows/testing.yml
+    with:
+      cluster: ${{ matrix.cluster }}
+      python-version: ${{ matrix.python-version }}
+    secrets: inherit

.github/workflows/testing.yml

@@ -0,0 +1,60 @@
+on:
+  workflow_call:
+    inputs:
+      cluster:
+        required: true
+        type: string
+      python-version:
+        required: false
+        type: string
+        default: '3.11'
+    secrets:
+      CODECOV_TOKEN:
+        required: true
+
+  workflow_dispatch:
+
+
+jobs:
+  real-slurm-integration-tests:
+    name: integration tests with a real SLURM cluster
+    runs-on: self-hosted
+
+    steps:
+      - name: Check that we have the pre-existing connection to the SLURM cluster.
+        # TODO: mila cluster doesn't use 2FA yet, so we can actually create the connection
+        # to run the tests; we don't need it to be already running.
+        if: ${{ inputs.cluster != 'mila' }}
+        run:
+            # Check that the control socket is running on the self-hosted runner so
+            # that we don't have to go through 2FA on DRAC clusters.
+            ssh -O check -oStrictHostKeyChecking=no ${{ inputs.cluster }}
+
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ inputs.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ inputs.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install poetry
+          poetry install --with=dev
+
+      - name: Launch integration tests
+        id: self_hosted_integration_tests
+        run: poetry run pytest --slow --cov=milatools --cov-report=xml --cov-append -vvv --log-level=DEBUG
+        timeout-minutes: 30
+        env:
+          SLURM_CLUSTER: ${{ inputs.cluster }}
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ./coverage.xml
+          flags: integrationtests
+          env_vars: PLATFORM,PYTHON
+          name: codecov-umbrella
+          fail_ci_if_error: false