Setup a build workflow with unit tests and code coverage [RT-58] (#3)

* Tweak the build workflow Signed-off-by: Fabrice Normandin <[email protected]> * Move the build workflow Signed-off-by: Fabrice Normandin <[email protected]> * Fix typo in build.yml Signed-off-by: Fabrice Normandin <[email protected]> * Only run on pull requests (not twice) Signed-off-by: Fabrice Normandin <[email protected]> * Use `pdm install` to install dependencies Signed-off-by: Fabrice Normandin <[email protected]> * Use `pdm run` to run tests Signed-off-by: Fabrice Normandin <[email protected]> * Add missing pytest-cov dev dependency Signed-off-by: Fabrice Normandin <[email protected]> * Add some badges to the README.md Signed-off-by: Fabrice Normandin <[email protected]> * Fix failing test in main_test.py Signed-off-by: Fabrice Normandin <[email protected]> * Fix bug in conftest.py when no GPU on host Signed-off-by: Fabrice Normandin <[email protected]> * Round simple stats to avoid minute differences - Seems like the regression tests are failing due to extremely small differences in the statistics like 1e-9 difference. Not sure what's causing this issue though. Signed-off-by: Fabrice Normandin <[email protected]> * Reduce precision further (something fishy going on) Signed-off-by: Fabrice Normandin <[email protected]> * Reduce simple attributes precision further Signed-off-by: Fabrice Normandin <[email protected]> --------- Signed-off-by: Fabrice Normandin <[email protected]>
mila-iqia · May 31, 2024 · 48a033d · 48a033d
1 parent 7d41742
commit 48a033d
Show file tree

Hide file tree

Showing 13 changed files with 236 additions and 106 deletions.
diff --git a/.github/build.yml b/.github/build.yml
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -0,0 +1,110 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python application
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+# https://stackoverflow.com/a/72408109/6388696
+# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-concurrency-to-cancel-any-in-progress-job-or-run
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  linting:
+    name: Run linting/pre-commit checks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+      - run: pip install pre-commit
+      - run: pre-commit --version
+      - run: pre-commit install
+      - run: pre-commit run --all-files
+
+  unit_tests:
+    needs: [linting]
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      max-parallel: 4
+      matrix:
+        platform: [ubuntu-latest]
+        python-version: ['3.12']
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - run: pip install pdm
+    - name: Install dependencies
+      run: pdm install
+    - name: Test with pytest (very fast)
+      run: pdm run pytest -v --shorter-than=1.0 --cov=project --cov-report=xml --cov-append
+    - name: Test with pytest (fast)
+      run: pdm run pytest -v --cov=project --cov-report=xml --cov-append
+
+    - name: Store coverage report as an artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: coverage-reports-unit-tests-${{ matrix.platform }}-${{ matrix.python-version }}
+        path: ./coverage.xml
+
+  integration_tests:
+    needs: [unit_tests]
+    runs-on: self-hosted
+    strategy:
+      max-parallel: 1
+      matrix:
+        python-version: ['3.12']
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - run: pip install pdm
+    - name: Install dependencies
+      run: pdm install
+
+    - name: Test with pytest
+      run: pdm run pytest -v --cov=project --cov-report=xml --cov-append
+
+    - name: Test with pytest (only slow tests)
+      run: pdm run pytest -v -m slow --slow --cov=project --cov-report=xml --cov-append
+
+    - name: Store coverage report as an artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: coverage-reports-integration-tests-${{ matrix.python-version }}
+        path: ./coverage.xml
+
+  # https://about.codecov.io/blog/uploading-code-coverage-in-a-separate-job-on-github-actions/
+  upload-coverage-codecov:
+    needs: [integration_tests]
+    runs-on: ubuntu-latest
+    name: Upload coverage reports to Codecov
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: coverage-reports-*
+          merge-multiple: false
+          # download all the artifacts in this directory (each .coverage.xml will be in a subdirectory)
+          # Next step if this doesn't work would be to give the coverage files a unique name and use merge-multiple: true
+          path: coverage_reports
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          directory: coverage_reports
+          fail_ci_if_error: true
diff --git a/README.md b/README.md
@@ -1 +1,4 @@
 # research_template
+
+![Build](https://github.com/mila-iqia/ResearchTemplate/workflows/build.yml/badge.svg)
+[![codecov](https://codecov.io/gh/mila-iqia/ResearchTemplate/graph/badge.svg?token=I2DYLK8NTD)](https://codecov.io/gh/mila-iqia/ResearchTemplate)
diff --git a/pdm.lock b/pdm.lock
diff --git a/project/conftest.py b/project/conftest.py
@@ -5,7 +5,7 @@
 import sys
 import typing
 import warnings
-from collections.abc import Callable, Generator
+from collections.abc import Generator
 from contextlib import contextmanager
 from logging import getLogger as get_logger
 from pathlib import Path
@@ -27,7 +27,7 @@
 from project.datamodules.image_classification import (
     ImageClassificationDataModule,
 )
-from project.datamodules.vision.base import VisionDataModule, num_cpus_on_node
+from project.datamodules.vision.base import VisionDataModule
 from project.experiment import (
     instantiate_algorithm,
     instantiate_datamodule,
@@ -183,9 +183,6 @@ def accelerator(request: pytest.FixtureRequest):
     return accelerator
 
 
-_cuda_available = torch.cuda.is_available()
-
-
 @pytest.fixture(
     scope="session",
     params=None,
@@ -198,24 +195,7 @@ def num_devices_to_use(accelerator: str, request: pytest.FixtureRequest) -> int:
         return num_gpus  # Use only one GPU by default.
     else:
         assert accelerator == "cpu"
-        return request.param
-
-
-def run_with_multiple_devices(test_fn: Callable) -> pytest.MarkDecorator:
-    if torch.cuda.is_available():
-        gpus = torch.cuda.device_count()
-        return pytest.mark.parametrize(
-            num_devices_to_use.__name__,
-            list(range(1, gpus + 1)),
-            indirect=True,
-            ids=[f"gpus={i}" for i in range(1, gpus + 1)],
-        )
-    return pytest.mark.parametrize(
-        num_devices_to_use.__name__,
-        [num_cpus_on_node()],
-        indirect=True,
-        ids=[""],
-    )(test_fn)
+        return getattr(request, "param", 1)
 
 
 @pytest.fixture(scope="session")

diff --git a/project/datamodules/datamodules_test.py b/project/datamodules/datamodules_test.py
@@ -70,7 +70,7 @@ def test_first_batch(
 
     fig.suptitle(f"First batch of datamodule {type(datamodule).__name__}")
     figure_path, _ = get_test_source_and_temp_file_paths(
-        ".png", request=request, original_datadir=original_datadir, datadir=datadir
+        extension=".png", request=request, original_datadir=original_datadir, datadir=datadir
     )
     figure_path.parent.mkdir(exist_ok=True, parents=True)
     fig.savefig(figure_path)

diff --git a/project/datamodules/datamodules_test/test_first_batch/cifar10.yaml b/project/datamodules/datamodules_test/test_first_batch/cifar10.yaml
@@ -1,20 +1,20 @@
 '0':
   device: cpu
   hash: 1082905456378942323
-  max: 2.125603675842285
-  mean: -0.007423439994454384
-  min: -1.9888888597488403
+  max: 2.126
+  mean: -0.007
+  min: -1.989
   shape:
   - 128
   - 3
   - 32
   - 32
-  sum: -2919.015380859375
+  sum: -2919.015
 '1':
   device: cpu
   hash: 3692171093056153318
   max: 9
-  mean: 4.5546875
+  mean: 4.555
   min: 0
   shape:
   - 128

diff --git a/project/datamodules/datamodules_test/test_first_batch/fashion_mnist.yaml b/project/datamodules/datamodules_test/test_first_batch/fashion_mnist.yaml
@@ -1,20 +1,20 @@
 '0':
   device: cpu
   hash: -3706536913713083016
-  max: 2.821486711502075
-  mean: 0.47488248348236084
-  min: -0.4242129623889923
+  max: 2.821
+  mean: 0.475
+  min: -0.424
   shape:
   - 128
   - 1
   - 28
   - 28
-  sum: 47655.40625
+  sum: 47655.406
 '1':
   device: cpu
   hash: -4023601292826392021
   max: 9
-  mean: 4.5546875
+  mean: 4.555
   min: 0
   shape:
   - 128

diff --git a/project/datamodules/datamodules_test/test_first_batch/mnist.yaml b/project/datamodules/datamodules_test/test_first_batch/mnist.yaml
@@ -1,20 +1,20 @@
 '0':
   device: cpu
   hash: 4338584025941619046
-  max: 2.821486711502075
-  mean: 0.014241953380405903
-  min: -0.4242129623889923
+  max: 2.821
+  mean: 0.014
+  min: -0.424
   shape:
   - 128
   - 1
   - 28
   - 28
-  sum: 1429.20849609375
+  sum: 1429.208
 '1':
   device: cpu
   hash: 1596942422053415325
   max: 9
-  mean: 4.2421875
+  mean: 4.242
   min: 0
   shape:
   - 128