From 4aac28ecb47f1cbbf5f769d999ef2eb278b3b810 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Tue, 7 May 2024 12:02:44 -0700 Subject: [PATCH 1/9] Switch to manage.py tests (temporarily) This way we can try passing the --parallel flag as a quick way to see if it will work on CI. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e7d920c..ce4f8952 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,7 +78,7 @@ jobs: run: python manage.py collectstatic --noinput --settings=config.settings.test - name: Run tests - run: coverage run -p -m pytest + run: coverage run -p manage.py test --settings=config.settings.test - name: Upload coverage data uses: actions/upload-artifact@v4 From b20d4d435beb3423e18620e88acf70db78a48d5e Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Tue, 7 May 2024 12:29:10 -0700 Subject: [PATCH 2/9] Add the --parallel keyword when testing in CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce4f8952..8468212e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,7 +78,7 @@ jobs: run: python manage.py collectstatic --noinput --settings=config.settings.test - name: Run tests - run: coverage run -p manage.py test --settings=config.settings.test + run: coverage run -p manage.py test --parallel --settings=config.settings.test - name: Upload coverage data uses: actions/upload-artifact@v4 From 1110e67111896839f8759be585165054e0d5b1a1 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Tue, 7 May 2024 13:52:33 -0700 Subject: [PATCH 3/9] Try using parallelization with pytest --- .github/workflows/ci.yml | 2 +- requirements/test-requirements.in | 2 ++ requirements/test-requirements.txt | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8468212e..3a2ee655 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,7 +78,7 @@ jobs: run: python manage.py collectstatic --noinput --settings=config.settings.test - name: Run tests - run: coverage run -p manage.py test --parallel --settings=config.settings.test + run: coverage run -p -m pytest -n auto - name: Upload coverage data uses: actions/upload-artifact@v4 diff --git a/requirements/test-requirements.in b/requirements/test-requirements.in index c5e95951..d20e07ab 100644 --- a/requirements/test-requirements.in +++ b/requirements/test-requirements.in @@ -2,6 +2,8 @@ # Pytest pytest # https://github.com/pytest-dev/pytest +# Parallelized tests +pytest-xdist # https://github.com/pytest-dev/pytest-xdist # Pytest improvements - progress bar, etc. pytest-sugar # https://github.com/Frozenball/pytest-sugar # Mock HTTP responses diff --git a/requirements/test-requirements.txt b/requirements/test-requirements.txt index 1f22702f..1e502a6b 100644 --- a/requirements/test-requirements.txt +++ b/requirements/test-requirements.txt @@ -22,6 +22,8 @@ django-test-migrations==1.3.0 # via -r requirements/test-requirements.in exceptiongroup==1.2.0 # via pytest +execnet==2.1.1 + # via pytest-xdist factory-boy==3.3.0 # via -r requirements/test-requirements.in faker==23.2.1 @@ -59,6 +61,8 @@ pytest-django==4.8.0 # via -r requirements/test-requirements.in pytest-sugar==1.0.0 # via -r requirements/test-requirements.in +pytest-xdist==3.6.1 + # via -r requirements/test-requirements.in python-dateutil==2.8.2 # via # -c requirements/requirements.txt From 29d25cc5c60b14beaeffe335b8c007e200450c8f Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Tue, 7 May 2024 14:04:33 -0700 Subject: [PATCH 4/9] Add a "coverage report" step to CI --- .github/workflows/ci.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a2ee655..4f14b331 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -113,11 +113,14 @@ jobs: run: | mv ./artifacts/coverage-data*/.coverage* . ls -la .coverage* - - - name: Combine coverage data - run: | python -m coverage combine python -m coverage xml + ls -la .coverage* + + - name: Report coverage + run: | + python -m coverage report + - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 with: From a2aa962710dea152ac1cd519060e17c582cb8d82 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Tue, 7 May 2024 15:18:33 -0700 Subject: [PATCH 5/9] Use pytest-cov plugin for coverage This handles the parallel/multithreaded tests better than regular coverage, which excluded a lot of lines (due to parallelization/ combination of coverage reports before they were done?) --- .github/workflows/ci.yml | 2 +- requirements/test-requirements.in | 2 ++ requirements/test-requirements.txt | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f14b331..2b707bad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,7 +78,7 @@ jobs: run: python manage.py collectstatic --noinput --settings=config.settings.test - name: Run tests - run: coverage run -p -m pytest -n auto + run: pytest --cov=primed -n auto - name: Upload coverage data uses: actions/upload-artifact@v4 diff --git a/requirements/test-requirements.in b/requirements/test-requirements.in index d20e07ab..16ef8bcc 100644 --- a/requirements/test-requirements.in +++ b/requirements/test-requirements.in @@ -4,6 +4,8 @@ pytest # https://github.com/pytest-dev/pytest # Parallelized tests pytest-xdist # https://github.com/pytest-dev/pytest-xdist +# Coverage integrated with pytest +pytest-cov # Pytest improvements - progress bar, etc. pytest-sugar # https://github.com/Frozenball/pytest-sugar # Mock HTTP responses diff --git a/requirements/test-requirements.txt b/requirements/test-requirements.txt index 1e502a6b..24234199 100644 --- a/requirements/test-requirements.txt +++ b/requirements/test-requirements.txt @@ -57,6 +57,8 @@ pytest==8.2.0 # -r requirements/test-requirements.in # pytest-django # pytest-sugar +pytest-cov==5.0.0 + # via -r requirements/test-requirements.in pytest-django==4.8.0 # via -r requirements/test-requirements.in pytest-sugar==1.0.0 From ba4c8d8f8a6826cf53d03ae1f240d35145692cbc Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Tue, 7 May 2024 15:32:43 -0700 Subject: [PATCH 6/9] Add debugging step of listing files The CI isn't finding the coverage report to upload... --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b707bad..dc397954 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,6 +80,9 @@ jobs: - name: Run tests run: pytest --cov=primed -n auto + - name: List files for debugging purposes + run: ls -lhta + - name: Upload coverage data uses: actions/upload-artifact@v4 with: From 56fa15d530f9c8e357eb0f69e7c4954c1b5509b1 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Tue, 7 May 2024 15:48:55 -0700 Subject: [PATCH 7/9] Rename output coverage file to fix coverage upload Using pytest-xdist with pytest-cov creates a .coverage file, not a file with the machine etc. suffix. Rename the file after the tests such that it can be uploaded as an artifact with a unique name compared to the coverage runs from the other jobs. --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc397954..64e96ad0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,7 +78,9 @@ jobs: run: python manage.py collectstatic --noinput --settings=config.settings.test - name: Run tests - run: pytest --cov=primed -n auto + run: | + pytest --cov=primed -n auto + mv .coverage .coverage-${{ strategy.job-index }} - name: List files for debugging purposes run: ls -lhta @@ -87,7 +89,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: coverage-data-${{ strategy.job-index }} - path: .coverage.* + path: .coverage-${{ strategy.job-index }} coverage: needs: From c7bb06f7afee93f1245c68f37f85cffbce005d5d Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Tue, 7 May 2024 16:06:07 -0700 Subject: [PATCH 8/9] Try specifying the filenames when running coverage combine --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 64e96ad0..2990fd26 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,9 +116,10 @@ jobs: - name: Merge coverage files run: | + ls -laR ./artifacts mv ./artifacts/coverage-data*/.coverage* . ls -la .coverage* - python -m coverage combine + python -m coverage combine .coverage-* python -m coverage xml ls -la .coverage* From aefc3c5c9cbad94c93c9f3e3c9047e7ce4f6c5c3 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Tue, 7 May 2024 16:21:30 -0700 Subject: [PATCH 9/9] Combine coverage metrics in place instead of moving files --- .github/workflows/ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2990fd26..68d3bacc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,10 +116,7 @@ jobs: - name: Merge coverage files run: | - ls -laR ./artifacts - mv ./artifacts/coverage-data*/.coverage* . - ls -la .coverage* - python -m coverage combine .coverage-* + python -m coverage combine ./artifacts/coverage-data*/.coverage-* python -m coverage xml ls -la .coverage*