From 3263425da6411e3b2150d0dd67a536049d9116a3 Mon Sep 17 00:00:00 2001 From: Mike Sullivan Date: Fri, 1 Mar 2024 16:27:26 +0000 Subject: [PATCH 01/18] added --run-tests open in pytest --- Makefile | 4 ++-- conftest.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index cffd26fd75f..87d8ba355c0 100644 --- a/Makefile +++ b/Makefile @@ -43,10 +43,10 @@ install-dev-requirements: python ./setup.py create_dev_env test: - python -m pytest -n auto + python -m pytest -n auto --run-tests test-verbose: - python -m pytest -vs -o log_cli=true + python -m pytest -vs -o log_cli=true --run-tests test-system: ${XVFBRUN} python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-system-tests diff --git a/conftest.py b/conftest.py index 897272ec8ce..92acdd0877e 100644 --- a/conftest.py +++ b/conftest.py @@ -13,18 +13,26 @@ def _test_gui_system_filename_match(basename: str) -> bool: return "gui_system" in basename and "_test.py" in basename +def _test_normal_filename_match(path) -> bool: + return "gui_system" not in path.basename and "eyes_tests" not in str(path) and "_test.py" in path.basename + + def pytest_addoption(parser): parser.addoption("--run-system-tests", action="store_true", default=False, help="Run GUI system tests") + parser.addoption("--run-tests", action="store_true", default=False, help="Run normal tests") def pytest_configure(config): config.addinivalue_line("markers", "system: GUI system tests") + config.addinivalue_line("markers", "normal: normal tests") def pytest_ignore_collect(path, config): # When running GUI system tests, ignore all other files if config.getoption("--run-system-tests") and path.isfile() and not _test_gui_system_filename_match(path.basename): return True + elif config.getoption("--run-tests") and path.isfile() and not _test_normal_filename_match(path): + return True else: return False @@ -35,6 +43,11 @@ def pytest_collection_modifyitems(config, items): for item in items: if "system" in item.keywords: item.add_marker(skip_system) + elif not config.getoption("--run-tests"): + skip_system = pytest.mark.skip(reason="use --run-tests option to run") + for item in items: + if "system" not in item.keywords and "eyes_tests" not in item.path: + item.add_marker(skip_system) # Leak track for tests From 7efcfbbafe32f770ada23351eb124724b4d5fb35 Mon Sep 17 00:00:00 2001 From: Mike Sullivan Date: Fri, 1 Mar 2024 16:46:02 +0000 Subject: [PATCH 02/18] release note added --- docs/release_notes/next/dev-2082-pytest_separation | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/release_notes/next/dev-2082-pytest_separation diff --git a/docs/release_notes/next/dev-2082-pytest_separation b/docs/release_notes/next/dev-2082-pytest_separation new file mode 100644 index 00000000000..a7db586e305 --- /dev/null +++ b/docs/release_notes/next/dev-2082-pytest_separation @@ -0,0 +1 @@ +#2082: Normal, System, and Screenshot test are now explicitly separated in the Makefile \ No newline at end of file From 06c86ea59916c5c0e8f596023944f68e402e0f56 Mon Sep 17 00:00:00 2001 From: Mike Sullivan Date: Tue, 5 Mar 2024 11:44:07 +0000 Subject: [PATCH 03/18] custom flags now use custom markers to select appropriate tests --- Makefile | 8 ++++---- conftest.py | 52 ++++++++++++++++++++++++---------------------------- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/Makefile b/Makefile index 87d8ba355c0..4d687478dda 100644 --- a/Makefile +++ b/Makefile @@ -43,22 +43,22 @@ install-dev-requirements: python ./setup.py create_dev_env test: - python -m pytest -n auto --run-tests + python -m pytest -n auto --run-unit-tests test-verbose: - python -m pytest -vs -o log_cli=true --run-tests + python -m pytest -vs -o log_cli=true --run-unit-tests test-system: ${XVFBRUN} python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-system-tests test-screenshots: -mkdir ${TEST_RESULT_DIR} - APPLITOOLS_API_KEY=local APPLITOOLS_IMAGE_DIR=${TEST_RESULT_DIR} ${XVFBRUN} pytest -p no:xdist -p no:randomly -p no:cov mantidimaging/eyes_tests/ -vs + APPLITOOLS_API_KEY=local APPLITOOLS_IMAGE_DIR=${TEST_RESULT_DIR} ${XVFBRUN} pytest -p no:xdist -p no:randomly -p no:cov mantidimaging/eyes_tests/ -vs --run-eyes-tests @echo "Screenshots writen to" ${TEST_RESULT_DIR} test-screenshots-win: -mkdir ${TEST_RESULT_DIR} - ${XVFBRUN} pytest -p no:xdist -p no:randomly -p no:cov mantidimaging/eyes_tests/ -vs + ${XVFBRUN} pytest -p no:xdist -p no:randomly -p no:cov mantidimaging/eyes_tests/ -vs --run-eyes-tests @echo "Screenshots writen to" ${TEST_RESULT_DIR} mypy: diff --git a/conftest.py b/conftest.py index 92acdd0877e..9bd811980cc 100644 --- a/conftest.py +++ b/conftest.py @@ -9,45 +9,41 @@ from mantidimaging.core.utility.leak_tracker import leak_tracker -def _test_gui_system_filename_match(basename: str) -> bool: - return "gui_system" in basename and "_test.py" in basename - - -def _test_normal_filename_match(path) -> bool: - return "gui_system" not in path.basename and "eyes_tests" not in str(path) and "_test.py" in path.basename - - def pytest_addoption(parser): parser.addoption("--run-system-tests", action="store_true", default=False, help="Run GUI system tests") - parser.addoption("--run-tests", action="store_true", default=False, help="Run normal tests") + parser.addoption("--run-unit-tests", action="store_true", default=False, help="Run unit tests") + parser.addoption("--run-eyes-tests", action="store_true", default=False, help="Run eyes tests") def pytest_configure(config): config.addinivalue_line("markers", "system: GUI system tests") - config.addinivalue_line("markers", "normal: normal tests") + config.addinivalue_line("markers", "unit: unit tests") + config.addinivalue_line("markers", "eyes: eyes tests") -def pytest_ignore_collect(path, config): - # When running GUI system tests, ignore all other files - if config.getoption("--run-system-tests") and path.isfile() and not _test_gui_system_filename_match(path.basename): - return True - elif config.getoption("--run-tests") and path.isfile() and not _test_normal_filename_match(path): - return True - else: - return False +allowed_markers = [] +skipped_tests = [] def pytest_collection_modifyitems(config, items): - if not config.getoption("--run-system-tests"): - skip_system = pytest.mark.skip(reason="use --run-system-tests option to run") - for item in items: - if "system" in item.keywords: - item.add_marker(skip_system) - elif not config.getoption("--run-tests"): - skip_system = pytest.mark.skip(reason="use --run-tests option to run") - for item in items: - if "system" not in item.keywords and "eyes_tests" not in item.path: - item.add_marker(skip_system) + if config.getoption("--run-system-tests"): + allowed_markers.append(pytest.mark.system.mark) + if config.getoption("--run-unit-tests"): + allowed_markers.append(pytest.mark.unit.mark) + if config.getoption("--run-eyes-tests"): + allowed_markers.append(pytest.mark.eyes.mark) + for item in items: + if all(test not in item.nodeid for test in ["gui_system", "eyes_test"]): + item.add_marker(pytest.mark.unit) + if "gui_system" in item.nodeid: + item.add_marker(pytest.mark.system) + if "eyes_test" in item.nodeid: + item.add_marker(pytest.mark.eyes) + if any(mark in allowed_markers for mark in item.own_markers): + pass + else: + item.add_marker(pytest.mark.skip(reason="Test not selected")) + skipped_tests.append(item.nodeid) # Leak track for tests From 1757f6038d899bc23d1e209bb57d8ef95ab4f8d0 Mon Sep 17 00:00:00 2001 From: Mike Sullivan Date: Tue, 5 Mar 2024 15:08:07 +0000 Subject: [PATCH 04/18] Windows.yml uses makefile commands --- .github/workflows/windows.yml | 6 +++--- Makefile | 9 +++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 9be6283e12a..e30fce28633 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -87,7 +87,7 @@ jobs: timeout-minutes: 10 shell: bash -l {0} run: | - python -m pytest --cov --cov-report=xml -n auto -o log_cli=true --ignore=mantidimaging/eyes_tests --durations=10 + make test-gh-actions - name: Get test data shell: bash -l {0} @@ -100,7 +100,7 @@ jobs: - name: GUI Tests System shell: bash -l {0} run: | - python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-system-tests --durations=10 + make test-gh-system timeout-minutes: 15 - name: Set display resolution for screenshot tests @@ -116,7 +116,7 @@ jobs: APPLITOOLS_BATCH_ID: ${{ github.sha }} GITHUB_BRANCH_NAME: ${{ github.head_ref }} run: | - python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true mantidimaging/eyes_tests --durations=10 + make test-gh-screenshots timeout-minutes: 15 # Label as 'windows-build-test' for testing purposes. diff --git a/Makefile b/Makefile index 4d687478dda..b472179f105 100644 --- a/Makefile +++ b/Makefile @@ -48,9 +48,15 @@ test: test-verbose: python -m pytest -vs -o log_cli=true --run-unit-tests +test-gh-actions: + python -m pytest --cov --cov-report=xml -n auto -o log_cli=true --run-unit-tests --durations=10 + test-system: ${XVFBRUN} python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-system-tests +test-gh-system: + python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-system-tests --durations=10 + test-screenshots: -mkdir ${TEST_RESULT_DIR} APPLITOOLS_API_KEY=local APPLITOOLS_IMAGE_DIR=${TEST_RESULT_DIR} ${XVFBRUN} pytest -p no:xdist -p no:randomly -p no:cov mantidimaging/eyes_tests/ -vs --run-eyes-tests @@ -61,6 +67,9 @@ test-screenshots-win: ${XVFBRUN} pytest -p no:xdist -p no:randomly -p no:cov mantidimaging/eyes_tests/ -vs --run-eyes-tests @echo "Screenshots writen to" ${TEST_RESULT_DIR} +test-gh-screenshots: + python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-eyes-tests --durations=10 + mypy: python -m mypy --ignore-missing-imports --no-site-packages ${SOURCE_DIRS} From 11885f8f91a8814b5dfd750ff9c683f64adc2ae4 Mon Sep 17 00:00:00 2001 From: Mike Sullivan Date: Tue, 5 Mar 2024 15:19:05 +0000 Subject: [PATCH 05/18] tabs --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index b472179f105..d8f4def3f71 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ build-conda-package-release: .remind-for-user .remind-for-anaconda-api install-b install-dev-requirements: - python ./setup.py create_dev_env + python ./setup.py create_dev_env test: python -m pytest -n auto --run-unit-tests @@ -49,13 +49,13 @@ test-verbose: python -m pytest -vs -o log_cli=true --run-unit-tests test-gh-actions: - python -m pytest --cov --cov-report=xml -n auto -o log_cli=true --run-unit-tests --durations=10 + python -m pytest --cov --cov-report=xml -n auto -o log_cli=true --run-unit-tests --durations=10 test-system: ${XVFBRUN} python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-system-tests test-gh-system: - python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-system-tests --durations=10 + python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-system-tests --durations=10 test-screenshots: -mkdir ${TEST_RESULT_DIR} @@ -68,7 +68,7 @@ test-screenshots-win: @echo "Screenshots writen to" ${TEST_RESULT_DIR} test-gh-screenshots: - python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-eyes-tests --durations=10 + python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-eyes-tests --durations=10 mypy: python -m mypy --ignore-missing-imports --no-site-packages ${SOURCE_DIRS} From 17f7a97954ddb21dd80f744ce84a433cbb22381d Mon Sep 17 00:00:00 2001 From: Mike Sullivan Date: Tue, 5 Mar 2024 15:30:03 +0000 Subject: [PATCH 06/18] Makefile separator fix --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d8f4def3f71..287e821e4e9 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ build-conda-package-release: .remind-for-user .remind-for-anaconda-api install-b install-dev-requirements: - python ./setup.py create_dev_env + python ./setup.py create_dev_env test: python -m pytest -n auto --run-unit-tests From c8de4f8ed1abdad2e1e015d1d7216a863bd5a9ad Mon Sep 17 00:00:00 2001 From: Mike Sullivan Date: Tue, 5 Mar 2024 16:41:08 +0000 Subject: [PATCH 07/18] Applitools api key made local when undefined --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 287e821e4e9..3969aed324f 100644 --- a/Makefile +++ b/Makefile @@ -8,10 +8,13 @@ SOURCE_DIRS=mantidimaging scripts docs/ext/ CHANNELS=$(shell cat environment.yml | sed -ne '/channels:/,/dependencies:/{//!p}' | grep '^ -' | sed 's/ - / --append channels /g' | tr -d '\n') ifeq ($(OS),Windows_NT) + ifndef $(APPLITOOLS_API_KEY) + $(info 'No Applitools API key has been found, switching to local') XVFBRUN= TEST_RESULT_DIR:=$(TEMP)\mantidimaging_tests export APPLITOOLS_API_KEY=local export APPLITOOLS_IMAGE_DIR:=${TEST_RESULT_DIR} + endif else XVFBRUN=xvfb-run --auto-servernum TEST_RESULT_DIR:=$(shell mktemp -d) From f6810bf80fb6744590c800cc9fc14e090d44c029 Mon Sep 17 00:00:00 2001 From: Mike Sullivan Date: Tue, 5 Mar 2024 17:17:27 +0000 Subject: [PATCH 08/18] issues setting temp screenshot files --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 3969aed324f..fdc857719be 100644 --- a/Makefile +++ b/Makefile @@ -10,10 +10,11 @@ CHANNELS=$(shell cat environment.yml | sed -ne '/channels:/,/dependencies:/{//!p ifeq ($(OS),Windows_NT) ifndef $(APPLITOOLS_API_KEY) $(info 'No Applitools API key has been found, switching to local') + $(info variable TEMP = $(TEMP)) XVFBRUN= - TEST_RESULT_DIR:=$(TEMP)\mantidimaging_tests - export APPLITOOLS_API_KEY=local - export APPLITOOLS_IMAGE_DIR:=${TEST_RESULT_DIR} + TEST_RESULT_DIR:=$(TEMP)\mantidimaging_tests + export APPLITOOLS_API_KEY=local + export APPLITOOLS_IMAGE_DIR:=${TEST_RESULT_DIR} endif else XVFBRUN=xvfb-run --auto-servernum From 8f77642556d5b1169faf6f142e197c092d6260a4 Mon Sep 17 00:00:00 2001 From: Mike Sullivan Date: Tue, 5 Mar 2024 17:31:06 +0000 Subject: [PATCH 09/18] debug --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index fdc857719be..6121b5973a4 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,8 @@ ifeq ($(OS),Windows_NT) TEST_RESULT_DIR:=$(TEMP)\mantidimaging_tests export APPLITOOLS_API_KEY=local export APPLITOOLS_IMAGE_DIR:=${TEST_RESULT_DIR} + $(info variable APPLITOOLS_API_KEY = $(APPLITOOLS_API_KEY)) + $(info variable APPLITOOLS_IMAGE_DIR = $(APPLITOOLS_IMAGE_DIR)) endif else XVFBRUN=xvfb-run --auto-servernum From 82691923f95d58a512cfdc5768b6500e564a340c Mon Sep 17 00:00:00 2001 From: Mike Sullivan Date: Tue, 5 Mar 2024 18:56:10 +0000 Subject: [PATCH 10/18] debug --- mantidimaging/eyes_tests/base_eyes.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mantidimaging/eyes_tests/base_eyes.py b/mantidimaging/eyes_tests/base_eyes.py index 3dac982df0e..76f56bbc625 100644 --- a/mantidimaging/eyes_tests/base_eyes.py +++ b/mantidimaging/eyes_tests/base_eyes.py @@ -51,6 +51,10 @@ QApplication.setFont(QFont("Sans Serif", 10)) +print(f"base_eyes.py: {APPLITOOLS_BATCH_ID=}") +print(f"base_eyes.py: {API_KEY_PRESENT=}") +print(f"base_eyes.py: {TEST_NAME=}") +print(f"base_eyes.py: {APPLITOOLS_IMAGE_DIR=}") @unittest.skipIf(API_KEY_PRESENT is None, "API Key is not defined in the environment, so Eyes tests are skipped.") @unittest.skipUnless(os.path.exists(LOAD_SAMPLE), LOAD_SAMPLE_MISSING_MESSAGE) From 3edc33b9637ccaf98855bf05d25b41cb66a83de4 Mon Sep 17 00:00:00 2001 From: Mike Sullivan Date: Wed, 6 Mar 2024 08:55:27 +0000 Subject: [PATCH 11/18] debug --- mantidimaging/eyes_tests/base_eyes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mantidimaging/eyes_tests/base_eyes.py b/mantidimaging/eyes_tests/base_eyes.py index 76f56bbc625..26173d1e717 100644 --- a/mantidimaging/eyes_tests/base_eyes.py +++ b/mantidimaging/eyes_tests/base_eyes.py @@ -56,6 +56,7 @@ print(f"base_eyes.py: {TEST_NAME=}") print(f"base_eyes.py: {APPLITOOLS_IMAGE_DIR=}") + @unittest.skipIf(API_KEY_PRESENT is None, "API Key is not defined in the environment, so Eyes tests are skipped.") @unittest.skipUnless(os.path.exists(LOAD_SAMPLE), LOAD_SAMPLE_MISSING_MESSAGE) @start_qapplication From 764e8238e6854c89342abbacceba284c4ae24860 Mon Sep 17 00:00:00 2001 From: Mike Sullivan Date: Wed, 6 Mar 2024 14:12:06 +0000 Subject: [PATCH 12/18] debug --- .github/workflows/windows.yml | 2 +- Makefile | 26 ++++++++++++-------------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index e30fce28633..5e0266170c4 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -87,7 +87,7 @@ jobs: timeout-minutes: 10 shell: bash -l {0} run: | - make test-gh-actions + make test-gh-unit - name: Get test data shell: bash -l {0} diff --git a/Makefile b/Makefile index 6121b5973a4..362b89a74dc 100644 --- a/Makefile +++ b/Makefile @@ -8,21 +8,20 @@ SOURCE_DIRS=mantidimaging scripts docs/ext/ CHANNELS=$(shell cat environment.yml | sed -ne '/channels:/,/dependencies:/{//!p}' | grep '^ -' | sed 's/ - / --append channels /g' | tr -d '\n') ifeq ($(OS),Windows_NT) - ifndef $(APPLITOOLS_API_KEY) - $(info 'No Applitools API key has been found, switching to local') - $(info variable TEMP = $(TEMP)) XVFBRUN= TEST_RESULT_DIR:=$(TEMP)\mantidimaging_tests - export APPLITOOLS_API_KEY=local - export APPLITOOLS_IMAGE_DIR:=${TEST_RESULT_DIR} - $(info variable APPLITOOLS_API_KEY = $(APPLITOOLS_API_KEY)) - $(info variable APPLITOOLS_IMAGE_DIR = $(APPLITOOLS_IMAGE_DIR)) - endif else XVFBRUN=xvfb-run --auto-servernum TEST_RESULT_DIR:=$(shell mktemp -d) endif +test-local-setup: + ifeq ($(OS),Windows_NT) + -mkdir ${TEST_RESULT_DIR} + @echo "created test directory" ${TEST_RESULT_DIR} + export APPLITOOLS_API_KEY=local + export APPLITOOLS_IMAGE_DIR:=${TEST_RESULT_DIR} + endif install-build-requirements: @echo "Installing packages required for starting the build process" @@ -48,16 +47,16 @@ build-conda-package-release: .remind-for-user .remind-for-anaconda-api install-b install-dev-requirements: python ./setup.py create_dev_env -test: +test: test-local-setup python -m pytest -n auto --run-unit-tests -test-verbose: +test-verbose: test-local-setup python -m pytest -vs -o log_cli=true --run-unit-tests -test-gh-actions: +test-gh-unit: python -m pytest --cov --cov-report=xml -n auto -o log_cli=true --run-unit-tests --durations=10 -test-system: +test-system: test-local-setup ${XVFBRUN} python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-system-tests test-gh-system: @@ -68,8 +67,7 @@ test-screenshots: APPLITOOLS_API_KEY=local APPLITOOLS_IMAGE_DIR=${TEST_RESULT_DIR} ${XVFBRUN} pytest -p no:xdist -p no:randomly -p no:cov mantidimaging/eyes_tests/ -vs --run-eyes-tests @echo "Screenshots writen to" ${TEST_RESULT_DIR} -test-screenshots-win: - -mkdir ${TEST_RESULT_DIR} +test-screenshots-win: test-local-setup ${XVFBRUN} pytest -p no:xdist -p no:randomly -p no:cov mantidimaging/eyes_tests/ -vs --run-eyes-tests @echo "Screenshots writen to" ${TEST_RESULT_DIR} From b0da73193221246da1732181f7a4335544d239c6 Mon Sep 17 00:00:00 2001 From: Mike Sullivan Date: Wed, 6 Mar 2024 14:32:31 +0000 Subject: [PATCH 13/18] debug --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 362b89a74dc..40c03666f1f 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ test-verbose: test-local-setup python -m pytest -vs -o log_cli=true --run-unit-tests test-gh-unit: - python -m pytest --cov --cov-report=xml -n auto -o log_cli=true --run-unit-tests --durations=10 + python -m pytest -vs --cov --cov-report=xml -n auto -o log_cli=true --run-unit-tests --durations=10 test-system: test-local-setup ${XVFBRUN} python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-system-tests From caa4ceb1c07e54a30a530993d548903d7af35130 Mon Sep 17 00:00:00 2001 From: Mike Sullivan Date: Wed, 6 Mar 2024 14:56:56 +0000 Subject: [PATCH 14/18] debug --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 40c03666f1f..9188a03d4b7 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,7 @@ test-verbose: test-local-setup python -m pytest -vs -o log_cli=true --run-unit-tests test-gh-unit: + @echo "APPLITOOLS_IMAGE_DIR = " ${APPLITOOLS_IMAGE_DIR} python -m pytest -vs --cov --cov-report=xml -n auto -o log_cli=true --run-unit-tests --durations=10 test-system: test-local-setup From e2724a26b5f61994a6e330ff07f64ff24fc37545 Mon Sep 17 00:00:00 2001 From: Mike Sullivan Date: Wed, 6 Mar 2024 15:07:46 +0000 Subject: [PATCH 15/18] debug --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 9188a03d4b7..a13fae96e50 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ CHANNELS=$(shell cat environment.yml | sed -ne '/channels:/,/dependencies:/{//!p ifeq ($(OS),Windows_NT) XVFBRUN= TEST_RESULT_DIR:=$(TEMP)\mantidimaging_tests + APPLITOOLS_IMAGE_DIR= else XVFBRUN=xvfb-run --auto-servernum TEST_RESULT_DIR:=$(shell mktemp -d) From 30486ae4dda3e1ba17ae5c3fcedc0985f869f1e4 Mon Sep 17 00:00:00 2001 From: Mike Sullivan Date: Wed, 6 Mar 2024 15:49:45 +0000 Subject: [PATCH 16/18] debug --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a13fae96e50..a06b6a50957 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,8 @@ CHANNELS=$(shell cat environment.yml | sed -ne '/channels:/,/dependencies:/{//!p ifeq ($(OS),Windows_NT) XVFBRUN= TEST_RESULT_DIR:=$(TEMP)\mantidimaging_tests - APPLITOOLS_IMAGE_DIR= + export APPLITOOLS_IMAGE_DIR= + $(info APPLITOOLS_IMAGE_DIR = $(APPLITOOLS_IMAGE_DIR)) else XVFBRUN=xvfb-run --auto-servernum TEST_RESULT_DIR:=$(shell mktemp -d) @@ -49,7 +50,7 @@ install-dev-requirements: python ./setup.py create_dev_env test: test-local-setup - python -m pytest -n auto --run-unit-tests + python -m pytest -n auto --run-unit-tests -vs test-verbose: test-local-setup python -m pytest -vs -o log_cli=true --run-unit-tests From 98cf74f7c22279ca5328fb8e4c347636c096cd60 Mon Sep 17 00:00:00 2001 From: Mike Sullivan Date: Wed, 6 Mar 2024 16:11:51 +0000 Subject: [PATCH 17/18] debug1 --- .github/workflows/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 5e0266170c4..f8a49650b5c 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -87,7 +87,7 @@ jobs: timeout-minutes: 10 shell: bash -l {0} run: | - make test-gh-unit + python -m pytest -vs --cov --cov-report=xml -n auto -o log_cli=true --run-unit-tests --durations=10 - name: Get test data shell: bash -l {0} From cd4cd7944cb407b84fea164ae476e1b298438ade Mon Sep 17 00:00:00 2001 From: Mike Sullivan Date: Wed, 6 Mar 2024 16:33:13 +0000 Subject: [PATCH 18/18] debug1 --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index a06b6a50957..d6fda5d2ea2 100644 --- a/Makefile +++ b/Makefile @@ -56,14 +56,14 @@ test-verbose: test-local-setup python -m pytest -vs -o log_cli=true --run-unit-tests test-gh-unit: - @echo "APPLITOOLS_IMAGE_DIR = " ${APPLITOOLS_IMAGE_DIR} - python -m pytest -vs --cov --cov-report=xml -n auto -o log_cli=true --run-unit-tests --durations=10 + python -m pytest -vs --cov --cov-report=xml -n auto -o log_cli=true --run-unit-tests --durations=10 --ignore=mantidimaging/eyes_tests test-system: test-local-setup ${XVFBRUN} python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-system-tests test-gh-system: - python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-system-tests --durations=10 + APPLITOOLS_IMAGE_DIR= + python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-system-tests --durations=10 --ignore=mantidimaging/eyes_tests test-screenshots: -mkdir ${TEST_RESULT_DIR} @@ -75,6 +75,7 @@ test-screenshots-win: test-local-setup @echo "Screenshots writen to" ${TEST_RESULT_DIR} test-gh-screenshots: + APPLITOOLS_IMAGE_DIR = python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-eyes-tests --durations=10 mypy: