2105 GitHub actions use makefile

.github/workflows/windows.yml

@@ -87,7 +87,7 @@ jobs:
         timeout-minutes: 10
         shell: bash -l {0}
         run: |
-          python -m pytest --cov --cov-report=xml -n auto -o log_cli=true --ignore=mantidimaging/eyes_tests --durations=10
+          python -m pytest -vs --cov --cov-report=xml -n auto -o log_cli=true --run-unit-tests --durations=10
 
       - name: Get test data
         shell: bash -l {0}
@@ -100,7 +100,7 @@ jobs:
       - name: GUI Tests System
         shell: bash -l {0}
         run: |
-          python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-system-tests --durations=10
+          make test-gh-system
         timeout-minutes: 15
 
       - name: Set display resolution for screenshot tests
@@ -116,7 +116,7 @@ jobs:
           APPLITOOLS_BATCH_ID: ${{ github.sha }}
           GITHUB_BRANCH_NAME: ${{ github.head_ref }}
         run: |
-          python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true mantidimaging/eyes_tests --durations=10
+          make test-gh-screenshots
         timeout-minutes: 15
 
       # Label as 'windows-build-test' for testing purposes.

Makefile

@@ -9,14 +9,21 @@ CHANNELS=$(shell cat environment.yml | sed -ne '/channels:/,/dependencies:/{//!p
 
 ifeq ($(OS),Windows_NT)
     XVFBRUN=
-	TEST_RESULT_DIR:=$(TEMP)\mantidimaging_tests
-	export APPLITOOLS_API_KEY=local
-	export APPLITOOLS_IMAGE_DIR:=${TEST_RESULT_DIR}
+    TEST_RESULT_DIR:=$(TEMP)\mantidimaging_tests
+    export APPLITOOLS_IMAGE_DIR=
+    $(info APPLITOOLS_IMAGE_DIR = $(APPLITOOLS_IMAGE_DIR))
 else
 	XVFBRUN=xvfb-run --auto-servernum
 	TEST_RESULT_DIR:=$(shell mktemp -d)
 endif
 
+test-local-setup:
+    ifeq ($(OS),Windows_NT)
+	-mkdir ${TEST_RESULT_DIR}
+	@echo "created test directory" ${TEST_RESULT_DIR}
+    export APPLITOOLS_API_KEY=local
+    export APPLITOOLS_IMAGE_DIR:=${TEST_RESULT_DIR}
+    endif
 
 install-build-requirements:
 	@echo "Installing packages required for starting the build process"
@@ -42,25 +49,35 @@ build-conda-package-release: .remind-for-user .remind-for-anaconda-api install-b
 install-dev-requirements:
 	python ./setup.py create_dev_env
 
-test:
-	python -m pytest -n auto
+test: test-local-setup
+	python -m pytest -n auto --run-unit-tests -vs
+
+test-verbose: test-local-setup
+	python -m pytest -vs -o log_cli=true --run-unit-tests
 
-test-verbose:
-	python -m pytest -vs -o log_cli=true
+test-gh-unit:
+	python -m pytest -vs --cov --cov-report=xml -n auto -o log_cli=true --run-unit-tests --durations=10 --ignore=mantidimaging/eyes_tests
 
-test-system:
+test-system: test-local-setup
 	${XVFBRUN} python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-system-tests
 
+test-gh-system:
+	APPLITOOLS_IMAGE_DIR=
+	python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-system-tests --durations=10 --ignore=mantidimaging/eyes_tests
+
 test-screenshots:
 	-mkdir ${TEST_RESULT_DIR}
-	APPLITOOLS_API_KEY=local APPLITOOLS_IMAGE_DIR=${TEST_RESULT_DIR} ${XVFBRUN} pytest -p no:xdist -p no:randomly -p no:cov mantidimaging/eyes_tests/ -vs
+	APPLITOOLS_API_KEY=local APPLITOOLS_IMAGE_DIR=${TEST_RESULT_DIR} ${XVFBRUN} pytest -p no:xdist -p no:randomly -p no:cov mantidimaging/eyes_tests/ -vs --run-eyes-tests
 	@echo "Screenshots writen to" ${TEST_RESULT_DIR}
 
-test-screenshots-win:
-	-mkdir ${TEST_RESULT_DIR}
-	${XVFBRUN} pytest -p no:xdist -p no:randomly -p no:cov mantidimaging/eyes_tests/ -vs
+test-screenshots-win: test-local-setup
+	${XVFBRUN} pytest -p no:xdist -p no:randomly -p no:cov mantidimaging/eyes_tests/ -vs --run-eyes-tests
 	@echo "Screenshots writen to" ${TEST_RESULT_DIR}
 
+test-gh-screenshots:
+	APPLITOOLS_IMAGE_DIR =
+	python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov -o log_cli=true --run-eyes-tests --durations=10
+
 mypy:
 	python -m mypy --ignore-missing-imports --no-site-packages ${SOURCE_DIRS}
 

conftest.py

@@ -9,32 +9,41 @@
 from mantidimaging.core.utility.leak_tracker import leak_tracker
 
 
-def _test_gui_system_filename_match(basename: str) -> bool:
-    return "gui_system" in basename and "_test.py" in basename
-
-
 def pytest_addoption(parser):
     parser.addoption("--run-system-tests", action="store_true", default=False, help="Run GUI system tests")
+    parser.addoption("--run-unit-tests", action="store_true", default=False, help="Run unit tests")
+    parser.addoption("--run-eyes-tests", action="store_true", default=False, help="Run eyes tests")
 
 
 def pytest_configure(config):
     config.addinivalue_line("markers", "system: GUI system tests")
+    config.addinivalue_line("markers", "unit: unit tests")
+    config.addinivalue_line("markers", "eyes: eyes tests")
 
 
-def pytest_ignore_collect(path, config):
-    # When running GUI system tests, ignore all other files
-    if config.getoption("--run-system-tests") and path.isfile() and not _test_gui_system_filename_match(path.basename):
-        return True
-    else:
-        return False
+allowed_markers = []
+skipped_tests = []
 
 
 def pytest_collection_modifyitems(config, items):
-    if not config.getoption("--run-system-tests"):
-        skip_system = pytest.mark.skip(reason="use --run-system-tests option to run")
-        for item in items:
-            if "system" in item.keywords:
-                item.add_marker(skip_system)
+    if config.getoption("--run-system-tests"):
+        allowed_markers.append(pytest.mark.system.mark)
+    if config.getoption("--run-unit-tests"):
+        allowed_markers.append(pytest.mark.unit.mark)
+    if config.getoption("--run-eyes-tests"):
+        allowed_markers.append(pytest.mark.eyes.mark)
+    for item in items:
+        if all(test not in item.nodeid for test in ["gui_system", "eyes_test"]):
+            item.add_marker(pytest.mark.unit)
+        if "gui_system" in item.nodeid:
+            item.add_marker(pytest.mark.system)
+        if "eyes_test" in item.nodeid:
+            item.add_marker(pytest.mark.eyes)
+        if any(mark in allowed_markers for mark in item.own_markers):
+            pass
+        else:
+            item.add_marker(pytest.mark.skip(reason="Test not selected"))
+            skipped_tests.append(item.nodeid)
 
 
 # Leak track for tests

docs/release_notes/next/dev-2082-pytest_separation

@@ -0,0 +1 @@
+#2082: Normal, System, and Screenshot test are now explicitly separated in the Makefile

mantidimaging/eyes_tests/base_eyes.py

@@ -51,6 +51,11 @@
 
 QApplication.setFont(QFont("Sans Serif", 10))
 
+print(f"base_eyes.py: {APPLITOOLS_BATCH_ID=}")
+print(f"base_eyes.py: {API_KEY_PRESENT=}")
+print(f"base_eyes.py: {TEST_NAME=}")
+print(f"base_eyes.py: {APPLITOOLS_IMAGE_DIR=}")
+
 
 @unittest.skipIf(API_KEY_PRESENT is None, "API Key is not defined in the environment, so Eyes tests are skipped.")
 @unittest.skipUnless(os.path.exists(LOAD_SAMPLE), LOAD_SAMPLE_MISSING_MESSAGE)