Add a mark on strangely-failing test in main_test

Signed-off-by: Fabrice Normandin <[email protected]>
mila-iqia · Nov 20, 2024 · 035e205 · 035e205
1 parent 6cbdaf2
commit 035e205
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 7 deletions.
diff --git a/project/algorithms/image_classifier_test.py b/project/algorithms/image_classifier_test.py
@@ -6,7 +6,7 @@
 
 from project.algorithms.testsuites.lightning_module_tests import LightningModuleTests
 from project.configs import Config
-from project.conftest import command_line_overrides, skip_on_macos_in_CI
+from project.conftest import command_line_overrides, fails_on_macOS_in_CI
 from project.datamodules.image_classification.cifar10 import CIFAR10DataModule
 from project.datamodules.image_classification.image_classification import (
     ImageClassificationDataModule,
@@ -31,7 +31,7 @@ def test_example_experiment_defaults(experiment_config: Config) -> None:
     assert isinstance(experiment_config.datamodule, CIFAR10DataModule)
 
 
-@skip_on_macos_in_CI
+@fails_on_macOS_in_CI
 @run_for_all_configs_of_type("algorithm", ImageClassifier)
 @run_for_all_configs_of_type("datamodule", ImageClassificationDataModule)
 @run_for_all_configs_of_type("algorithm/network", torch.nn.Module, excluding=PreTrainedModel)

diff --git a/project/algorithms/jax_image_classifier_test.py b/project/algorithms/jax_image_classifier_test.py
@@ -5,7 +5,7 @@
 import pytest
 
 from project.algorithms.jax_image_classifier import JaxImageClassifier
-from project.conftest import skip_on_macos_in_CI
+from project.conftest import fails_on_macOS_in_CI
 from project.datamodules.image_classification.image_classification import (
     ImageClassificationDataModule,
 )
@@ -19,7 +19,7 @@ def prevent_jax_from_reserving_all_the_vram(monkeypatch: pytest.MonkeyPatch):
     monkeypatch.setenv("XLA_PYTHON_CLIENT_PREALLOCATE", "false")
 
 
-@skip_on_macos_in_CI
+@fails_on_macOS_in_CI
 @run_for_all_configs_of_type("algorithm", JaxImageClassifier)
 @run_for_all_configs_of_type("algorithm/network", flax.linen.Module)
 @run_for_all_configs_of_type("datamodule", ImageClassificationDataModule)

diff --git a/project/conftest.py b/project/conftest.py
@@ -68,6 +68,7 @@
 from pathlib import Path
 from typing import Literal
 
+import hydra.errors
 import jax
 import lightning
 import lightning.pytorch
@@ -127,11 +128,15 @@
 )
 
 
-skip_on_macos_in_CI = pytest.mark.skipif(
+fails_on_macOS_in_CI = pytest.mark.xfail(
     sys.platform == "darwin" and IN_GITHUB_CI,
-    # raises=(RuntimeError, hydra.errors.InstantiationException),
+    raises=(RuntimeError, hydra.errors.InstantiationException),
     reason="Raises 'MPS backend out of memory' error on MacOS in GitHub CI.",
 )
+skip_on_macOS_in_CI = pytest.mark.skipif(
+    sys.platform == "darwin" and IN_GITHUB_CI,
+    reason="TODO: Fails for some reason on MacOS in GitHub CI.",
+)
 
 
 @pytest.fixture(autouse=True)

diff --git a/project/main_test.py b/project/main_test.py
@@ -14,7 +14,7 @@
 from omegaconf import DictConfig
 
 import project.main
-from project.conftest import command_line_overrides
+from project.conftest import command_line_overrides, skip_on_macOS_in_CI
 from project.utils.env_vars import REPO_ROOTDIR, SLURM_JOB_ID
 from project.utils.hydra_utils import resolve_dictconfig
 from project.utils.testutils import IN_GITHUB_CI
@@ -195,6 +195,7 @@ def test_can_run_experiment(
     project.main.main()
 
 
+@skip_on_macOS_in_CI
 @pytest.mark.parametrize(
     command_line_overrides.__name__, ["algorithm=image_classifier"], indirect=True
 )