From 9e90cf86bac5593a1d886d79f08df9f133504f1d Mon Sep 17 00:00:00 2001 From: "pierre.delaunay" Date: Tue, 9 Jul 2024 16:37:12 -0400 Subject: [PATCH] Update Python to 3.11 --- .github/workflows/tests_integration.yml | 2 +- .github/workflows/tests_run.yml | 2 +- README.md | 2 +- benchmate/benchmate/datagen.py | 6 +- milabench/cli/schedule.py | 2 +- milabench/scripts/milabench_run.bash | 2 +- milabench/validation/error.py | 34 +++++++-- poetry.lock | 76 +++---------------- pyproject.toml | 2 +- tests/test_mock.py | 12 ++- .../test_milabench_bad_run_before_install.txt | 4 +- .../test_exception_tracking.txt | 2 +- 12 files changed, 64 insertions(+), 82 deletions(-) diff --git a/.github/workflows/tests_integration.yml b/.github/workflows/tests_integration.yml index 40823fc10..6ed7d42b8 100644 --- a/.github/workflows/tests_integration.yml +++ b/.github/workflows/tests_integration.yml @@ -47,7 +47,7 @@ jobs: steps: - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Check out repository code uses: actions/checkout@v3 diff --git a/.github/workflows/tests_run.yml b/.github/workflows/tests_run.yml index 41b8a5fb3..cd98468a7 100644 --- a/.github/workflows/tests_run.yml +++ b/.github/workflows/tests_run.yml @@ -55,7 +55,7 @@ jobs: - uses: conda-incubator/setup-miniconda@v2 with: auto-activate-base: false - python-version: 3.10 + python-version: 3.11 miniconda-version: "latest" activate-environment: test diff --git a/README.md b/README.md index daf12daff..94b1c3849 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ The benchmark suite has been validated on the following configurations: | Python version | GPU | Configuration file | | - | - | - | -| 3.9.12 (conda) | 2 node x 8xNVIDIA A100 80GB | config/standard.yaml | +| 3.11 (conda) | 2 node x 8xNVIDIA A100 80GB | config/standard.yaml | | 3.9.12 (conda) | 8x NVIDIA RTX8000 48GB | config/standard.yaml | | 3.9.16 (conda) | 2x NVIDIA K80 | config/ci.yaml | | 3.9.16 (conda) | 2x AMD MI100 | config/ci.yaml | diff --git a/benchmate/benchmate/datagen.py b/benchmate/benchmate/datagen.py index f1263d897..9f3ca95eb 100644 --- a/benchmate/benchmate/datagen.py +++ b/benchmate/benchmate/datagen.py @@ -97,9 +97,13 @@ def generate_fakeimagenet(): parser.add_argument("--image-size", default=[3, 384, 384], type=int, nargs="+") parser.add_argument("--val", default=0.1, type=float, nargs="+") parser.add_argument("--test", default=0.1, type=float, nargs="+") - + args, _ = parser.parse_known_args() + if overrides := os.getenv("MILABENCH_TESTING_PREPARE"): + args.batch_size, args.batch_count = overrides.split(",") + args.batch_size, args.batch_count = int(args.batch_size), int(args.batch_count) + data_directory = os.environ["MILABENCH_DIR_DATA"] dest = os.path.join(data_directory, f"FakeImageNet") diff --git a/milabench/cli/schedule.py b/milabench/cli/schedule.py index 0315d4aaa..a6b4724f8 100644 --- a/milabench/cli/schedule.py +++ b/milabench/cli/schedule.py @@ -139,7 +139,7 @@ class SetupOptions: origin: str = "https://github.com/mila-iqia/milabench.git" config: str = "milabench/config/standard.yaml" env: str = "./env" - python: str = "3.10" + python: str = "3.11" fun: str = "run" def deduce_remote(self, current_branch): diff --git a/milabench/scripts/milabench_run.bash b/milabench/scripts/milabench_run.bash index 409d13797..78051f9e0 100755 --- a/milabench/scripts/milabench_run.bash +++ b/milabench/scripts/milabench_run.bash @@ -20,7 +20,7 @@ function usage() { } ARCH="cuda" -PYTHON="3.10" +PYTHON="3.11" BRANCH="master" ORIGIN="https://github.com/mila-iqia/milabench.git" LOC="$SLURM_TMPDIR/$SLURM_JOB_ID" diff --git a/milabench/validation/error.py b/milabench/validation/error.py index 6abcdccb5..30baf1697 100644 --- a/milabench/validation/error.py +++ b/milabench/validation/error.py @@ -39,7 +39,7 @@ def raised_exception(self): return self.lines[-1] -def _extract_traceback(lines) -> list[ParsedTraceback]: +def _extract_traceback(lines, is_install) -> list[ParsedTraceback]: output = [] traceback = None parsing_pip_error = False @@ -52,6 +52,16 @@ def push_trace(): parsing_pip_error = False output.append(traceback) + # Only extract pip error during installs + # to avoid false positive + def is_pip_error(line): + nonlocal is_install + + if not is_install: + return False + + return "ERROR" in line and not parsing_pip_error + for line in lines: line = line.rstrip() @@ -60,12 +70,13 @@ def push_trace(): break # "ERROR" comes from pip install - if "Traceback" in line or ("ERROR" in line and not parsing_pip_error): + if "Traceback" in line or is_pip_error(line): # New traceback push old one push_trace() - if "ERROR" in line: + if is_pip_error(line): parsing_pip_error = True + traceback = ParsedTraceback([]) if traceback and line != "": @@ -81,11 +92,22 @@ class Layer(ValidationLayer): def __init__(self, **kwargs) -> None: self.errors = defaultdict(PackError) + self.is_prepare = False + self.is_install = False def on_stop(self, entry): error = self.errors[entry.tag] error.early_stop = True + def on_config(self, entry): + # config is not passed for install & prepare run + self.is_prepare = False + self.is_install = False + + def on_start(self, entry): + cmd = entry.data.get("command", ["nothing"])[0] + self.is_install = cmd == "pip" + def on_line(self, entry): error = self.errors[entry.tag] if entry.pipe == "stderr": @@ -107,7 +129,7 @@ def report_exceptions(self, summary, error, short): if error.trace: exceptions = [ParsedTraceback(error.trace.splitlines())] else: - exceptions = _extract_traceback(error.stderr) + exceptions = _extract_traceback(error.stderr, self.is_install) if len(exceptions) == 0: summary.add("* No traceback info about the error") @@ -133,7 +155,9 @@ def report(self, summary, short=False, **kwargs): failures = 0 success = 0 - for name, error in self.errors.items(): + items = sorted(self.errors.items(), key=lambda item: str(item[0])) + + for name, error in items: total = sum(error.code) if total == 0: diff --git a/poetry.lock b/poetry.lock index 05ec27a3f..03d3c80e3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -160,8 +160,6 @@ mypy-extensions = ">=0.4.3" packaging = ">=22.0" pathspec = ">=0.9.0" platformdirs = ">=2" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} [package.extras] colorama = ["colorama (>=0.4.3)"] @@ -198,10 +196,8 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "os_name == \"nt\""} -importlib-metadata = {version = ">=4.6", markers = "python_full_version < \"3.10.2\""} packaging = ">=19.1" pyproject_hooks = "*" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} [package.extras] docs = ["furo (>=2023.08.17)", "sphinx (>=7.0,<8.0)", "sphinx-argparse-cli (>=1.5)", "sphinx-autodoc-typehints (>=1.10)", "sphinx-issues (>=3.0.0)"] @@ -212,13 +208,13 @@ virtualenv = ["virtualenv (>=20.0.35)"] [[package]] name = "certifi" -version = "2024.6.2" +version = "2024.7.4" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"}, - {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, + {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, + {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, ] [[package]] @@ -452,9 +448,6 @@ files = [ {file = "coverage-7.5.4.tar.gz", hash = "sha256:a44963520b069e12789d0faea4e9fdb1e410cdc4aab89d94f7f55cbb7fef0353"}, ] -[package.dependencies] -tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} - [package.extras] toml = ["tomli"] @@ -716,25 +709,6 @@ files = [ {file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"}, ] -[[package]] -name = "importlib-metadata" -version = "8.0.0" -description = "Read metadata from Python packages" -optional = false -python-versions = ">=3.8" -files = [ - {file = "importlib_metadata-8.0.0-py3-none-any.whl", hash = "sha256:15584cf2b1bf449d98ff8a6ff1abef57bf20f3ac6454f431736cd3e660921b2f"}, - {file = "importlib_metadata-8.0.0.tar.gz", hash = "sha256:188bd24e4c346d3f0a933f275c2fec67050326a856b9a359881d7c2a697e8812"}, -] - -[package.dependencies] -zipp = ">=0.5" - -[package.extras] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -perf = ["ipython"] -test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"] - [[package]] name = "importlib-resources" version = "6.4.0" @@ -1075,7 +1049,6 @@ files = [ [package.dependencies] numpy = [ - {version = ">=1.22.4", markers = "python_version < \"3.11\""}, {version = ">=1.23.2", markers = "python_version == \"3.11\""}, {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, ] @@ -1121,13 +1094,13 @@ files = [ [[package]] name = "pip" -version = "24.1.1" +version = "24.1.2" description = "The PyPA recommended tool for installing Python packages." optional = false python-versions = ">=3.8" files = [ - {file = "pip-24.1.1-py3-none-any.whl", hash = "sha256:efca15145a95e95c00608afeab66311d40bfb73bb2266a855befd705e6bb15a0"}, - {file = "pip-24.1.1.tar.gz", hash = "sha256:5aa64f65e1952733ee0a9a9b1f52496ebdb3f3077cc46f80a16d983b58d1180a"}, + {file = "pip-24.1.2-py3-none-any.whl", hash = "sha256:7cd207eed4c60b0f411b444cd1464198fe186671c323b6cd6d433ed80fc9d247"}, + {file = "pip-24.1.2.tar.gz", hash = "sha256:e5458a0b89f2755e0ee8c0c77613fe5273e05f337907874d64f13171a898a7ff"}, ] [[package]] @@ -1147,7 +1120,6 @@ click = ">=8" pip = ">=22.2" pyproject_hooks = "*" setuptools = "*" -tomli = {version = "*", markers = "python_version < \"3.11\""} wheel = "*" [package.extras] @@ -1695,13 +1667,13 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "setuptools" -version = "70.2.0" +version = "70.3.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-70.2.0-py3-none-any.whl", hash = "sha256:b8b8060bb426838fbe942479c90296ce976249451118ef566a5a0b7d8b78fb05"}, - {file = "setuptools-70.2.0.tar.gz", hash = "sha256:bd63e505105011b25c3c11f753f7e3b8465ea739efddaccef8f0efac2137bac1"}, + {file = "setuptools-70.3.0-py3-none-any.whl", hash = "sha256:fe384da74336c398e0d956d1cae0669bc02eed936cdb1d49b57de1990dc11ffc"}, + {file = "setuptools-70.3.0.tar.gz", hash = "sha256:f171bab1dfbc86b132997f26a119f6056a57950d058587841a0082e8830f9dc5"}, ] [package.extras] @@ -2000,17 +1972,6 @@ files = [ {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] - [[package]] name = "torchcompat" version = "1.1.1" @@ -2171,22 +2132,7 @@ files = [ [package.extras] test = ["pytest (>=6.0.0)", "setuptools (>=65)"] -[[package]] -name = "zipp" -version = "3.19.2" -description = "Backport of pathlib-compatible object wrapper for zip files" -optional = false -python-versions = ">=3.8" -files = [ - {file = "zipp-3.19.2-py3-none-any.whl", hash = "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c"}, - {file = "zipp-3.19.2.tar.gz", hash = "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19"}, -] - -[package.extras] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] - [metadata] lock-version = "2.0" -python-versions = ">=3.10,<4.0" -content-hash = "ac862de7cec59d54e21cdc3280c72810aa15b88da6e9b6e67fb03d7e89228177" +python-versions = ">=3.11,<4.0" +content-hash = "37f41740fb3e7ed03846f245c32e2676033b09e31d3963706628edec3dcbef1b" diff --git a/pyproject.toml b/pyproject.toml index 6695d6440..aac1361cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ license = "MIT" [tool.poetry.dependencies] voir = ">=0.2.14" benchmate = {path = "benchmate", develop = false} -python = ">=3.10,<4.0" +python = ">=3.11,<4.0" giving = "^0.4.0" ptera = "^1.2.0" coleo = "^0.3.0" diff --git a/tests/test_mock.py b/tests/test_mock.py index c2d5549d1..099cc64fc 100644 --- a/tests/test_mock.py +++ b/tests/test_mock.py @@ -82,6 +82,10 @@ def test_milabench(monkeypatch, bench, module_tmp_dir, standard_config): with filecount_inc(module_tmp_dir, "install"): run_cli("install", *args, "--select", bench) + + # Reduce the number of images we generate to make the CI faster + # and reduce disk space since we will not be using them anyway + monkeypatch.setenv("MILABENCH_TESTING_PREPARE", "10,10") with filecount_inc(module_tmp_dir, "prepare"): run_cli("prepare", *args, "--select", bench) @@ -108,7 +112,7 @@ def cleanpath(out, tmppath): -def test_milabench_bad_install(monkeypatch, tmp_path, config, capsys, file_regression): +def test_milabench_bad_install(monkeypatch, tmp_path, config, file_regression, capsys): args= [ "--base", str(tmp_path), "--config", str(config("benchio_bad")) @@ -124,7 +128,11 @@ def test_milabench_bad_install(monkeypatch, tmp_path, config, capsys, file_regre all = capsys.readouterr() stdout = cleanpath(all.out, tmp_path) stdout = "\n".join(stdout.split("\n")[-15:-2]) - file_regression.check(stdout) + + # PIP often prints a warning about a new version which gets caught by the error reporting + # the message might be useful but it also changes quite often + assert "ERROR: Could not find a version that satisfies the requirement this_package_does_not_exist" in stdout + # file_regression.check(stdout) def test_milabench_bad_prepare(monkeypatch, tmp_path, config, capsys, file_regression): diff --git a/tests/test_mock/test_milabench_bad_run_before_install.txt b/tests/test_mock/test_milabench_bad_run_before_install.txt index 64460477f..daecde84f 100644 --- a/tests/test_mock/test_milabench_bad_run_before_install.txt +++ b/tests/test_mock/test_milabench_bad_run_before_install.txt @@ -1,4 +1,4 @@ -benchio.1 +benchio.0 ========= * Error codes = 1 * 1 exceptions found @@ -23,7 +23,7 @@ benchio.1 | File "$SRC/conda/lib/python3.11/subprocess.py", line 1950, in _execute_child | raise child_exception_type(errno_num, err_msg, err_filename) | FileNotFoundError: [Errno 2] No such file or directory: '$TMP/venv/benchio/bin/voir' -benchio.0 +benchio.1 ========= * Error codes = 1 * 1 exceptions found diff --git a/tests/test_validation/test_exception_tracking.txt b/tests/test_validation/test_exception_tracking.txt index 8275c6a74..226fd3895 100644 --- a/tests/test_validation/test_exception_tracking.txt +++ b/tests/test_validation/test_exception_tracking.txt @@ -1,6 +1,6 @@ opt-1_3b.local ============== - * Error code = 1 + * Error codes = 1 * 9 exceptions found * 7 x ModuleNotFoundError: No module named 'datasets' | Traceback (most recent call last):