Update Python to 3.11

.github/workflows/tests_integration.yml

@@ -47,7 +47,7 @@ jobs:
     steps:
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: Check out repository code
         uses: actions/checkout@v3

.github/workflows/tests_run.yml

@@ -55,7 +55,7 @@ jobs:
       - uses: conda-incubator/setup-miniconda@v2
         with:
           auto-activate-base: false
-          python-version: 3.10
+          python-version: 3.11
           miniconda-version: "latest"
           activate-environment: test
 

README.md

@@ -83,7 +83,7 @@ The benchmark suite has been validated on the following configurations:
 
 | Python version |          GPU                   |   Configuration file |
 |       -        |        -                       |           -          |
-| 3.9.12 (conda) | 2 node x 8xNVIDIA A100 80GB    | config/standard.yaml |
+| 3.11   (conda) | 2 node x 8xNVIDIA A100 80GB    | config/standard.yaml |
 | 3.9.12 (conda) | 8x NVIDIA RTX8000 48GB         | config/standard.yaml |
 | 3.9.16 (conda) | 2x NVIDIA K80                  | config/ci.yaml       |
 | 3.9.16 (conda) | 2x AMD MI100                   | config/ci.yaml       |

benchmate/benchmate/datagen.py

@@ -97,9 +97,13 @@ def generate_fakeimagenet():
     parser.add_argument("--image-size", default=[3, 384, 384], type=int, nargs="+")
     parser.add_argument("--val", default=0.1, type=float, nargs="+")
     parser.add_argument("--test", default=0.1, type=float, nargs="+")
-    
+
     args, _ = parser.parse_known_args()
 
+    if overrides := os.getenv("MILABENCH_TESTING_PREPARE"):
+        args.batch_size, args.batch_count = overrides.split(",")
+        args.batch_size, args.batch_count = int(args.batch_size), int(args.batch_count)
+
     data_directory = os.environ["MILABENCH_DIR_DATA"]
 
     dest = os.path.join(data_directory, f"FakeImageNet")

milabench/cli/schedule.py

@@ -139,7 +139,7 @@ class SetupOptions:
     origin: str = "https://github.com/mila-iqia/milabench.git"
     config: str = "milabench/config/standard.yaml"
     env: str = "./env"
-    python: str = "3.10"
+    python: str = "3.11"
     fun: str = "run"
 
     def deduce_remote(self, current_branch):

milabench/scripts/milabench_run.bash

@@ -20,7 +20,7 @@ function usage() {
 }
 
 ARCH="cuda"
-PYTHON="3.10"
+PYTHON="3.11"
 BRANCH="master"
 ORIGIN="https://github.com/mila-iqia/milabench.git"
 LOC="$SLURM_TMPDIR/$SLURM_JOB_ID"

milabench/validation/error.py

@@ -39,7 +39,7 @@ def raised_exception(self):
         return self.lines[-1]
 
 
-def _extract_traceback(lines) -> list[ParsedTraceback]:
+def _extract_traceback(lines, is_install) -> list[ParsedTraceback]:
     output = []
     traceback = None
     parsing_pip_error = False
@@ -52,6 +52,16 @@ def push_trace():
                 parsing_pip_error = False
             output.append(traceback)
 
+    # Only extract pip error during installs
+    # to avoid false positive
+    def is_pip_error(line):
+        nonlocal is_install
+
+        if not is_install:
+            return False
+
+        return "ERROR" in line and not parsing_pip_error
+
     for line in lines:
         line = line.rstrip()
 
@@ -60,12 +70,13 @@ def push_trace():
             break
 
         # "ERROR" comes from pip install
-        if "Traceback" in line or ("ERROR" in line and not parsing_pip_error):
+        if "Traceback" in line or is_pip_error(line):
             # New traceback push old one
             push_trace()
 
-            if "ERROR" in line:
+            if is_pip_error(line):
                 parsing_pip_error = True
+
             traceback = ParsedTraceback([])
 
         if traceback and line != "":
@@ -81,11 +92,22 @@ class Layer(ValidationLayer):
 
     def __init__(self, **kwargs) -> None:
         self.errors = defaultdict(PackError)
+        self.is_prepare = False
+        self.is_install = False
 
     def on_stop(self, entry):
         error = self.errors[entry.tag]
         error.early_stop = True
 
+    def on_config(self, entry):
+        # config is not passed for install & prepare run
+        self.is_prepare = False
+        self.is_install = False
+
+    def on_start(self, entry):
+        cmd = entry.data.get("command", ["nothing"])[0]
+        self.is_install = cmd == "pip"
+
     def on_line(self, entry):
         error = self.errors[entry.tag]
         if entry.pipe == "stderr":
@@ -107,7 +129,7 @@ def report_exceptions(self, summary, error, short):
         if error.trace:
             exceptions = [ParsedTraceback(error.trace.splitlines())]
         else:
-            exceptions = _extract_traceback(error.stderr)
+            exceptions = _extract_traceback(error.stderr, self.is_install)
 
         if len(exceptions) == 0:
             summary.add("* No traceback info about the error")
@@ -133,7 +155,9 @@ def report(self, summary, short=False, **kwargs):
         failures = 0
         success = 0
 
-        for name, error in self.errors.items():
+        items = sorted(self.errors.items(), key=lambda item: str(item[0]))
+
+        for name, error in items:
             total = sum(error.code)
 
             if total == 0:

pyproject.toml

@@ -12,7 +12,7 @@ license = "MIT"
 [tool.poetry.dependencies]
 voir =  ">=0.2.14"
 benchmate =  {path = "benchmate", develop = false}
-python = ">=3.10,<4.0"
+python = ">=3.11,<4.0"
 giving = "^0.4.0"
 ptera = "^1.2.0"
 coleo = "^0.3.0"

tests/test_mock.py

@@ -82,6 +82,10 @@ def test_milabench(monkeypatch, bench, module_tmp_dir, standard_config):
     with filecount_inc(module_tmp_dir, "install"):
         run_cli("install", *args, "--select", bench)
 
+
+    # Reduce the number of images we generate to make the CI faster
+    # and reduce disk space since we will not be using them anyway
+    monkeypatch.setenv("MILABENCH_TESTING_PREPARE", "10,10")
     with filecount_inc(module_tmp_dir, "prepare"):
         run_cli("prepare", *args, "--select", bench)
 
@@ -108,7 +112,7 @@ def cleanpath(out, tmppath):
 
 
 
-def test_milabench_bad_install(monkeypatch, tmp_path, config, capsys, file_regression):
+def test_milabench_bad_install(monkeypatch, tmp_path, config, file_regression, capsys):
     args= [
         "--base", str(tmp_path),
         "--config", str(config("benchio_bad"))
@@ -124,7 +128,11 @@ def test_milabench_bad_install(monkeypatch, tmp_path, config, capsys, file_regre
     all = capsys.readouterr()
     stdout = cleanpath(all.out, tmp_path)
     stdout = "\n".join(stdout.split("\n")[-15:-2])
-    file_regression.check(stdout)
+
+    # PIP often prints a warning about a new version which gets caught by the error reporting
+    # the message might be useful but it also changes quite often
+    assert "ERROR: Could not find a version that satisfies the requirement this_package_does_not_exist" in stdout
+    # file_regression.check(stdout)
 
 
 def test_milabench_bad_prepare(monkeypatch, tmp_path, config, capsys, file_regression):

tests/test_mock/test_milabench_bad_run_before_install.txt

@@ -1,4 +1,4 @@
-benchio.1
+benchio.0
 =========
   * Error codes = 1
   * 1 exceptions found
@@ -23,7 +23,7 @@ benchio.1
         |   File "$SRC/conda/lib/python3.11/subprocess.py", line 1950, in _execute_child
         |     raise child_exception_type(errno_num, err_msg, err_filename)
         | FileNotFoundError: [Errno 2] No such file or directory: '$TMP/venv/benchio/bin/voir'
-benchio.0
+benchio.1
 =========
   * Error codes = 1
   * 1 exceptions found

tests/test_validation/test_exception_tracking.txt

@@ -1,6 +1,6 @@
 opt-1_3b.local
 ==============
-  * Error code = 1
+  * Error codes = 1
   * 9 exceptions found
     * 7 x ModuleNotFoundError: No module named 'datasets'
         | Traceback (most recent call last):