diff --git a/benchmate/benchmate/datagen.py b/benchmate/benchmate/datagen.py index 7f3c612b4..f1263d897 100644 --- a/benchmate/benchmate/datagen.py +++ b/benchmate/benchmate/datagen.py @@ -93,6 +93,7 @@ def generate_fakeimagenet(): parser.add_argument("--batch-size", default=512, type=int) parser.add_argument("--batch-count", default=60, type=int) parser.add_argument("--device-count", default=device_count(), type=int) + parser.add_argument("--device", default=None, type=str) parser.add_argument("--image-size", default=[3, 384, 384], type=int, nargs="+") parser.add_argument("--val", default=0.1, type=float, nargs="+") parser.add_argument("--test", default=0.1, type=float, nargs="+") @@ -111,8 +112,6 @@ def generate_fakeimagenet(): f"test": int(total_images * args.test), } - - generate_sets(dest, size_spec, args.image_size) print("Done!") diff --git a/milabench/utils.py b/milabench/utils.py index 9c2de0433..2e732200d 100644 --- a/milabench/utils.py +++ b/milabench/utils.py @@ -200,7 +200,10 @@ def result(self): def report(self, **kwargs): """Generate a full report containing warnings from all loggers""" summary = Summary() - for _, layer in self.funs.items(): + + items = sorted(self.funs.items(), key=lambda item: str(item[0])) + + for _, layer in items: if hasattr(layer, "report"): layer.report(summary, **kwargs) summary.show() @@ -211,6 +214,8 @@ def multilogger(*logs, **kwargs): """Combine loggers into a single context manager""" results = dict() + logs = sorted(logs, key=lambda l: str(type(l))) + with ExitStack() as stack: for log in logs: results[type(log)] = stack.enter_context(log) diff --git a/tests/test_mock.py b/tests/test_mock.py index 6fb549c55..c2d5549d1 100644 --- a/tests/test_mock.py +++ b/tests/test_mock.py @@ -100,8 +100,11 @@ def test_milabench(monkeypatch, bench, module_tmp_dir, standard_config): ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -def cleanpath(out): - return out.replace(str(ROOT), "$SRC") +def cleanpath(out, tmppath): + return (out + .replace(str(ROOT), "$SRC") + .replace(str(tmppath), "$TMP") + ) @@ -119,7 +122,7 @@ def test_milabench_bad_install(monkeypatch, tmp_path, config, capsys, file_regre # Check that the error was extracted all = capsys.readouterr() - stdout = cleanpath(all.out) + stdout = cleanpath(all.out, tmp_path) stdout = "\n".join(stdout.split("\n")[-15:-2]) file_regression.check(stdout) @@ -137,7 +140,7 @@ def test_milabench_bad_prepare(monkeypatch, tmp_path, config, capsys, file_regre # Check that the error was extracted all = capsys.readouterr() - stdout = cleanpath(all.out) + stdout = cleanpath(all.out, tmp_path) stdout = "\n".join(stdout.split("\n")[-12:-2]) file_regression.check(stdout) @@ -171,15 +174,13 @@ def test_milabench_bad_run(monkeypatch, tmp_path, config, capsys, file_regressio # Check that the error was extracted all = capsys.readouterr() - stdout = cleanpath(all.out) + stdout = cleanpath(all.out, tmp_path) stdout = "\n".join(stdout.split("\n")[-53:-2]) file_regression.check(stdout) def test_milabench_bad_run_before_install(monkeypatch, tmp_path, config, capsys, file_regression): - from milabench.cli.dry import assume_gpu - # # Do a bad run # @@ -201,6 +202,6 @@ def test_milabench_bad_run_before_install(monkeypatch, tmp_path, config, capsys, # Check that the error was extracted all = capsys.readouterr() - stdout = cleanpath(all.out) + stdout = cleanpath(all.out, tmp_path) stdout = "\n".join(stdout.split("\n")[-53:-2]) file_regression.check(stdout) diff --git a/tests/test_mock/test_milabench_bad_run.txt b/tests/test_mock/test_milabench_bad_run.txt index 24cd81a11..8ae8f13e7 100644 --- a/tests/test_mock/test_milabench_bad_run.txt +++ b/tests/test_mock/test_milabench_bad_run.txt @@ -5,17 +5,17 @@ benchio.0 * 1 exceptions found * 1 x RuntimeError | Traceback (most recent call last): - | File "/tmp/pytest-of-delaunap/pytest-35/test_milabench_bad_run0/venv/benchio/bin/voir", line 8, in + | File "$TMP/venv/benchio/bin/voir", line 8, in | sys.exit(main()) | ^^^^^^ - | File "/tmp/pytest-of-delaunap/pytest-35/test_milabench_bad_run0/venv/benchio/lib/python3.11/site-packages/voir/cli.py", line 124, in main + | File "$TMP/venv/benchio/lib/python3.11/site-packages/voir/cli.py", line 124, in main | ov(sys.argv[1:] if argv is None else argv) - | File "/tmp/pytest-of-delaunap/pytest-35/test_milabench_bad_run0/venv/benchio/lib/python3.11/site-packages/voir/phase.py", line 331, in __call__ + | File "$TMP/venv/benchio/lib/python3.11/site-packages/voir/phase.py", line 331, in __call__ | self._run(*args, **kwargs) - | File "/tmp/pytest-of-delaunap/pytest-35/test_milabench_bad_run0/venv/benchio/lib/python3.11/site-packages/voir/overseer.py", line 242, in _run + | File "$TMP/venv/benchio/lib/python3.11/site-packages/voir/overseer.py", line 242, in _run | set_value(func()) | ^^^^^^ - | File "/tmp/pytest-of-delaunap/pytest-35/test_milabench_bad_run0/venv/benchio/lib/python3.11/site-packages/voir/scriptutils.py", line 37, in + | File "$TMP/venv/benchio/lib/python3.11/site-packages/voir/scriptutils.py", line 37, in | return lambda: exec(mainsection, glb, glb) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ | File "$SRC/milabench/tests/yoshua-benchio/main.py", line 46, in @@ -30,17 +30,17 @@ benchio.1 * 1 exceptions found * 1 x RuntimeError | Traceback (most recent call last): - | File "/tmp/pytest-of-delaunap/pytest-35/test_milabench_bad_run0/venv/benchio/bin/voir", line 8, in + | File "$TMP/venv/benchio/bin/voir", line 8, in | sys.exit(main()) | ^^^^^^ - | File "/tmp/pytest-of-delaunap/pytest-35/test_milabench_bad_run0/venv/benchio/lib/python3.11/site-packages/voir/cli.py", line 124, in main + | File "$TMP/venv/benchio/lib/python3.11/site-packages/voir/cli.py", line 124, in main | ov(sys.argv[1:] if argv is None else argv) - | File "/tmp/pytest-of-delaunap/pytest-35/test_milabench_bad_run0/venv/benchio/lib/python3.11/site-packages/voir/phase.py", line 331, in __call__ + | File "$TMP/venv/benchio/lib/python3.11/site-packages/voir/phase.py", line 331, in __call__ | self._run(*args, **kwargs) - | File "/tmp/pytest-of-delaunap/pytest-35/test_milabench_bad_run0/venv/benchio/lib/python3.11/site-packages/voir/overseer.py", line 242, in _run + | File "$TMP/venv/benchio/lib/python3.11/site-packages/voir/overseer.py", line 242, in _run | set_value(func()) | ^^^^^^ - | File "/tmp/pytest-of-delaunap/pytest-35/test_milabench_bad_run0/venv/benchio/lib/python3.11/site-packages/voir/scriptutils.py", line 37, in + | File "$TMP/venv/benchio/lib/python3.11/site-packages/voir/scriptutils.py", line 37, in | return lambda: exec(mainsection, glb, glb) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ | File "$SRC/milabench/tests/yoshua-benchio/main.py", line 46, in diff --git a/tests/test_mock/test_milabench_bad_run_before_install.txt b/tests/test_mock/test_milabench_bad_run_before_install.txt index 61073ada5..64460477f 100644 --- a/tests/test_mock/test_milabench_bad_run_before_install.txt +++ b/tests/test_mock/test_milabench_bad_run_before_install.txt @@ -1,8 +1,8 @@ -benchio.0 +benchio.1 ========= * Error codes = 1 * 1 exceptions found - * 1 x FileNotFoundError: [Errno 2] No such file or directory: '/tmp/pytest-of-delaunap/pytest-52/test_milabench_bad_run_before_0/venv/benchio/bin/voir' + * 1 x FileNotFoundError: [Errno 2] No such file or directory: '$TMP/venv/benchio/bin/voir' | Traceback (most recent call last): | File "$SRC/milabench/milabench/commands/executors.py", line 35, in execute | return await run( @@ -22,12 +22,12 @@ benchio.0 | self._execute_child(args, executable, preexec_fn, close_fds, | File "$SRC/conda/lib/python3.11/subprocess.py", line 1950, in _execute_child | raise child_exception_type(errno_num, err_msg, err_filename) - | FileNotFoundError: [Errno 2] No such file or directory: '/tmp/pytest-of-delaunap/pytest-52/test_milabench_bad_run_before_0/venv/benchio/bin/voir' -benchio.1 + | FileNotFoundError: [Errno 2] No such file or directory: '$TMP/venv/benchio/bin/voir' +benchio.0 ========= * Error codes = 1 * 1 exceptions found - * 1 x FileNotFoundError: [Errno 2] No such file or directory: '/tmp/pytest-of-delaunap/pytest-52/test_milabench_bad_run_before_0/venv/benchio/bin/voir' + * 1 x FileNotFoundError: [Errno 2] No such file or directory: '$TMP/venv/benchio/bin/voir' | Traceback (most recent call last): | File "$SRC/milabench/milabench/commands/executors.py", line 35, in execute | return await run( @@ -47,4 +47,4 @@ benchio.1 | self._execute_child(args, executable, preexec_fn, close_fds, | File "$SRC/conda/lib/python3.11/subprocess.py", line 1950, in _execute_child | raise child_exception_type(errno_num, err_msg, err_filename) - | FileNotFoundError: [Errno 2] No such file or directory: '/tmp/pytest-of-delaunap/pytest-52/test_milabench_bad_run_before_0/venv/benchio/bin/voir' + | FileNotFoundError: [Errno 2] No such file or directory: '$TMP/venv/benchio/bin/voir'