Track pytorch version

.gitignore

@@ -20,7 +20,6 @@ sqlite.db
 .mongo/
 .pin/tmp-*
 
-
 .no_report
 trash/
 workspace/

milabench/_version.py

@@ -1,5 +1,5 @@
 """This file is generated, do not modify"""
 
-__tag__ = "v0.0.6-30-g62b44f0"
-__commit__ = "62b44f0e6190dd686c9c2f9ac89c2a383b8ebb9f"
-__date__ = "2023-11-07 10:45:02 -0500"
+__tag__ = "v0.0.6-39-gceefb2eb"
+__commit__ = "ceefb2eb67f303987011937b6af7aa647746e884"
+__date__ = "2023-11-28 13:34:55 -0500"

milabench/executors.py

@@ -11,10 +11,11 @@
 from voir.instruments.gpu import get_gpu_info
 
 from . import pack
-from .alt_async import destroy
+from .alt_async import destroy, run
 from .fs import XPath
 from .merge import merge
 from .metadata import machine_metadata
+from .structs import BenchLogEntry
 from .utils import select_nodes
 
 
@@ -33,6 +34,34 @@ async def force_terminate(pack, delay):
             destroy(proc)
 
 
+async def execute(pack, *args, cwd=None, env={}, external=False, **kwargs):
+    """Run a command in the virtual environment.
+
+    Unless specified otherwise, the command is run with
+    ``self.dirs.code`` as the cwd.
+
+    Arguments:
+        args: The arguments to the command
+        cwd: The cwd to use (defaults to ``self.dirs.code``)
+    """
+    args = [str(x) for x in args]
+    if cwd is None:
+        cwd = pack.dirs.code
+
+    exec_env = pack.full_env(env) if not external else {**os.environ, **env}
+
+    return await run(
+        args,
+        **kwargs,
+        info={"pack": pack},
+        env=exec_env,
+        constructor=BenchLogEntry,
+        cwd=cwd,
+        process_accumulator=pack.processes,
+    )
+
+
+# TODO Move this to command
 class Executor:
     """Base class for an execution plan
 
@@ -126,9 +155,9 @@ async def execute(self, phase="run", timeout=False, timeout_delay=600, **kwargs)
         timeout_tasks = []
         for pack, argv, _kwargs in self.commands():
             await pack.send(event="config", data=pack.config)
-            await pack.send(event="meta", data=machine_metadata())
+            await pack.send(event="meta", data=machine_metadata(pack))
 
-            fut = pack.execute(*argv, **{**_kwargs, **kwargs})
+            fut = execute(pack, *argv, **{**_kwargs, **kwargs})
             coro.append(fut)
 
             if timeout:

milabench/metadata.py

@@ -1,12 +1,15 @@
 import os
 from datetime import datetime
 import cpuinfo
+import subprocess
 import traceback
+import json
 
 from voir.instruments.gpu import get_gpu_info
 
 from ._version import __commit__, __tag__, __date__
-from .vcs import retrieve_git_versions
+from .scripts.vcs import retrieve_git_versions
+import milabench.scripts.torchversion as torchversion
 
 
 def _get_gpu_info():
@@ -17,13 +20,32 @@ def _get_gpu_info():
         return {}
 
 
-def machine_metadata():
+def fetch_torch_version(pack):
+    cwd = pack.dirs.code
+    exec_env = pack.full_env(dict())
+
+    result = subprocess.run(
+        [str(x) for x in ["python", torchversion.__file__]],
+        env=exec_env,
+        cwd=cwd,
+        capture_output=True,
+    )
+
+    return json.loads(result.stdout)
+
+
+def machine_metadata(pack=None):
     """Retrieve machine metadata"""
 
     uname = os.uname()
     gpus = _get_gpu_info()
     cpu = cpuinfo.get_cpu_info()
 
+    if pack is None:
+        torchv = torchversion.get_pytorch_version()
+    else:
+        torchv = fetch_torch_version(pack)
+
     return {
         "cpu": {
             "count": os.cpu_count(),
@@ -43,4 +65,5 @@ def machine_metadata():
             __commit__,
             __date__,
         ),
+        "pytorch": torchv,
     }

milabench/pack.py

@@ -19,7 +19,7 @@ class defines good default behavior.
 from .fs import XPath
 from .merge import merge
 from .structs import BenchLogEntry
-from .utils import assemble_options, make_constraints_file, relativize
+from .utils import assemble_options, make_constraints_file, relativize, deprecated
 
 
 class PackageCore:
@@ -193,6 +193,7 @@ def conda_install(self, *args, **kwargs):
         args = [str(x) for x in args]
         return self._nox_session.conda_install(*args, **kwargs, silent=False)
 
+    @deprecated
     async def execute(self, *args, cwd=None, env={}, external=False, **kwargs):
         """Run a command in the virtual environment.
 
@@ -203,21 +204,9 @@ async def execute(self, *args, cwd=None, env={}, external=False, **kwargs):
             args: The arguments to the command
             cwd: The cwd to use (defaults to ``self.dirs.code``)
         """
-        args = [str(x) for x in args]
-        if cwd is None:
-            cwd = self.dirs.code
-
-        exec_env = self.full_env(env) if not external else {**os.environ, **env}
+        from .executors import execute
 
-        return await run(
-            args,
-            **kwargs,
-            info={"pack": self},
-            env=exec_env,
-            constructor=BenchLogEntry,
-            cwd=cwd,
-            process_accumulator=self.processes,
-        )
+        return execute(pack, *args, cwd=cwd, env=env, external=external, **kwargs)
 
     async def python(self, *args, **kwargs):
         """Run a Python script.

milabench/scripts/torchversion.py

@@ -0,0 +1,66 @@
+def get_pytorch_version():
+    def clean(k: str):
+        pad = "  - "
+        if k.startswith(pad):
+            return k[len(pad) :].strip()
+        return k.strip()
+
+    def find_config(lines, key):
+        for line in lines:
+            if key in line:
+                return clean(line)
+
+        return None
+
+    def parse_build_settings(settings):
+        flags = dict()
+
+        if settings is None:
+            return flags
+
+        _, settings = settings.split(":")
+        for setting in settings.split(","):
+            try:
+                k, v = setting.split("=", maxsplit=1)
+                flags[k.strip()] = v.strip()
+            except ValueError:
+                pass
+
+        return flags
+
+    try:
+        import torch
+
+        conf = torch.__config__.show().split("\n")
+
+        compiler = conf[1]
+        cpp = find_config(conf, "C++ Version")
+        intel = find_config(conf, "oneAPI")
+        mkl = find_config(conf, "OpenMP")
+        openmp = find_config(conf, "OpenMP")
+        lapack = find_config(conf, "LAPACK")
+        nnpack = find_config(conf, "NNPACK")
+        cpu = find_config(conf, "CPU")
+        build_settings = find_config(conf, "Build settings")
+
+        return dict(
+            torch=torch.__version__,
+            compiler=clean(compiler),
+            cpp=clean(cpp),
+            intel=clean(intel),
+            mkl=clean(mkl),
+            openmp=clean(openmp),
+            lapack=clean(lapack),
+            nnpack=clean(nnpack),
+            cpu=clean(cpu),
+            build_settings=parse_build_settings(build_settings),
+        )
+
+    except ImportError:
+        return dict()
+
+
+if __name__ == "__main__":
+    import json
+
+    print(json.dumps(get_pytorch_version()))

milabench/vcs.py → milabench/scripts/vcs.py

@@ -4,7 +4,7 @@
 import subprocess
 import warnings
 
-ROOT = os.path.join(os.path.dirname(__file__), "..")
+ROOT = os.path.join(os.path.dirname(__file__), "..", "..")
 
 
 def _exec(cmd, default):

milabench/utils.py

@@ -7,8 +7,10 @@
 import importlib
 import pkgutil
 import traceback
+import functools
 from functools import wraps
 from typing import Any
+import warnings
 
 from ovld import ovld
 
@@ -17,6 +19,19 @@
 from milabench.validation.validation import Summary
 
 
+def deprecated(func):
+    @functools.wraps(func)
+    def new_func(*args, **kwargs):
+        warnings.warn(
+            "Call to deprecated function {}.".format(func.__name__),
+            category=DeprecationWarning,
+            stacklevel=2,
+        )
+        return func(*args, **kwargs)
+
+    return new_func
+
+
 class Named:
     """A named object.
     This class can be used to construct objects with a name that will be used

pyproject.toml

@@ -51,7 +51,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry.build]
 generate-setup-file = false
-script = "milabench/vcs.py"
+script = "milabench/scripts/vcs.py"
 
 [tool.poetry.scripts]
 milabench = "milabench.cli:main"

scripts/schedule.sh

@@ -0,0 +1,25 @@
+
+
+WORKSPACE="$(pwd)/workspace"
+
+
+ARCH="cuda"
+PYTHON="3.9"
+BRANCH="master"
+ORIGIN="https://github.com/mila-iqia/milabench.git"
+LOC="$SLURM_TMPDIR"
+CONFIG="$(pwd)/config/standard.yaml"
+BASE="$WORKSPACE"
+
+export HF_HOME=$BASE/cache
+export HF_DATASETS_CACHE=$BASE/cache
+export TORCH_HOME=$BASE/cache
+export XDG_CACHE_HOME=$BASE/cache
+
+export MILABENCH_GPU_ARCH=$ARCH
+export MILABENCH_DASH=no 
+export PYTHONUNBUFFERED=1
+export MILABENCH_BASE=$BASE
+export MILABENCH_CONFIG=$CONFIG
+
+# . scripts/schedule.sh && milabench run --select resnet50