Add covalent aws ec2 launcher and push_reports

covalent is not compatible with milabench as it requires sqlalchemy<2.0.0

benchmarks/_template/requirements.cpu.txt

@@ -0,0 +1,46 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile --output-file=benchmarks/_template/requirements.cpu.txt benchmarks/_template/requirements.in
+#
+antlr4-python3-runtime==4.9.3
+    # via omegaconf
+asttokens==2.4.1
+    # via giving
+codefind==0.1.3
+    # via ptera
+executing==1.2.0
+    # via varname
+giving==0.4.2
+    # via
+    #   ptera
+    #   voir
+markdown-it-py==3.0.0
+    # via rich
+mdurl==0.1.2
+    # via markdown-it-py
+omegaconf==2.3.0
+    # via voir
+ovld==0.3.2
+    # via voir
+ptera==1.4.1
+    # via voir
+pygments==2.17.2
+    # via rich
+pynvml==11.5.0
+    # via voir
+pyyaml==6.0.1
+    # via omegaconf
+reactivex==4.0.4
+    # via giving
+rich==13.7.0
+    # via voir
+six==1.16.0
+    # via asttokens
+typing-extensions==4.10.0
+    # via reactivex
+varname==0.10.0
+    # via giving
+voir==0.2.12
+    # via -r benchmarks/_template/requirements.in

config/test-system.yaml

@@ -0,0 +1,16 @@
+system:
+  # Nodes list
+  nodes:
+    - name: manager
+      ip: 1.1.1.1
+      main: true
+      user: user
+
+  # Cloud instances profiles
+  cloud_profiles:
+    ec2:
+      profile: mb_test_sog_2
+      username: ubuntu
+      instance_type: t2.micro
+      volume_size: 8
+      region: us-east-2

config/test.yaml

@@ -0,0 +1,24 @@
+_defaults:
+  max_duration: 600
+  voir:
+    options:
+      stop: 60
+      interval: "1s"
+
+test:
+  inherits: _defaults
+  group: test_remote
+  install_group: test_remote
+  definition: ../benchmarks/_template
+  plan:
+    method: per_gpu
+    run_on: ec2
+
+testing:
+  inherits: _defaults
+  definition: ../benchmarks/_template
+  group: test_remote_2
+  install_group: test_remote_2
+  plan:
+    method: per_gpu
+    run_on: ec2

milabench/__init__.py

@@ -0,0 +1,5 @@
+import pathlib
+
+ROOT_FOLDER = pathlib.Path(__file__).resolve().parent.parent
+CONFIG_FOLDER = ROOT_FOLDER / "config"
+BENCHMARK_FOLDER = ROOT_FOLDER / "benchmarks"

milabench/cli/__init__.py

@@ -12,6 +12,7 @@
 from .pr import cli_write_report_to_pr
 from .prepare import cli_prepare
 from .publish import cli_publish
+from .purge_cloud import cli_purge_cloud
 from .report import cli_report
 from .run import cli_run
 from .schedule import cli_schedule
@@ -37,6 +38,10 @@ def pin():
         """Pin the benchmarks' dependencies."""
         cli_pin()
 
+    def purge_cloud():
+        """Purge running cloud instannces the benchmarks' dependencies."""
+        cli_purge_cloud()
+
     def dev():
         """Create a shell in a benchmark's environment for development."""
         cli_dev()

milabench/cli/badges/__main__.py

@@ -0,0 +1,51 @@
+import pathlib
+import subprocess
+import sys
+
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv[1:]
+
+    try:
+        import pybadges
+    except ImportError:
+        cache_dir = pathlib.Path(f"/tmp/milabench/{pathlib.Path(__file__).name}_venv")
+        python3 = str(cache_dir / "bin/python3")
+        try:
+            subprocess.run([
+                python3,
+                "-c",
+                "import pybadges"
+            ], check=True)
+        except (FileNotFoundError, subprocess.CalledProcessError):
+            cache_dir.mkdir(parents=True, exist_ok=True)
+            subprocess.run([sys.executable, "-m", "virtualenv", str(cache_dir)], check=True)
+            subprocess.run([python3, "-m", "pip", "install", "-U", "pip"], check=True)
+            subprocess.run([
+                python3,
+                "-m",
+                "pip",
+                "install",
+                "-r",
+                str(pathlib.Path(__file__).resolve().parent / "requirements.txt")
+            ], check=True)
+            subprocess.run([
+                python3,
+                "-c",
+                "import pybadges"
+            ], check=True)
+        return subprocess.call(
+            [python3, __file__, *argv],
+        )
+
+    return subprocess.run([
+        sys.executable,
+        "-m",
+        "pybadges",
+        *argv
+    ], check=True).returncode
+
+
+if __name__ == "__main__":
+    sys.exit(main())

milabench/cli/badges/requirements.txt

@@ -0,0 +1 @@
+pybadges

milabench/cli/covalent/__main__.py

@@ -0,0 +1,202 @@
+import argparse
+import asyncio
+import os
+import pathlib
+import subprocess
+import sys
+import tempfile
+
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv[1:]
+
+    try:
+        import covalent as ct
+        from covalent._shared_files.util_classes import RESULT_STATUS
+        from covalent_ec2_plugin import ec2
+    except ImportError:
+        cache_dir = pathlib.Path(f"/tmp/milabench/{pathlib.Path(__file__).name}_venv")
+        python3 = str(cache_dir / "bin/python3")
+        try:
+            subprocess.run([
+                python3,
+                "-c",
+                "import covalent ; from covalent.executor import EC2Executor"
+            ], check=True)
+        except (FileNotFoundError, subprocess.CalledProcessError):
+            cache_dir.mkdir(parents=True, exist_ok=True)
+            subprocess.run([sys.executable, "-m", "virtualenv", str(cache_dir)], check=True)
+            subprocess.run([python3, "-m", "pip", "install", "-U", "pip"], check=True)
+            subprocess.run([
+                python3,
+                "-m",
+                "pip",
+                "install",
+                "-r",
+                str(pathlib.Path(__file__).resolve().parent / "requirements.txt")
+            ], check=True)
+            subprocess.run([
+                python3,
+                "-c",
+                "import covalent ; from covalent.executor import EC2Executor"
+            ], check=True)
+        return subprocess.call(
+            [python3, __file__, *argv],
+        )
+
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers()
+    for p in ("ec2",):
+        try:
+            if p == "ec2":
+                from covalent_ec2_plugin import ec2
+            subparser = subparsers.add_parser(p)
+            subparser.add_argument(f"--setup", action="store_true")
+            subparser.add_argument(f"--teardown", action="store_true")
+            for param, default in ec2._EXECUTOR_PLUGIN_DEFAULTS.items():
+                subparser.add_argument(f"--{param.replace('_', '-')}", default=default)
+        except ImportError:
+            continue
+
+    try:
+        cv_argv, argv = argv[:argv.index("--")], argv[argv.index("--")+1:]
+    except ValueError:
+        cv_argv, argv = argv, [] 
+
+    args = parser.parse_args(cv_argv)
+
+    def _popen(cmd, *args, _env=None, **kwargs):
+        _debug_f = open("/home/ubuntu/debug", "wt")
+        print(cmd, *args, _env, kwargs, sep="\n", file=_debug_f, flush=True)
+        _env = _env if _env is not None else {}
+
+        for envvar in _env.keys():
+            envvar_val = _env[envvar]
+
+            if not envvar_val:
+                continue
+
+            envvar_val = pathlib.Path(envvar_val).expanduser()
+            if str(envvar_val) != _env[envvar]:
+                _env[envvar] = str(envvar_val)
+
+        if "MILABENCH_CONFIG_CONTENT" in _env:
+            _config_dir = pathlib.Path(_env["MILABENCH_CONFIG"]).parent
+            with tempfile.NamedTemporaryFile("wt", dir=str(_config_dir), suffix=".yaml", delete=False) as _f:
+                _f.write(_env["MILABENCH_CONFIG_CONTENT"])
+                _env["MILABENCH_CONFIG"] = _f.name
+
+        try:
+            cmd = (str(pathlib.Path(cmd[0]).expanduser()), *cmd[1:])
+        except IndexError:
+            pass
+
+        cwd = kwargs.pop("cwd", None)
+        if cwd is not None:
+            cwd = str(pathlib.Path(cwd).expanduser())
+
+        _env = {**os.environ.copy(), **kwargs.pop("env", {}), **_env}
+
+        kwargs = {
+            **kwargs,
+            "cwd": cwd,
+            "env": _env,
+            "stdout": subprocess.PIPE,
+            "stderr": subprocess.PIPE,
+        }
+        print(cmd, *args, _env, kwargs, sep="\n", file=_debug_f, flush=True)
+        p = subprocess.Popen(cmd, *args, **kwargs)
+
+        stdout_chunks = []
+        while True:
+            line = p.stdout.readline()
+            if not line:
+                break
+            line_str = line.decode("utf-8").strip()
+            stdout_chunks.append(line_str)
+            print(line_str)
+
+        _, stderr = p.communicate()
+        stderr = stderr.decode("utf-8").strip()
+        stdout = os.linesep.join(stdout_chunks)
+
+        if p.returncode != 0:
+            raise subprocess.CalledProcessError(
+                p.returncode,
+                (cmd, args, kwargs),
+                stdout,
+                stderr
+            )
+        return p.returncode, os.linesep.join([stdout, stderr])
+
+    executor = None
+    if cv_argv[0] == "ec2":
+        executor:ct.executor.BaseExecutor = ct.executor.EC2Executor(
+            **{k:v for k,v in vars(args).items() if k not in ("setup", "teardown")}
+        )
+
+    @ct.lattice
+    def lattice(argv=()):
+        venv = pathlib.Path("~/venv")
+        code = pathlib.Path("~/milabench")
+        _env = {
+            "MILABENCH_BASE": "~/benches",
+        }
+        env_config = os.environ.get("MILABENCH_CONFIG", None)
+
+        if env_config is not None:
+            env_config = pathlib.Path(env_config)
+            _env["MILABENCH_CONFIG"] = f"{code / (env_config.relative_to(env_config.parent.parent))}"
+            _env["MILABENCH_CONFIG_CONTENT"] = env_config.read_text()
+
+        return ct.electron(
+            _popen,
+            executor=executor,
+            deps_bash=ct.DepsBash([
+                f"[ -d {code} ] || git clone https://github.com/mila-iqia/milabench.git {code}",
+                f"git -C {code} checkout -B stable origin/stable",
+                f"python3 -m virtualenv {venv}",
+                f"{venv}/bin/python3 -m pip install -U pip",
+                f"{venv}/bin/python3 -m pip install -U -e {code}",
+            ]),
+        )(
+            [f"{venv}/bin/python3", "-m", "milabench", *argv],
+            cwd=str(code),
+            _env=_env,
+        )
+
+    return_code = 0
+    try:
+        dispatch_id = None
+        result = None
+        if argv:
+            dispatch_id = ct.dispatch(lattice, disable_run=False)(argv)
+            result = ct.get_result(dispatch_id=dispatch_id, wait=True)
+            return_code = result.result[0] if result.result is not None else 1
+        elif args.setup:
+            asyncio.run(executor.setup({}))
+            print(f"hostname::>{executor.hostname}")
+            print(f"username::>{executor.username}")
+            print(f"ssh_key_file::>{executor.ssh_key_file}")
+            print(f"env::>~/.condaenvrc")
+    finally:
+        result = ct.get_result(dispatch_id=dispatch_id, wait=False) if dispatch_id else None
+        status = result.get_node_result(0)["status"] if result else RESULT_STATUS.COMPLETED
+        results_dir = result.results_dir if result else None
+        if args.teardown or status not in (
+            RESULT_STATUS.CANCELLED,
+            RESULT_STATUS.COMPLETED,
+            RESULT_STATUS.FAILED,
+        ):
+            asyncio.run(
+                executor.teardown(
+                    {"dispatch_id": dispatch_id, "node_id": 0, "results_dir": results_dir}
+                )
+            )
+
+    return return_code
+
+
+if __name__ == "__main__":
+    sys.exit(main())

milabench/cli/covalent/requirements.txt

@@ -0,0 +1,2 @@
+covalent
+covalent-ec2-plugin @ git+https://github.com/satyaog/covalent-ec2-plugin.git@feature/milabench