diff --git a/.github/workflows/tests_unit.yml b/.github/workflows/tests_unit.yml index ba08558c4..b6d1df49f 100644 --- a/.github/workflows/tests_unit.yml +++ b/.github/workflows/tests_unit.yml @@ -50,6 +50,24 @@ jobs: # poetry install --with dev + - name: Simple Template + run: | + source $(poetry env info -p)/bin/activate + milabench new --name simplebench --template simple + cd benchmarks/simplebench + make tests + cd .. + rm -rf simplebench + + - name: Voir Template + run: | + source $(poetry env info -p)/bin/activate + milabench new --name voirbench --template voir + cd benchmarks/voirbench + make tests + cd .. + rm -rf voirbench + - name: tests run: | source $(poetry env info -p)/bin/activate diff --git a/.gitignore b/.gitignore index 5c661d462..f4ccd7a37 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,8 @@ dry/ stderr.txt stdout.txt + +base/ + +benchmarks/simple +benchmarks/voir diff --git a/benchmarks/_templates/simple/Makefile b/benchmarks/_templates/simple/Makefile new file mode 100644 index 000000000..f295ac5bb --- /dev/null +++ b/benchmarks/_templates/simple/Makefile @@ -0,0 +1,4 @@ +tests: + milabench install --config dev.yaml --base base + milabench prepare --config dev.yaml --base base + milabench run --config dev.yaml --base base diff --git a/benchmarks/_template/README.md b/benchmarks/_templates/simple/README.md similarity index 83% rename from benchmarks/_template/README.md rename to benchmarks/_templates/simple/README.md index bb348e387..239a2dfaf 100644 --- a/benchmarks/_template/README.md +++ b/benchmarks/_templates/simple/README.md @@ -1,4 +1,4 @@ -# {{STEM}} +# Template Rewrite this README to explain what the benchmark is! diff --git a/benchmarks/_template/benchfile.py b/benchmarks/_templates/simple/benchfile.py similarity index 88% rename from benchmarks/_template/benchfile.py rename to benchmarks/_templates/simple/benchfile.py index 633ad2235..08a51cef0 100644 --- a/benchmarks/_template/benchfile.py +++ b/benchmarks/_templates/simple/benchfile.py @@ -1,7 +1,7 @@ from milabench.pack import Package -class TheBenchmark(Package): +class Template(Package): # Requirements file installed by install(). It can be empty or absent. base_requirements = "requirements.in" @@ -26,8 +26,6 @@ async def install(self): async def prepare(self): await super().prepare() # super() call executes prepare_script - async def run(self): - return await super().run() -__pack__ = TheBenchmark +__pack__ = Template diff --git a/benchmarks/_template/dev.yaml b/benchmarks/_templates/simple/dev.yaml similarity index 90% rename from benchmarks/_template/dev.yaml rename to benchmarks/_templates/simple/dev.yaml index 93c661ad5..f25e977b2 100644 --- a/benchmarks/_template/dev.yaml +++ b/benchmarks/_templates/simple/dev.yaml @@ -1,5 +1,5 @@ -{{STEM}}: +template: inherits: _defaults definition: . install-variant: unpinned diff --git a/benchmarks/_template/main.py b/benchmarks/_templates/simple/main.py similarity index 66% rename from benchmarks/_template/main.py rename to benchmarks/_templates/simple/main.py index 99f0f0adc..b9fe484dd 100644 --- a/benchmarks/_template/main.py +++ b/benchmarks/_templates/simple/main.py @@ -5,33 +5,45 @@ # be deleted. import time +import random import torchcompat.core as accelerator from benchmate.observer import BenchObserver +def criterion(*args, **kwargs): + return random.normalvariate() + + def main(): device = accelerator.fetch_device(0) # <= This is your cuda device - - observer = BenchObserver(batch_size_fn=lambda batch: 1) + + observer = BenchObserver( + batch_size_fn=lambda batch: 1 + ) # optimizer = observer.optimizer(optimizer) # criterion = observer.criterion(criterion) - dataloader = [1, 2, 3, 4] + dataloader = list(range(6000)) - for epoch in range(10): + for epoch in range(10000): for i in observer.iterate(dataloader): # avoid .item() # avoid torch.cuda; use accelerator from torchcompat instead # avoid torch.cuda.synchronize or accelerator.synchronize # y = model(i) - # loss = criterion(y) + loss = criterion() # loss.backward() # optimizer.step() + + observer.record_loss(loss) time.sleep(0.1) + assert epoch < 2, "milabench stopped the train script before the end of training" + assert i < 72, "milabench stopped the train script before the end of training" + if __name__ == "__main__": main() diff --git a/benchmarks/_template/prepare.py b/benchmarks/_templates/simple/prepare.py similarity index 100% rename from benchmarks/_template/prepare.py rename to benchmarks/_templates/simple/prepare.py diff --git a/benchmarks/_template/requirements.in b/benchmarks/_templates/simple/requirements.in similarity index 77% rename from benchmarks/_template/requirements.in rename to benchmarks/_templates/simple/requirements.in index cd5826d4a..94575179d 100644 --- a/benchmarks/_template/requirements.in +++ b/benchmarks/_templates/simple/requirements.in @@ -1 +1,2 @@ voir>=0.2.9,<0.3 +torch \ No newline at end of file diff --git a/benchmarks/_template/voirfile.py b/benchmarks/_templates/simple/voirfile.py similarity index 76% rename from benchmarks/_template/voirfile.py rename to benchmarks/_templates/simple/voirfile.py index 8008df299..d93f886cd 100644 --- a/benchmarks/_template/voirfile.py +++ b/benchmarks/_templates/simple/voirfile.py @@ -26,11 +26,6 @@ class Config: @configurable def instrument_main(ov, options: Config): - try: - import torch - except ImportError: - torch = None - yield ov.phases.init if options.dash: @@ -38,13 +33,6 @@ def instrument_main(ov, options: Config): ov.require( log("value", "progress", "rate", "units", "loss", "gpudata", context="task"), - rate( - interval=options.interval, - skip=options.skip, - sync=torch.cuda.synchronize - if torch and torch.cuda.is_available() - else None, - ), early_stop(n=options.stop, key="rate", task="train"), monitor_monogpu(poll_interval=options.gpu_poll), ) diff --git a/benchmarks/_templates/voir/Makefile b/benchmarks/_templates/voir/Makefile new file mode 100644 index 000000000..f295ac5bb --- /dev/null +++ b/benchmarks/_templates/voir/Makefile @@ -0,0 +1,4 @@ +tests: + milabench install --config dev.yaml --base base + milabench prepare --config dev.yaml --base base + milabench run --config dev.yaml --base base diff --git a/benchmarks/_templates/voir/README.md b/benchmarks/_templates/voir/README.md new file mode 100644 index 000000000..239a2dfaf --- /dev/null +++ b/benchmarks/_templates/voir/README.md @@ -0,0 +1,4 @@ + +# Template + +Rewrite this README to explain what the benchmark is! diff --git a/benchmarks/_templates/voir/benchfile.py b/benchmarks/_templates/voir/benchfile.py new file mode 100644 index 000000000..4ab9e833f --- /dev/null +++ b/benchmarks/_templates/voir/benchfile.py @@ -0,0 +1,42 @@ +from milabench.pack import Package + + +SOURCE_DIR = "src" +REPO_URL = "https://github.com/Delaunay/extern_example.git" +BRANCH = "a524286ab6364bca6729dd6ef4936e175a87c7e4" + + +class Template(Package): + # Requirements file installed by install(). It can be empty or absent. + base_requirements = "requirements.in" + + # The preparation script called by prepare(). It must be executable, + # but it can be any type of script. It can be empty or absent. + prepare_script = "prepare.py" + + # The main script called by run(). It must be a Python file. It has to + # be present. + main_script = f"{SOURCE_DIR}/main.py" + + # You can remove the functions below if you don't need to modify them. + + def make_env(self): + # Return a dict of environment variables for prepare_script and + # main_script. + return super().make_env() + + async def install(self): + await super().install() + + source_destination = self.dirs.code / SOURCE_DIR + if not source_destination.exists(): + source_destination.clone_subtree( + REPO_URL, BRANCH + ) + + async def prepare(self): + await super().prepare() # super() call executes prepare_script + + + +__pack__ = Template diff --git a/benchmarks/_templates/voir/dev.yaml b/benchmarks/_templates/voir/dev.yaml new file mode 100644 index 000000000..f25e977b2 --- /dev/null +++ b/benchmarks/_templates/voir/dev.yaml @@ -0,0 +1,7 @@ + +template: + inherits: _defaults + definition: . + install-variant: unpinned + plan: + method: per_gpu diff --git a/benchmarks/_templates/voir/prepare.py b/benchmarks/_templates/voir/prepare.py new file mode 100755 index 000000000..32bd5901d --- /dev/null +++ b/benchmarks/_templates/voir/prepare.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +import os + +if __name__ == "__main__": + # If you need the whole configuration: + # config = json.loads(os.environ["MILABENCH_CONFIG"]) + + data_directory = os.environ["MILABENCH_DIR_DATA"] + + # Download (or generate) the needed dataset(s). You are responsible + # to check if it has already been properly downloaded or not, and to + # do nothing if it has been. + print("Hello I am doing some data stuff!") + + # If there is nothing to download or generate, just delete this file. diff --git a/benchmarks/_templates/voir/requirements.in b/benchmarks/_templates/voir/requirements.in new file mode 100644 index 000000000..94575179d --- /dev/null +++ b/benchmarks/_templates/voir/requirements.in @@ -0,0 +1,2 @@ +voir>=0.2.9,<0.3 +torch \ No newline at end of file diff --git a/benchmarks/_templates/voir/voirfile.py b/benchmarks/_templates/voir/voirfile.py new file mode 100644 index 000000000..854ced70b --- /dev/null +++ b/benchmarks/_templates/voir/voirfile.py @@ -0,0 +1,69 @@ +from dataclasses import dataclass + +from voir.phase import StopProgram +from voir import configurable +from voir.instruments import dash, early_stop, log +from benchmate.monitor import monitor_monogpu +from benchmate.observer import BenchObserver + + +@dataclass +class Config: + """voir configuration""" + + # Whether to display the dash or not + dash: bool = False + + # How often to log the rates + interval: str = "1s" + + # Number of rates to skip before logging + skip: int = 5 + + # Number of rates to log before stopping + stop: int = 20 + + # Number of seconds between each gpu poll + gpu_poll: int = 3 + + +@configurable +def instrument_main(ov, options: Config): + yield ov.phases.init + + + yield ov.phases.load_script + + if options.dash: + ov.require(dash) + + ov.require( + log("value", "progress", "rate", "units", "loss", "gpudata", context="task"), + early_stop(n=options.stop, key="rate", task="train"), + monitor_monogpu(poll_interval=options.gpu_poll), + ) + + # + # Insert milabench tools + # + observer = BenchObserver( + earlystop=options.stop + options.skip, + batch_size_fn=lambda x: 1 + ) + + probe = ov.probe("//my_dataloader_creator() as loader", overridable=True) + probe['loader'].override(observer.loader) + + probe = ov.probe("//my_criterion_creator() as criterion", overridable=True) + probe['criterion'].override(observer.criterion) + + probe = ov.probe("//my_optimizer_creator() as optimizer", overridable=True) + probe['optimizer'].override(observer.optimizer) + + # + # Run the benchmark + # + try: + yield ov.phases.run_script + except StopProgram: + print("early stopped") \ No newline at end of file diff --git a/benchmarks/torchvision_ddp/benchfile.py b/benchmarks/torchvision_ddp/benchfile.py index 3fc2cb054..4b2d56c8e 100644 --- a/benchmarks/torchvision_ddp/benchfile.py +++ b/benchmarks/torchvision_ddp/benchfile.py @@ -8,7 +8,6 @@ class TorchvisionBenchmarkDDP(Package): def build_run_plan(self) -> "Command": import milabench.commands as cmd - pack = cmd.PackCommand(self, *self.argv, lazy=True) pack = cmd.ActivatorCommand(pack, use_stdout=True) return pack diff --git a/benchmate/benchmate/observer.py b/benchmate/benchmate/observer.py index 0ada095b4..83a6dfa08 100644 --- a/benchmate/benchmate/observer.py +++ b/benchmate/benchmate/observer.py @@ -68,7 +68,7 @@ def override_return_value(self, function, override): raise RuntimeError("Not running through voir") def iterate(self, iterator): - return self.loader(loader) + return self.loader(iterator) def loader(self, loader): """Wrap a dataloader or an iterable which enable accurate measuring of time spent in the loop's body""" @@ -94,7 +94,7 @@ def new_backward(*args, **kwargs): loss.backward = new_backward - self.record_loss(loss.detach()) + self.record_loss(loss) return loss return wrapped diff --git a/docs/index.rst b/docs/index.rst index 3ac990fcf..ebbb27383 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,12 +7,15 @@ Welcome to milabench's documentation! :caption: Contents: usage.rst + recipes.rst + new_benchmarks.rst + docker.rst dev-usage.rst - new_benchmarks.rst reference.rst sizer.rst + Indices and tables ================== diff --git a/docs/new_benchmarks.rst b/docs/new_benchmarks.rst index 058b99c0d..e348a28be 100644 --- a/docs/new_benchmarks.rst +++ b/docs/new_benchmarks.rst @@ -2,23 +2,30 @@ Creating a new benchmark ------------------------ -To define a new benchmark (let's assume it is called ``ornatebench``), make a copy of ``benchmarks/_template`` using ``cp-template``: +To define a new benchmark (let's assume it is called ``ornatebench``), .. code-block:: bash - cp-template benchmarks/_template/ benchmarks/ornatebench + git clone https://github.com/mila-iqia/milabench.git + git checkout -b ornatebench + + pip install -e milabench/ + milabench new --name ornatebench + You should see a directory with the following structure: .. code-block:: - ornatebench - |- README.md # Document the benchmark here - |- benchfile.py # Benchmark definition file - |- main.py # Executed by milabench run - |- prepare.py # Executed by milabench prepare (EXECUTABLE) - |- requirements.in # Python requirements to install from pip - |- voirfile.py # Probes and extra instruments + milabench + └── benchmarks + └── ornatebench + ├── README.md # Document the benchmark here + ├── benchfile.py # Benchmark definition file + ├── main.py # Executed by milabench run + ├── prepare.py # Executed by milabench prepare (EXECUTABLE) + ├── requirements.in # Python requirements to install from pip + └── voirfile.py # Probes and extra instruments Some of these files may be unnecessary depending on the benchmark. @@ -26,30 +33,37 @@ First of all, if you want to verify that everything works, you can use the ``dev .. code-block:: bash - # You can also use --config - export MILABENCH_CONFIG=benchmarks/ornatebench/dev.yaml + cd milabench/benchmarks/ornatebench + + milabench install --config dev.yaml --base . - milabench install - milabench prepare - milabench run + milabench prepare --config dev.yaml --base . + + milabench run --config dev.yaml --base . Overview ~~~~~~~~ - benchfile.py ++++++++++++ -``benchfile.py`` defines what to do on ``milabench install/prepare/run``. It is run from the benchmark directory directly, in the *current* virtual environment, but it can create *new processes* in the virtual environment of the benchmark. +``benchfile.py`` defines what to do on ``milabench install/prepare/run``. +It is run from the benchmark directory directly, in the *current* virtual environment, +but it can create *new processes* in the virtual environment of the benchmark. -By default it will dispatch to ``requirements.in`` for install requirements, ``prepare.py`` for prep work and downloading datasets, and ``main.py`` for running the actual benchmark. If that is suitable you may not need to change it at all. +By default it will dispatch to ``requirements.in`` for install requirements, +``prepare.py`` for prep work and downloading datasets, and +``main.py`` for running the actual benchmark. +If that is suitable you may not need to change it at all. requirements.in +++++++++++++++ -Write all of the benchmark's requirements in this file. Use ``milabench install --config benchmarks/ornatebench/dev.yaml`` to install them during development (add ``--force`` if you made changes and want to reinstall.) +Write all of the benchmark's requirements in this file. +Use ``milabench install --config benchmarks/ornatebench/dev.yaml`` +to install them during development (add ``--force`` if you made changes and want to reinstall.) prepare.py @@ -123,7 +137,6 @@ voirfile.py The voirfile contains instrumentation for the main script. You can usually just leave it as it is. By default, it will: -* Compute the train "rate" (number of samples per second) using events from ``voir.iterate``. * Forcefully stop the program after a certain number of rate measurements. * Monitor GPU usage. @@ -131,9 +144,11 @@ The voirfile contains instrumentation for the main script. You can usually just Development ~~~~~~~~~~~ -To develop the benchmark, first run ``milabench dev --config benchmarks/BENCHNAME/dev.yaml``. This will activate the benchmark's virtual environment and put you into a shell. +To develop the benchmark, first run ``milabench dev --config benchmarks/BENCHNAME/dev.yaml``. +This will activate the benchmark's virtual environment and put you into a shell. -Then, try and run ``voir --dash main.py``. This should show you a little dashboard and display losses, train rate calculations and one or more progress bars. +Then, try and run ``voir --dash main.py``. This should show you a little dashboard and display losses, +train rate calculations and one or more progress bars. From there, you can develop as you would any other Python program. @@ -159,9 +174,43 @@ This will create ``requirements..txt`` for these two architectures. These ``--variant unpinned`` means installing directly from ``requirements.in``. This can be useful during development, but less stable over time since various dependencies may break. -.. Adapting existing code -.. ~~~~~~~~~~~~~~~~~~~~~~ +Adapting existing repository +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To simplify the creation of benchmarks, milabench can use ptera and voir to override or wrap code from a third party +without modifying the third party code. + +.. code-block:: bash + + git clone https://github.com/mila-iqia/milabench.git + git checkout -b ornatebench + + pip install -e milabench/ + milabench new --name ornatebench --repo-url https://github.com/Delaunay/extern_example.git + + +The instrumentation is inserted inside ``voirfile.py`` using the ``Overseer.probe``, examples can +be found in ptera `documentation `_ + + +Wrap a return value ++++++++++++++++++++ + + +.. code-block:: python + + class Wrapper: + def __init__(self, value): + self.value = value + + def wrap(original): + return Wrapper(original) + + probe = ov.probe("//my_optimizer_creator() as optimizer", overridable=True) + probe['optimizer'].override(wrap) + -.. Now, let's say you want to adapt code from the repo at ``https://github.com/snakeoilplz/agi``, more specifically the ``train.py`` script. +* ``//my_optimizer_creator() as optimizer``: get the return value of a function inside the main script +* ``/module.path.function() as optimizer``: get the return value of a function inside a module +* ``/module.path.function > loss_fn``: get a variable inside a function inside a module -.. TODO diff --git a/docs/start.rst b/docs/recipes.rst similarity index 99% rename from docs/start.rst rename to docs/recipes.rst index ce5c8b8d3..91b3f9620 100644 --- a/docs/start.rst +++ b/docs/recipes.rst @@ -1,6 +1,5 @@ - -Getting Started -=============== +Running Milabench +================= Base Setup ---------- @@ -12,17 +11,22 @@ Base Setup mkdir milabench cd milabench git clone https://github.com/mila-iqia/milabench.git + conda activate base python --version Python 3.11.4 + virtualenv ./env source ./env/bin/activate pip install -e milabench/ + export MILABENCH_WORDIR="$(pwd)" export MILABENCH_BASE="$MILABENCH_WORDIR/results" export MILABENCH_CONFIG="$MILABENCH_WORDIR/milabench/config/standard.yaml" export BENCHMARK_VENV="$MILABENCH_WORDIR/results/venv/torch" + module load cuda/12.3.2 # <= or set CUDA_HOME to the right spot + milabench install milabench prepare milabench run diff --git a/milabench/cli/__init__.py b/milabench/cli/__init__.py index e0d57c1e9..205942e47 100644 --- a/milabench/cli/__init__.py +++ b/milabench/cli/__init__.py @@ -20,9 +20,14 @@ from .sql import cli_sqlsetup from .summary import cli_summary from .resolve import cli_resolve +from .new import cli_new class Main: + def new(): + """Create a new benchmark from template""" + return cli_new() + def run(): """Run the benchmarks.""" return cli_run() diff --git a/milabench/cli/new.py b/milabench/cli/new.py new file mode 100644 index 000000000..9f80c5629 --- /dev/null +++ b/milabench/cli/new.py @@ -0,0 +1,115 @@ + +from dataclasses import dataclass +import os +import pathlib +import argparse + +from coleo import Option, tooled + +benchmark = (pathlib.Path(__file__).parent / '..' / '..' / "benchmarks").resolve() +template = benchmark / "_templates" + + +multigpu = "\n".join([ + "method: njobs", + " n: 1\n", +]) + +multinode = "\n".join([ + " num_machines: 2", + " requires_capabilities:", + " - \"len(nodes) >= ${num_machines}\"\n", +]) + +placeholder_repo = "https://github.com/Delaunay/extern_example.git" + + +# fmt: off +@dataclass +class Arguments: + name : str + template : str = "simple" + repo_url : str = None + multi_gpu : bool = False + multi_node : bool = False +# fmt: on + + +@tooled +def arguments(): + # Name of the benchmark + name: Option & str + + # Number of times to repeat the benchmark + template: Option & str = "simple" + + # Repo URL to clone + repo_url: Option & str = None + + # is benchmark multi gpu + multi_gpu: Option & bool = False + + # is the benchmark is multi node + multi_node: Option & bool = False + + return Arguments(name, template, repo_url, multi_gpu, multi_node) + + +@tooled +def cli_new(args=None): + """Create a new benchmark from the template""" + + if args is None: + args = arguments() + + if args.repo_url is not None: + args.template = "voir" + + package_name = args.name.capitalize() + + template_dir = template / args.template + destination = benchmark / args.name + os.makedirs(destination, exist_ok=True) + + for file in os.listdir(template_dir): + if file in ("base",): + continue + + source = template_dir / file + dest = destination / file + + with open(source, "r") as fp: + content = fp.read() + + placeholders = [ + ("Template", package_name), + ("template", args.name), + (placeholder_repo, args.repo_url), + ] + + if args.multi_gpu: + placeholders.append(("method: per_gpu\n", multigpu)) + + if args.repo_url: + placeholders.append(("method: per_gpu\n", args.repo_url)) + + if args.multi_node: + placeholders((None, multinode)) + + for placeholder, value in placeholders: + if value is not None: + if placeholder is not None: + content = content.replace(placeholder, value) + else: + content += value + + with open(dest, "w") as fp: + fp.write(content) + + st = os.stat(source) + os.chown(dest, st.st_uid, st.st_gid) + os.chmod(dest, st.st_mode) + + +if __name__ == "__main__": + cli_new() diff --git a/milabench/commands/__init__.py b/milabench/commands/__init__.py index f6f19a87a..76bec08b9 100644 --- a/milabench/commands/__init__.py +++ b/milabench/commands/__init__.py @@ -612,7 +612,7 @@ class PerGPU(ListCommand): """ def __init__(self, executor: Command, gpus: list = None, **kwargs) -> None: - if gpus is None: + if gpus is None or len(gpus) == 0: gpus = [{"device": 0, "selection_variable": "CPU_VISIBLE_DEVICE"}] self.devices = gpus diff --git a/milabench/pack.py b/milabench/pack.py index 5b30d05b8..5ad45ccff 100644 --- a/milabench/pack.py +++ b/milabench/pack.py @@ -374,7 +374,7 @@ async def install(self): """ assert self.phase == "install" - for reqs in self.requirements_files(self.config.get("install_variant", None)): + for reqs in self.requirements_files(self.config.get("install-variant", None)): if reqs.exists(): await self.pip_install("-r", reqs) else: