Add new bench to config file

mila-iqia · Aug 15, 2024 · c0930cb · c0930cb
1 parent 1c4d9a4
commit c0930cb
Show file tree

Hide file tree

Showing 46 changed files with 1,167 additions and 314 deletions.
diff --git a/README.md b/README.md
@@ -39,7 +39,7 @@ It will include all of the necessary data
     docker run -it --rm --ipc=host --gpus=all      \
           -v $(pwd)/results:/milabench/envs/runs   \
           $MILABENCH_IMAGE                         \
-          milabench run
+          bash -c "milabench prepare && milabench run"
 
     =================
     Benchmark results

diff --git a/benchmarks/geo_gnn/requirements-pre.in b/benchmarks/geo_gnn/requirements-pre.in
@@ -1 +1 @@
-torch<2.4
+torch
diff --git a/benchmarks/geo_gnn/requirements.in b/benchmarks/geo_gnn/requirements.in
@@ -1,6 +1,3 @@
-
---find-links https://data.pyg.org/whl/torch-2.3.0+cu121.html
-
 voir>=0.2.17,<0.3
 torch-geometric
 torch-cluster

diff --git a/benchmarks/llm/dev.yaml b/benchmarks/llm/dev.yaml
@@ -13,6 +13,27 @@ _llm:
     method: per_gpu
 
 
+llm-rlhf-single:
+  inherits: _llm
+  definition: .
+  install-variant: unpinned
+  plan:
+    method: per_gpu
+
+  argv:
+    "{milabench_code}/recipes/lora_finetune_single_device.py": true
+    --config: "{milabench_code}/configs/llama3_8B_lora_single_device.yaml"
+    epochs=1: true
+    output_dir={milabench_extra}/output: true
+    tokenizer.path={milabench_data}/llama3_8B/original/tokenizer.model: true
+    checkpointer.checkpoint_dir={milabench_data}/llama3_8B/original: true
+    checkpointer.output_dir={milabench_data}/llama3_8B/: true
+    metric_logger.log_dir={milabench_extra}/metrics: true
+    repo_id="meta-llama/Meta-Llama-3.1-8B": true
+    batch_size=8: true
+    gradient_accumulation_steps=8: true
+
+
 llm-lora-single:
   inherits: _llm
   definition: .

diff --git a/benchmarks/llm/recipes/ppo_full_finetune_single_device.py b/benchmarks/llm/recipes/ppo_full_finetune_single_device.py
diff --git a/benchmarks/stargan/README.md → benchmarks/retired/stargan/README.md b/benchmarks/stargan/README.md → benchmarks/retired/stargan/README.md
diff --git a/benchmarks/stargan/benchfile.py → benchmarks/retired/stargan/benchfile.py b/benchmarks/stargan/benchfile.py → benchmarks/retired/stargan/benchfile.py
diff --git a/benchmarks/stargan/prepare.py → benchmarks/retired/stargan/prepare.py b/benchmarks/stargan/prepare.py → benchmarks/retired/stargan/prepare.py
diff --git a/benchmarks/stargan/requirements.cuda.txt → ...rks/retired/stargan/requirements.cuda.txt b/benchmarks/stargan/requirements.cuda.txt → ...rks/retired/stargan/requirements.cuda.txt
diff --git a/benchmarks/stargan/requirements.hpu.txt → ...arks/retired/stargan/requirements.hpu.txt b/benchmarks/stargan/requirements.hpu.txt → ...arks/retired/stargan/requirements.hpu.txt
diff --git a/benchmarks/stargan/requirements.in → benchmarks/retired/stargan/requirements.in b/benchmarks/stargan/requirements.in → benchmarks/retired/stargan/requirements.in
diff --git a/benchmarks/stargan/requirements.rocm.txt → ...rks/retired/stargan/requirements.rocm.txt b/benchmarks/stargan/requirements.rocm.txt → ...rks/retired/stargan/requirements.rocm.txt
diff --git a/benchmarks/stargan/requirements.xpu.txt → ...arks/retired/stargan/requirements.xpu.txt b/benchmarks/stargan/requirements.xpu.txt → ...arks/retired/stargan/requirements.xpu.txt
diff --git a/benchmarks/stargan/stargan/LICENSE → benchmarks/retired/stargan/stargan/LICENSE b/benchmarks/stargan/stargan/LICENSE → benchmarks/retired/stargan/stargan/LICENSE
diff --git a/benchmarks/stargan/stargan/ORIGIN.md → benchmarks/retired/stargan/stargan/ORIGIN.md b/benchmarks/stargan/stargan/ORIGIN.md → benchmarks/retired/stargan/stargan/ORIGIN.md
diff --git a/benchmarks/stargan/stargan/README.md → benchmarks/retired/stargan/stargan/README.md b/benchmarks/stargan/stargan/README.md → benchmarks/retired/stargan/stargan/README.md
diff --git a/benchmarks/stargan/stargan/data_loader.py → ...ks/retired/stargan/stargan/data_loader.py b/benchmarks/stargan/stargan/data_loader.py → ...ks/retired/stargan/stargan/data_loader.py
diff --git a/benchmarks/stargan/stargan/download.sh → ...marks/retired/stargan/stargan/download.sh b/benchmarks/stargan/stargan/download.sh → ...marks/retired/stargan/stargan/download.sh
diff --git a/benchmarks/stargan/stargan/logger.py → benchmarks/retired/stargan/stargan/logger.py b/benchmarks/stargan/stargan/logger.py → benchmarks/retired/stargan/stargan/logger.py
diff --git a/benchmarks/stargan/stargan/main.py → benchmarks/retired/stargan/stargan/main.py b/benchmarks/stargan/stargan/main.py → benchmarks/retired/stargan/stargan/main.py
diff --git a/benchmarks/stargan/stargan/model.py → benchmarks/retired/stargan/stargan/model.py b/benchmarks/stargan/stargan/model.py → benchmarks/retired/stargan/stargan/model.py
diff --git a/benchmarks/stargan/stargan/solver.py → benchmarks/retired/stargan/stargan/solver.py b/benchmarks/stargan/stargan/solver.py → benchmarks/retired/stargan/stargan/solver.py
diff --git a/benchmarks/stargan/stargan/synth.py → benchmarks/retired/stargan/stargan/synth.py b/benchmarks/stargan/stargan/synth.py → benchmarks/retired/stargan/stargan/synth.py
diff --git a/benchmarks/stargan/voirfile.py → benchmarks/retired/stargan/voirfile.py b/benchmarks/stargan/voirfile.py → benchmarks/retired/stargan/voirfile.py
diff --git a/benchmarks/rlhf/Makefile b/benchmarks/rlhf/Makefile
diff --git a/benchmarks/rlhf/README.md b/benchmarks/rlhf/README.md
diff --git a/benchmarks/rlhf/dev.yaml b/benchmarks/rlhf/dev.yaml
diff --git a/benchmarks/rlhf/main.py b/benchmarks/rlhf/main.py
diff --git a/benchmarks/rlhf/requirements.in b/benchmarks/rlhf/requirements.in
diff --git a/benchmarks/rlhf/voirfile.py b/benchmarks/rlhf/voirfile.py
diff --git a/benchmarks/torch_ppo_atari_envpool/benchfile.py b/benchmarks/torch_ppo_atari_envpool/benchfile.py
diff --git a/benchmarks/torch_ppo_atari_envpool/prepare.py b/benchmarks/torch_ppo_atari_envpool/prepare.py
diff --git a/benchmarks/torch_ppo_atari_envpool/Makefile → benchmarks/torchatari/Makefile b/benchmarks/torch_ppo_atari_envpool/Makefile → benchmarks/torchatari/Makefile
@@ -5,7 +5,7 @@ endif
 
 export MILABENCH_BASE
 
-BENCH_NAME=torch_ppo_atari_envpool
+BENCH_NAME=torchatari
 MILABENCH_CONFIG=dev.yaml
 MILABENCH_ARGS=--config $(MILABENCH_CONFIG) --base $(MILABENCH_BASE)
 

diff --git a/benchmarks/torch_ppo_atari_envpool/README.md → benchmarks/torchatari/README.md b/benchmarks/torch_ppo_atari_envpool/README.md → benchmarks/torchatari/README.md
diff --git a/benchmarks/rlhf/benchfile.py → benchmarks/torchatari/benchfile.py b/benchmarks/rlhf/benchfile.py → benchmarks/torchatari/benchfile.py
@@ -1,7 +1,7 @@
 from milabench.pack import Package
 
 
-class Rlhf(Package):
+class Torchatari(Package):
     # Requirements file installed by install(). It can be empty or absent.
     base_requirements = "requirements.in"
 
@@ -28,4 +28,4 @@ async def prepare(self):
 
 
 
-__pack__ = Rlhf
+__pack__ = Torchatari
diff --git a/benchmarks/torch_ppo_atari_envpool/dev.yaml → benchmarks/torchatari/dev.yaml b/benchmarks/torch_ppo_atari_envpool/dev.yaml → benchmarks/torchatari/dev.yaml
@@ -1,5 +1,5 @@
 
-torch_ppo_atari_envpool:
+torchatari:
   max_duration: 600
   inherits: _defaults
   definition: .

diff --git a/benchmarks/torch_ppo_atari_envpool/main.py → benchmarks/torchatari/main.py b/benchmarks/torch_ppo_atari_envpool/main.py → benchmarks/torchatari/main.py
diff --git a/benchmarks/rlhf/prepare.py → benchmarks/torchatari/prepare.py b/benchmarks/rlhf/prepare.py → benchmarks/torchatari/prepare.py
diff --git a/...s/torch_ppo_atari_envpool/requirements.in → benchmarks/torchatari/requirements.in b/...s/torch_ppo_atari_envpool/requirements.in → benchmarks/torchatari/requirements.in
diff --git a/...marks/torch_ppo_atari_envpool/voirfile.py → benchmarks/torchatari/voirfile.py b/...marks/torch_ppo_atari_envpool/voirfile.py → benchmarks/torchatari/voirfile.py