Generate pinned dependencies

.pin/constraints-cuda-torch.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile --config=pyproject.toml --output-file=.pin/constraints-cuda-torch.txt --resolver=backtracking .pin/tmp-constraints.txt benchmarks/accelerate_opt/requirements.in benchmarks/dlrm/requirements.in benchmarks/huggingface/requirements.in benchmarks/rwkv/requirements.in benchmarks/stargan/requirements.in benchmarks/super-slomo/requirements.in benchmarks/timm/requirements.in benchmarks/torchvision/requirements.in
+#    pip-compile --config=pyproject.toml --output-file=.pin/constraints-cuda-torch.txt --resolver=backtracking .pin/tmp-constraints.txt benchmarks/accelerate_opt/requirements.in benchmarks/dlrm/requirements.in benchmarks/flops/requirements.in benchmarks/huggingface/requirements.in benchmarks/rwkv/requirements.in benchmarks/stargan/requirements.in benchmarks/super-slomo/requirements.in benchmarks/timm/requirements.in benchmarks/torchvision/requirements.in
 #
 --extra-index-url https://download.pytorch.org/whl/cu118
 
@@ -146,7 +146,7 @@ ninja==1.11.1.1
     #   deepspeed
 numpy==1.26.1
     # via
-    #   -r benchmarks/stargan/requirements.in
+    #   -r benchmarks/rwkv/requirements.in
     #   -r benchmarks/super-slomo/requirements.in
     #   accelerate
     #   datasets
@@ -294,7 +294,7 @@ tokenizers==0.14.1
     # via transformers
 torch==2.1.0+cu118
     # via
-    #   -r benchmarks/huggingface/requirements.in
+    #   -r benchmarks/accelerate_opt/requirements.in
     #   -r benchmarks/super-slomo/requirements.in
     #   accelerate
     #   deepspeed
@@ -313,8 +313,8 @@ torchrec==0.5.0+cu118
     # via -r benchmarks/dlrm/requirements.in
 torchvision==0.16.0+cu118
     # via
+    #   -r benchmarks/accelerate_opt/requirements.in
     #   -r benchmarks/super-slomo/requirements.in
-    #   -r benchmarks/torchvision/requirements.in
 torchviz==0.0.2
     # via -r benchmarks/dlrm/requirements.in
 torchx==0.6.0
@@ -361,7 +361,7 @@ varname==0.10.0
     # via giving
 voir @ git+https://github.com/breuleux/voir.git
     # via
-    #   -r benchmarks/huggingface/requirements.in
+    #   -r benchmarks/accelerate_opt/requirements.in
     #   -r benchmarks/super-slomo/requirements.in
 websocket-client==1.6.4
     # via docker

.pin/constraints-rocm-torch.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile --config=pyproject.toml --output-file=.pin/constraints-rocm-torch.txt --resolver=backtracking .pin/tmp-constraints.txt benchmarks/accelerate_opt/requirements.in benchmarks/dlrm/requirements.in benchmarks/huggingface/requirements.in benchmarks/rwkv/requirements.in benchmarks/stargan/requirements.in benchmarks/super-slomo/requirements.in benchmarks/timm/requirements.in benchmarks/torchvision/requirements.in
+#    pip-compile --config=pyproject.toml --output-file=.pin/constraints-rocm-torch.txt --resolver=backtracking .pin/tmp-constraints.txt benchmarks/accelerate_opt/requirements.in benchmarks/dlrm/requirements.in benchmarks/flops/requirements.in benchmarks/huggingface/requirements.in benchmarks/rwkv/requirements.in benchmarks/stargan/requirements.in benchmarks/super-slomo/requirements.in benchmarks/timm/requirements.in benchmarks/torchvision/requirements.in
 #
 --extra-index-url https://download.pytorch.org/whl/rocm5.6/
 
@@ -150,8 +150,8 @@ ninja==1.11.1.1
     #   deepspeed
 numpy==1.26.1
     # via
-    #   -r benchmarks/dlrm/requirements.in
     #   -r benchmarks/stargan/requirements.in
+    #   -r benchmarks/super-slomo/requirements.in
     #   accelerate
     #   datasets
     #   deepspeed
@@ -302,7 +302,7 @@ tokenizers==0.14.1
 torch==2.1.0+rocm5.6
     # via
     #   -r benchmarks/stargan/requirements.in
-    #   -r benchmarks/timm/requirements.in
+    #   -r benchmarks/super-slomo/requirements.in
     #   accelerate
     #   deepspeed
     #   pytorch-lightning
@@ -322,15 +322,15 @@ torchrec==0.5.0
 torchvision==0.16.0+rocm5.6
     # via
     #   -r benchmarks/stargan/requirements.in
-    #   -r benchmarks/timm/requirements.in
+    #   -r benchmarks/super-slomo/requirements.in
 torchviz==0.0.2
     # via -r benchmarks/dlrm/requirements.in
 torchx==0.6.0
     # via -r benchmarks/dlrm/requirements.in
 tqdm==4.66.1
     # via
-    #   -r benchmarks/dlrm/requirements.in
-    #   -r benchmarks/torchvision/requirements.in
+    #   -r benchmarks/flops/requirements.in
+    #   -r benchmarks/super-slomo/requirements.in
     #   datasets
     #   deepspeed
     #   evaluate
@@ -368,7 +368,7 @@ varname==0.10.0
 voir @ git+https://github.com/breuleux/voir.git
     # via
     #   -r benchmarks/stargan/requirements.in
-    #   -r benchmarks/timm/requirements.in
+    #   -r benchmarks/super-slomo/requirements.in
 websocket-client==1.6.4
     # via docker
 werkzeug==3.0.1

benchmarks/flops/prepare.py

@@ -0,0 +1 @@
+#!/usr/bin/env python

benchmarks/flops/requirements.cuda.txt

@@ -0,0 +1,81 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile --config=pyproject.toml --output-file=benchmarks/flops/requirements.cuda.txt --resolver=backtracking .pin/tmp-constraints-cuda-torchvision.txt benchmarks/flops/requirements.in
+#
+--extra-index-url https://download.pytorch.org/whl/cu118
+
+certifi==2023.7.22
+    # via
+    #   -c .pin/../.pin/constraints-cuda-torch.txt
+    #   requests
+charset-normalizer==3.3.1
+    # via
+    #   -c .pin/../.pin/constraints-cuda-torch.txt
+    #   requests
+filelock==3.12.4
+    # via
+    #   -c .pin/../.pin/constraints-cuda-torch.txt
+    #   torch
+    #   triton
+fsspec==2023.1.0
+    # via
+    #   -c .pin/../.pin/constraints-cuda-torch.txt
+    #   torch
+idna==3.4
+    # via
+    #   -c .pin/../.pin/constraints-cuda-torch.txt
+    #   requests
+jinja2==3.1.2
+    # via
+    #   -c .pin/../.pin/constraints-cuda-torch.txt
+    #   torch
+markupsafe==2.1.3
+    # via
+    #   -c .pin/../.pin/constraints-cuda-torch.txt
+    #   jinja2
+mpmath==1.3.0
+    # via
+    #   -c .pin/../.pin/constraints-cuda-torch.txt
+    #   sympy
+networkx==3.2
+    # via
+    #   -c .pin/../.pin/constraints-cuda-torch.txt
+    #   torch
+numpy==1.26.1
+    # via
+    #   -c .pin/../.pin/constraints-cuda-torch.txt
+    #   torchvision
+pillow==10.1.0
+    # via
+    #   -c .pin/../.pin/constraints-cuda-torch.txt
+    #   torchvision
+requests==2.31.0
+    # via
+    #   -c .pin/../.pin/constraints-cuda-torch.txt
+    #   torchvision
+sympy==1.12
+    # via
+    #   -c .pin/../.pin/constraints-cuda-torch.txt
+    #   torch
+torch==2.1.0+cu118
+    # via
+    #   -r benchmarks/flops/requirements.in
+    #   torchvision
+torchvision==0.16.0+cu118
+    # via -r benchmarks/flops/requirements.in
+tqdm==4.66.1
+    # via -r benchmarks/flops/requirements.in
+triton==2.1.0
+    # via
+    #   -c .pin/../.pin/constraints-cuda-torch.txt
+    #   torch
+typing-extensions==4.8.0
+    # via
+    #   -c .pin/../.pin/constraints-cuda-torch.txt
+    #   torch
+urllib3==1.26.18
+    # via
+    #   -c .pin/../.pin/constraints-cuda-torch.txt
+    #   requests

benchmarks/flops/requirements.in

@@ -1,4 +1,3 @@
 torch
 torchvision
 tqdm
-voir>=0.2.9,<0.3

benchmarks/flops/requirements.rocm.txt

@@ -0,0 +1,90 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile --config=pyproject.toml --output-file=benchmarks/flops/requirements.rocm.txt --resolver=backtracking .pin/tmp-constraints-rocm-torchvision.txt benchmarks/flops/requirements.in
+#
+--extra-index-url https://download.pytorch.org/whl/rocm5.6/
+
+certifi==2023.7.22
+    # via
+    #   -c .pin/../.pin/constraints-rocm-torch.txt
+    #   requests
+charset-normalizer==3.3.1
+    # via
+    #   -c .pin/../.pin/constraints-rocm-torch.txt
+    #   requests
+cmake==3.27.7
+    # via
+    #   -c .pin/../.pin/constraints-rocm-torch.txt
+    #   pytorch-triton-rocm
+filelock==3.12.4
+    # via
+    #   -c .pin/../.pin/constraints-rocm-torch.txt
+    #   pytorch-triton-rocm
+    #   torch
+fsspec==2023.1.0
+    # via
+    #   -c .pin/../.pin/constraints-rocm-torch.txt
+    #   torch
+idna==3.4
+    # via
+    #   -c .pin/../.pin/constraints-rocm-torch.txt
+    #   requests
+jinja2==3.1.2
+    # via
+    #   -c .pin/../.pin/constraints-rocm-torch.txt
+    #   torch
+lit==17.0.3
+    # via
+    #   -c .pin/../.pin/constraints-rocm-torch.txt
+    #   pytorch-triton-rocm
+markupsafe==2.1.3
+    # via
+    #   -c .pin/../.pin/constraints-rocm-torch.txt
+    #   jinja2
+mpmath==1.3.0
+    # via
+    #   -c .pin/../.pin/constraints-rocm-torch.txt
+    #   sympy
+networkx==3.2
+    # via
+    #   -c .pin/../.pin/constraints-rocm-torch.txt
+    #   torch
+numpy==1.26.1
+    # via
+    #   -c .pin/../.pin/constraints-rocm-torch.txt
+    #   torchvision
+pillow==10.1.0
+    # via
+    #   -c .pin/../.pin/constraints-rocm-torch.txt
+    #   torchvision
+pytorch-triton-rocm==2.1.0
+    # via
+    #   -c .pin/../.pin/constraints-rocm-torch.txt
+    #   torch
+requests==2.31.0
+    # via
+    #   -c .pin/../.pin/constraints-rocm-torch.txt
+    #   torchvision
+sympy==1.12
+    # via
+    #   -c .pin/../.pin/constraints-rocm-torch.txt
+    #   torch
+torch==2.1.0+rocm5.6
+    # via
+    #   -r benchmarks/flops/requirements.in
+    #   pytorch-triton-rocm
+    #   torchvision
+torchvision==0.16.0+rocm5.6
+    # via -r benchmarks/flops/requirements.in
+tqdm==4.66.1
+    # via -r benchmarks/flops/requirements.in
+typing-extensions==4.8.0
+    # via
+    #   -c .pin/../.pin/constraints-rocm-torch.txt
+    #   torch
+urllib3==1.26.18
+    # via
+    #   -c .pin/../.pin/constraints-rocm-torch.txt
+    #   requests

benchmarks/flops/voirfile.py

@@ -0,0 +1,40 @@
+from dataclasses import dataclass
+
+from voir import configurable
+from voir.instruments import dash, early_stop, gpu_monitor, log, rate
+
+
+@dataclass
+class Config:
+    """voir configuration"""
+
+    # Whether to display the dash or not
+    dash: bool = False
+
+    # How often to log the rates
+    interval: str = "1s"
+
+    # Number of rates to skip before logging
+    skip: int = 5
+
+    # Number of rates to log before stopping
+    stop: int = 1
+
+    # Number of seconds between each gpu poll
+    gpu_poll: int = 3
+
+
+@configurable
+def instrument_main(ov, options: Config):
+    import torch
+
+    yield ov.phases.init
+
+    if options.dash:
+        ov.require(dash)
+
+    ov.require(
+        log("value", "progress", "rate", "units", "loss", "gpudata", context="task"),
+        early_stop(n=options.stop, key="rate", task="train"),
+        gpu_monitor(poll_interval=options.gpu_poll),
+    )

config/base.yaml

@@ -26,7 +26,7 @@ _torchvision:
 _flops:
   inherits: _defaults
   definition: ../benchmarks/flops
-  group: torchvision
+  group: flops
   install_group: torch
   plan:
     method: per_gpu