Add flops benchmark

.gitignore

@@ -23,3 +23,5 @@ sqlite.db
 
 .no_report
 trash/
+workspace/
+slurm-*

.pin/constraints-cuda-torch.txt

@@ -2,13 +2,13 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile --config=pyproject.toml --output-file=.pin/constraints-cuda-torch.txt --resolver=backtracking .pin/tmp-constraints.txt benchmarks/accelerate_opt/requirements.in benchmarks/dlrm/requirements.in benchmarks/huggingface/requirements.in benchmarks/rwkv/requirements.in benchmarks/stargan/requirements.in benchmarks/super-slomo/requirements.in benchmarks/timm/requirements.in benchmarks/torchvision/requirements.in
+#    pip-compile --config=pyproject.toml --output-file=.pin/constraints-cuda-torch.txt --resolver=backtracking .pin/tmp-constraints.txt benchmarks/accelerate_opt/requirements.in benchmarks/dlrm/requirements.in benchmarks/flops/requirements.in benchmarks/huggingface/requirements.in benchmarks/rwkv/requirements.in benchmarks/stargan/requirements.in benchmarks/super-slomo/requirements.in benchmarks/timm/requirements.in benchmarks/torchvision/requirements.in
 #
 --extra-index-url https://download.pytorch.org/whl/cu118
 
 absl-py==2.0.0
     # via tensorboard
-accelerate==0.24.0
+accelerate==0.24.1
     # via -r benchmarks/accelerate_opt/requirements.in
 aiohttp==3.8.6
     # via
@@ -28,7 +28,7 @@ cachetools==5.3.2
     # via google-auth
 certifi==2023.7.22
     # via requests
-charset-normalizer==3.3.1
+charset-normalizer==3.3.2
     # via
     #   aiohttp
     #   requests
@@ -57,7 +57,7 @@ executing==1.2.0
     # via varname
 fbgemm-gpu==0.5.0+cu118
     # via torchrec
-filelock==3.12.4
+filelock==3.13.1
     # via
     #   huggingface-hub
     #   torch
@@ -82,15 +82,15 @@ giving==0.4.2
     # via
     #   ptera
     #   voir
-google-auth==2.23.3
+google-auth==2.23.4
     # via
     #   google-auth-oauthlib
     #   tensorboard
 google-auth-oauthlib==1.1.0
     # via tensorboard
 graphviz==0.20.1
     # via torchviz
-grpcio==1.59.0
+grpcio==1.59.2
     # via tensorboard
 hjson==3.1.0
     # via deepspeed
@@ -116,7 +116,7 @@ lightning-utilities==0.9.0
     # via
     #   pytorch-lightning
     #   torchmetrics
-markdown==3.5
+markdown==3.5.1
     # via tensorboard
 markdown-it-py==3.0.0
     # via rich
@@ -138,7 +138,7 @@ multiprocess==0.70.15
     #   evaluate
 mypy-extensions==1.0.0
     # via typing-inspect
-networkx==3.2
+networkx==3.2.1
     # via torch
 ninja==1.11.1.1
     # via
@@ -167,7 +167,7 @@ oauthlib==3.2.2
     # via requests-oauthlib
 omegaconf==2.3.0
     # via voir
-onnx==1.14.1
+onnx==1.15.0
     # via -r benchmarks/dlrm/requirements.in
 opencv-python==4.8.1.78
     # via -r benchmarks/super-slomo/requirements.in
@@ -185,7 +185,7 @@ packaging==23.2
     #   pytorch-lightning
     #   torchmetrics
     #   transformers
-pandas==2.1.1
+pandas==2.1.2
     # via
     #   datasets
     #   evaluate
@@ -203,7 +203,7 @@ ptera==1.4.1
     # via voir
 py-cpuinfo==9.0.0
     # via deepspeed
-pyarrow==13.0.0
+pyarrow==14.0.0
     # via datasets
 pyasn1==0.5.0
     # via
@@ -294,7 +294,7 @@ tokenizers==0.14.1
     # via transformers
 torch==2.1.0+cu118
     # via
-    #   -r benchmarks/huggingface/requirements.in
+    #   -r benchmarks/stargan/requirements.in
     #   -r benchmarks/super-slomo/requirements.in
     #   accelerate
     #   deepspeed
@@ -313,8 +313,8 @@ torchrec==0.5.0+cu118
     # via -r benchmarks/dlrm/requirements.in
 torchvision==0.16.0+cu118
     # via
+    #   -r benchmarks/stargan/requirements.in
     #   -r benchmarks/super-slomo/requirements.in
-    #   -r benchmarks/torchvision/requirements.in
 torchviz==0.0.2
     # via -r benchmarks/dlrm/requirements.in
 torchx==0.6.0
@@ -340,7 +340,6 @@ typing-extensions==4.8.0
     # via
     #   huggingface-hub
     #   lightning-utilities
-    #   onnx
     #   pydantic
     #   pyre-extensions
     #   pytorch-lightning
@@ -359,9 +358,9 @@ urllib3==1.26.18
     #   torchx
 varname==0.10.0
     # via giving
-voir @ git+https://github.com/breuleux/voir.git
+voir==0.2.11
     # via
-    #   -r benchmarks/huggingface/requirements.in
+    #   -r benchmarks/stargan/requirements.in
     #   -r benchmarks/super-slomo/requirements.in
 websocket-client==1.6.4
     # via docker

.pin/constraints-rocm-torch.txt

@@ -2,13 +2,13 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile --config=pyproject.toml --output-file=.pin/constraints-rocm-torch.txt --resolver=backtracking .pin/tmp-constraints.txt benchmarks/accelerate_opt/requirements.in benchmarks/dlrm/requirements.in benchmarks/huggingface/requirements.in benchmarks/rwkv/requirements.in benchmarks/stargan/requirements.in benchmarks/super-slomo/requirements.in benchmarks/timm/requirements.in benchmarks/torchvision/requirements.in
+#    pip-compile --config=pyproject.toml --output-file=.pin/constraints-rocm-torch.txt --resolver=backtracking .pin/tmp-constraints.txt benchmarks/accelerate_opt/requirements.in benchmarks/dlrm/requirements.in benchmarks/flops/requirements.in benchmarks/huggingface/requirements.in benchmarks/rwkv/requirements.in benchmarks/stargan/requirements.in benchmarks/super-slomo/requirements.in benchmarks/timm/requirements.in benchmarks/torchvision/requirements.in
 #
 --extra-index-url https://download.pytorch.org/whl/rocm5.6/
 
 absl-py==2.0.0
     # via tensorboard
-accelerate==0.24.0
+accelerate==0.24.1
     # via -r benchmarks/accelerate_opt/requirements.in
 aiohttp==3.8.6
     # via
@@ -28,7 +28,7 @@ cachetools==5.3.2
     # via google-auth
 certifi==2023.7.22
     # via requests
-charset-normalizer==3.3.1
+charset-normalizer==3.3.2
     # via
     #   aiohttp
     #   requests
@@ -59,7 +59,7 @@ executing==1.2.0
     # via varname
 fbgemm-gpu==0.5.0
     # via torchrec
-filelock==3.12.4
+filelock==3.13.1
     # via
     #   huggingface-hub
     #   pytorch-triton-rocm
@@ -84,15 +84,15 @@ giving==0.4.2
     # via
     #   ptera
     #   voir
-google-auth==2.23.3
+google-auth==2.23.4
     # via
     #   google-auth-oauthlib
     #   tensorboard
 google-auth-oauthlib==1.1.0
     # via tensorboard
 graphviz==0.20.1
     # via torchviz
-grpcio==1.59.0
+grpcio==1.59.2
     # via tensorboard
 hjson==3.1.0
     # via deepspeed
@@ -118,9 +118,9 @@ lightning-utilities==0.9.0
     # via
     #   pytorch-lightning
     #   torchmetrics
-lit==17.0.3
+lit==17.0.4
     # via pytorch-triton-rocm
-markdown==3.5
+markdown==3.5.1
     # via tensorboard
 markdown-it-py==3.0.0
     # via rich
@@ -142,7 +142,7 @@ multiprocess==0.70.15
     #   evaluate
 mypy-extensions==1.0.0
     # via typing-inspect
-networkx==3.2
+networkx==3.2.1
     # via torch
 ninja==1.11.1.1
     # via
@@ -151,7 +151,7 @@ ninja==1.11.1.1
 numpy==1.26.1
     # via
     #   -r benchmarks/dlrm/requirements.in
-    #   -r benchmarks/stargan/requirements.in
+    #   -r benchmarks/rwkv/requirements.in
     #   accelerate
     #   datasets
     #   deepspeed
@@ -172,7 +172,7 @@ oauthlib==3.2.2
     # via requests-oauthlib
 omegaconf==2.3.0
     # via voir
-onnx==1.14.1
+onnx==1.15.0
     # via -r benchmarks/dlrm/requirements.in
 opencv-python==4.8.1.78
     # via -r benchmarks/super-slomo/requirements.in
@@ -190,7 +190,7 @@ packaging==23.2
     #   pytorch-lightning
     #   torchmetrics
     #   transformers
-pandas==2.1.1
+pandas==2.1.2
     # via
     #   datasets
     #   evaluate
@@ -208,7 +208,7 @@ ptera==1.4.1
     # via voir
 py-cpuinfo==9.0.0
     # via deepspeed
-pyarrow==13.0.0
+pyarrow==14.0.0
     # via datasets
 pyasn1==0.5.0
     # via
@@ -301,8 +301,8 @@ tokenizers==0.14.1
     # via transformers
 torch==2.1.0+rocm5.6
     # via
-    #   -r benchmarks/stargan/requirements.in
-    #   -r benchmarks/timm/requirements.in
+    #   -r benchmarks/accelerate_opt/requirements.in
+    #   -r benchmarks/flops/requirements.in
     #   accelerate
     #   deepspeed
     #   pytorch-lightning
@@ -321,16 +321,16 @@ torchrec==0.5.0
     # via -r benchmarks/dlrm/requirements.in
 torchvision==0.16.0+rocm5.6
     # via
-    #   -r benchmarks/stargan/requirements.in
-    #   -r benchmarks/timm/requirements.in
+    #   -r benchmarks/accelerate_opt/requirements.in
+    #   -r benchmarks/flops/requirements.in
 torchviz==0.0.2
     # via -r benchmarks/dlrm/requirements.in
 torchx==0.6.0
     # via -r benchmarks/dlrm/requirements.in
 tqdm==4.66.1
     # via
     #   -r benchmarks/dlrm/requirements.in
-    #   -r benchmarks/torchvision/requirements.in
+    #   -r benchmarks/flops/requirements.in
     #   datasets
     #   deepspeed
     #   evaluate
@@ -346,7 +346,6 @@ typing-extensions==4.8.0
     # via
     #   huggingface-hub
     #   lightning-utilities
-    #   onnx
     #   pydantic
     #   pyre-extensions
     #   pytorch-lightning
@@ -365,10 +364,10 @@ urllib3==1.26.18
     #   torchx
 varname==0.10.0
     # via giving
-voir @ git+https://github.com/breuleux/voir.git
+voir==0.2.11
     # via
-    #   -r benchmarks/stargan/requirements.in
-    #   -r benchmarks/timm/requirements.in
+    #   -r benchmarks/accelerate_opt/requirements.in
+    #   -r benchmarks/flops/requirements.in
 websocket-client==1.6.4
     # via docker
 werkzeug==3.0.1

benchmarks/accelerate_opt/requirements.cuda.txt

@@ -6,7 +6,7 @@
 #
 --extra-index-url https://download.pytorch.org/whl/cu118
 
-accelerate==0.24.0
+accelerate==0.24.1
     # via -r benchmarks/accelerate_opt/requirements.in
 aiohttp==3.8.6
     # via
@@ -37,7 +37,7 @@ certifi==2023.7.22
     # via
     #   -c .pin/../.pin/constraints-cuda-torch.txt
     #   requests
-charset-normalizer==3.3.1
+charset-normalizer==3.3.2
     # via
     #   -c .pin/../.pin/constraints-cuda-torch.txt
     #   aiohttp
@@ -64,7 +64,7 @@ executing==1.2.0
     # via
     #   -c .pin/../.pin/constraints-cuda-torch.txt
     #   varname
-filelock==3.12.4
+filelock==3.13.1
     # via
     #   -c .pin/../.pin/constraints-cuda-torch.txt
     #   huggingface-hub
@@ -135,7 +135,7 @@ multiprocess==0.70.15
     #   -c .pin/../.pin/constraints-cuda-torch.txt
     #   datasets
     #   evaluate
-networkx==3.2
+networkx==3.2.1
     # via
     #   -c .pin/../.pin/constraints-cuda-torch.txt
     #   torch
@@ -171,7 +171,7 @@ packaging==23.2
     #   evaluate
     #   huggingface-hub
     #   transformers
-pandas==2.1.1
+pandas==2.1.2
     # via
     #   -c .pin/../.pin/constraints-cuda-torch.txt
     #   datasets
@@ -193,7 +193,7 @@ py-cpuinfo==9.0.0
     # via
     #   -c .pin/../.pin/constraints-cuda-torch.txt
     #   deepspeed
-pyarrow==13.0.0
+pyarrow==14.0.0
     # via
     #   -c .pin/../.pin/constraints-cuda-torch.txt
     #   datasets
@@ -313,7 +313,7 @@ varname==0.10.0
     # via
     #   -c .pin/../.pin/constraints-cuda-torch.txt
     #   giving
-voir @ git+https://github.com/breuleux/voir.git
+voir==0.2.11
     # via -r benchmarks/accelerate_opt/requirements.in
 xxhash==3.4.1
     # via