Skip to content

Commit

Permalink
Add flops benchmark (#169)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Delaunay
Delaunay authored Nov 6, 2023
1 parent 34d55e6 commit 8e62d37
Show file tree
Hide file tree
Showing 39 changed files with 876 additions and 140 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,5 @@ sqlite.db

.no_report
trash/
workspace/
slurm-*
31 changes: 15 additions & 16 deletions .pin/constraints-cuda-torch.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@
# This file is autogenerated by pip-compile with Python 3.11
# by the following command:
#
# pip-compile --config=pyproject.toml --output-file=.pin/constraints-cuda-torch.txt --resolver=backtracking .pin/tmp-constraints.txt benchmarks/accelerate_opt/requirements.in benchmarks/dlrm/requirements.in benchmarks/huggingface/requirements.in benchmarks/rwkv/requirements.in benchmarks/stargan/requirements.in benchmarks/super-slomo/requirements.in benchmarks/timm/requirements.in benchmarks/torchvision/requirements.in
# pip-compile --config=pyproject.toml --output-file=.pin/constraints-cuda-torch.txt --resolver=backtracking .pin/tmp-constraints.txt benchmarks/accelerate_opt/requirements.in benchmarks/dlrm/requirements.in benchmarks/flops/requirements.in benchmarks/huggingface/requirements.in benchmarks/rwkv/requirements.in benchmarks/stargan/requirements.in benchmarks/super-slomo/requirements.in benchmarks/timm/requirements.in benchmarks/torchvision/requirements.in
#
--extra-index-url https://download.pytorch.org/whl/cu118

absl-py==2.0.0
# via tensorboard
accelerate==0.24.0
accelerate==0.24.1
# via -r benchmarks/accelerate_opt/requirements.in
aiohttp==3.8.6
# via
Expand All @@ -28,7 +28,7 @@ cachetools==5.3.2
# via google-auth
certifi==2023.7.22
# via requests
charset-normalizer==3.3.1
charset-normalizer==3.3.2
# via
# aiohttp
# requests
Expand Down Expand Up @@ -57,7 +57,7 @@ executing==1.2.0
# via varname
fbgemm-gpu==0.5.0+cu118
# via torchrec
filelock==3.12.4
filelock==3.13.1
# via
# huggingface-hub
# torch
Expand All @@ -82,15 +82,15 @@ giving==0.4.2
# via
# ptera
# voir
google-auth==2.23.3
google-auth==2.23.4
# via
# google-auth-oauthlib
# tensorboard
google-auth-oauthlib==1.1.0
# via tensorboard
graphviz==0.20.1
# via torchviz
grpcio==1.59.0
grpcio==1.59.2
# via tensorboard
hjson==3.1.0
# via deepspeed
Expand All @@ -116,7 +116,7 @@ lightning-utilities==0.9.0
# via
# pytorch-lightning
# torchmetrics
markdown==3.5
markdown==3.5.1
# via tensorboard
markdown-it-py==3.0.0
# via rich
Expand All @@ -138,7 +138,7 @@ multiprocess==0.70.15
# evaluate
mypy-extensions==1.0.0
# via typing-inspect
networkx==3.2
networkx==3.2.1
# via torch
ninja==1.11.1.1
# via
Expand Down Expand Up @@ -167,7 +167,7 @@ oauthlib==3.2.2
# via requests-oauthlib
omegaconf==2.3.0
# via voir
onnx==1.14.1
onnx==1.15.0
# via -r benchmarks/dlrm/requirements.in
opencv-python==4.8.1.78
# via -r benchmarks/super-slomo/requirements.in
Expand All @@ -185,7 +185,7 @@ packaging==23.2
# pytorch-lightning
# torchmetrics
# transformers
pandas==2.1.1
pandas==2.1.2
# via
# datasets
# evaluate
Expand All @@ -203,7 +203,7 @@ ptera==1.4.1
# via voir
py-cpuinfo==9.0.0
# via deepspeed
pyarrow==13.0.0
pyarrow==14.0.0
# via datasets
pyasn1==0.5.0
# via
Expand Down Expand Up @@ -294,7 +294,7 @@ tokenizers==0.14.1
# via transformers
torch==2.1.0+cu118
# via
# -r benchmarks/huggingface/requirements.in
# -r benchmarks/stargan/requirements.in
# -r benchmarks/super-slomo/requirements.in
# accelerate
# deepspeed
Expand All @@ -313,8 +313,8 @@ torchrec==0.5.0+cu118
# via -r benchmarks/dlrm/requirements.in
torchvision==0.16.0+cu118
# via
# -r benchmarks/stargan/requirements.in
# -r benchmarks/super-slomo/requirements.in
# -r benchmarks/torchvision/requirements.in
torchviz==0.0.2
# via -r benchmarks/dlrm/requirements.in
torchx==0.6.0
Expand All @@ -340,7 +340,6 @@ typing-extensions==4.8.0
# via
# huggingface-hub
# lightning-utilities
# onnx
# pydantic
# pyre-extensions
# pytorch-lightning
Expand All @@ -359,9 +358,9 @@ urllib3==1.26.18
# torchx
varname==0.10.0
# via giving
voir @ git+https://github.com/breuleux/voir.git
voir==0.2.11
# via
# -r benchmarks/huggingface/requirements.in
# -r benchmarks/stargan/requirements.in
# -r benchmarks/super-slomo/requirements.in
websocket-client==1.6.4
# via docker
Expand Down
43 changes: 21 additions & 22 deletions .pin/constraints-rocm-torch.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@
# This file is autogenerated by pip-compile with Python 3.11
# by the following command:
#
# pip-compile --config=pyproject.toml --output-file=.pin/constraints-rocm-torch.txt --resolver=backtracking .pin/tmp-constraints.txt benchmarks/accelerate_opt/requirements.in benchmarks/dlrm/requirements.in benchmarks/huggingface/requirements.in benchmarks/rwkv/requirements.in benchmarks/stargan/requirements.in benchmarks/super-slomo/requirements.in benchmarks/timm/requirements.in benchmarks/torchvision/requirements.in
# pip-compile --config=pyproject.toml --output-file=.pin/constraints-rocm-torch.txt --resolver=backtracking .pin/tmp-constraints.txt benchmarks/accelerate_opt/requirements.in benchmarks/dlrm/requirements.in benchmarks/flops/requirements.in benchmarks/huggingface/requirements.in benchmarks/rwkv/requirements.in benchmarks/stargan/requirements.in benchmarks/super-slomo/requirements.in benchmarks/timm/requirements.in benchmarks/torchvision/requirements.in
#
--extra-index-url https://download.pytorch.org/whl/rocm5.6/

absl-py==2.0.0
# via tensorboard
accelerate==0.24.0
accelerate==0.24.1
# via -r benchmarks/accelerate_opt/requirements.in
aiohttp==3.8.6
# via
Expand All @@ -28,7 +28,7 @@ cachetools==5.3.2
# via google-auth
certifi==2023.7.22
# via requests
charset-normalizer==3.3.1
charset-normalizer==3.3.2
# via
# aiohttp
# requests
Expand Down Expand Up @@ -59,7 +59,7 @@ executing==1.2.0
# via varname
fbgemm-gpu==0.5.0
# via torchrec
filelock==3.12.4
filelock==3.13.1
# via
# huggingface-hub
# pytorch-triton-rocm
Expand All @@ -84,15 +84,15 @@ giving==0.4.2
# via
# ptera
# voir
google-auth==2.23.3
google-auth==2.23.4
# via
# google-auth-oauthlib
# tensorboard
google-auth-oauthlib==1.1.0
# via tensorboard
graphviz==0.20.1
# via torchviz
grpcio==1.59.0
grpcio==1.59.2
# via tensorboard
hjson==3.1.0
# via deepspeed
Expand All @@ -118,9 +118,9 @@ lightning-utilities==0.9.0
# via
# pytorch-lightning
# torchmetrics
lit==17.0.3
lit==17.0.4
# via pytorch-triton-rocm
markdown==3.5
markdown==3.5.1
# via tensorboard
markdown-it-py==3.0.0
# via rich
Expand All @@ -142,7 +142,7 @@ multiprocess==0.70.15
# evaluate
mypy-extensions==1.0.0
# via typing-inspect
networkx==3.2
networkx==3.2.1
# via torch
ninja==1.11.1.1
# via
Expand All @@ -151,7 +151,7 @@ ninja==1.11.1.1
numpy==1.26.1
# via
# -r benchmarks/dlrm/requirements.in
# -r benchmarks/stargan/requirements.in
# -r benchmarks/rwkv/requirements.in
# accelerate
# datasets
# deepspeed
Expand All @@ -172,7 +172,7 @@ oauthlib==3.2.2
# via requests-oauthlib
omegaconf==2.3.0
# via voir
onnx==1.14.1
onnx==1.15.0
# via -r benchmarks/dlrm/requirements.in
opencv-python==4.8.1.78
# via -r benchmarks/super-slomo/requirements.in
Expand All @@ -190,7 +190,7 @@ packaging==23.2
# pytorch-lightning
# torchmetrics
# transformers
pandas==2.1.1
pandas==2.1.2
# via
# datasets
# evaluate
Expand All @@ -208,7 +208,7 @@ ptera==1.4.1
# via voir
py-cpuinfo==9.0.0
# via deepspeed
pyarrow==13.0.0
pyarrow==14.0.0
# via datasets
pyasn1==0.5.0
# via
Expand Down Expand Up @@ -301,8 +301,8 @@ tokenizers==0.14.1
# via transformers
torch==2.1.0+rocm5.6
# via
# -r benchmarks/stargan/requirements.in
# -r benchmarks/timm/requirements.in
# -r benchmarks/accelerate_opt/requirements.in
# -r benchmarks/flops/requirements.in
# accelerate
# deepspeed
# pytorch-lightning
Expand All @@ -321,16 +321,16 @@ torchrec==0.5.0
# via -r benchmarks/dlrm/requirements.in
torchvision==0.16.0+rocm5.6
# via
# -r benchmarks/stargan/requirements.in
# -r benchmarks/timm/requirements.in
# -r benchmarks/accelerate_opt/requirements.in
# -r benchmarks/flops/requirements.in
torchviz==0.0.2
# via -r benchmarks/dlrm/requirements.in
torchx==0.6.0
# via -r benchmarks/dlrm/requirements.in
tqdm==4.66.1
# via
# -r benchmarks/dlrm/requirements.in
# -r benchmarks/torchvision/requirements.in
# -r benchmarks/flops/requirements.in
# datasets
# deepspeed
# evaluate
Expand All @@ -346,7 +346,6 @@ typing-extensions==4.8.0
# via
# huggingface-hub
# lightning-utilities
# onnx
# pydantic
# pyre-extensions
# pytorch-lightning
Expand All @@ -365,10 +364,10 @@ urllib3==1.26.18
# torchx
varname==0.10.0
# via giving
voir @ git+https://github.com/breuleux/voir.git
voir==0.2.11
# via
# -r benchmarks/stargan/requirements.in
# -r benchmarks/timm/requirements.in
# -r benchmarks/accelerate_opt/requirements.in
# -r benchmarks/flops/requirements.in
websocket-client==1.6.4
# via docker
werkzeug==3.0.1
Expand Down
14 changes: 7 additions & 7 deletions benchmarks/accelerate_opt/requirements.cuda.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
#
--extra-index-url https://download.pytorch.org/whl/cu118

accelerate==0.24.0
accelerate==0.24.1
# via -r benchmarks/accelerate_opt/requirements.in
aiohttp==3.8.6
# via
Expand Down Expand Up @@ -37,7 +37,7 @@ certifi==2023.7.22
# via
# -c .pin/../.pin/constraints-cuda-torch.txt
# requests
charset-normalizer==3.3.1
charset-normalizer==3.3.2
# via
# -c .pin/../.pin/constraints-cuda-torch.txt
# aiohttp
Expand All @@ -64,7 +64,7 @@ executing==1.2.0
# via
# -c .pin/../.pin/constraints-cuda-torch.txt
# varname
filelock==3.12.4
filelock==3.13.1
# via
# -c .pin/../.pin/constraints-cuda-torch.txt
# huggingface-hub
Expand Down Expand Up @@ -135,7 +135,7 @@ multiprocess==0.70.15
# -c .pin/../.pin/constraints-cuda-torch.txt
# datasets
# evaluate
networkx==3.2
networkx==3.2.1
# via
# -c .pin/../.pin/constraints-cuda-torch.txt
# torch
Expand Down Expand Up @@ -171,7 +171,7 @@ packaging==23.2
# evaluate
# huggingface-hub
# transformers
pandas==2.1.1
pandas==2.1.2
# via
# -c .pin/../.pin/constraints-cuda-torch.txt
# datasets
Expand All @@ -193,7 +193,7 @@ py-cpuinfo==9.0.0
# via
# -c .pin/../.pin/constraints-cuda-torch.txt
# deepspeed
pyarrow==13.0.0
pyarrow==14.0.0
# via
# -c .pin/../.pin/constraints-cuda-torch.txt
# datasets
Expand Down Expand Up @@ -313,7 +313,7 @@ varname==0.10.0
# via
# -c .pin/../.pin/constraints-cuda-torch.txt
# giving
voir @ git+https://github.com/breuleux/voir.git
voir==0.2.11
# via -r benchmarks/accelerate_opt/requirements.in
xxhash==3.4.1
# via
Expand Down
Loading

0 comments on commit 8e62d37

Please sign in to comment.