diff --git a/.pin/constraints-cuda-torch.txt b/.pin/constraints-cuda-torch.txt index f52d2aa9d..0f70803c6 100644 --- a/.pin/constraints-cuda-torch.txt +++ b/.pin/constraints-cuda-torch.txt @@ -1,16 +1,16 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --config=pyproject.toml --output-file=.pin/constraints-cuda-torch.txt --resolver=backtracking .pin/tmp-constraints.txt benchmarks/accelerate_opt/requirements.in benchmarks/dlrm/requirements.in benchmarks/flops/requirements.in benchmarks/huggingface/requirements.in benchmarks/rwkv/requirements.in benchmarks/stargan/requirements.in benchmarks/super-slomo/requirements.in benchmarks/timm/requirements.in benchmarks/torchvision/requirements.in +# pip-compile --output-file=.pin/constraints-cuda-torch.txt .pin/tmp-constraints.txt benchmarks/accelerate_opt/requirements.in benchmarks/dlrm/requirements.in benchmarks/flops/requirements.in benchmarks/huggingface/requirements.in benchmarks/llama/requirements.in benchmarks/rwkv/requirements.in benchmarks/stargan/requirements.in benchmarks/super-slomo/requirements.in benchmarks/timm/requirements.in benchmarks/torchvision/requirements.in # --extra-index-url https://download.pytorch.org/whl/cu118 -absl-py==2.0.0 +absl-py==2.1.0 # via tensorboard -accelerate==0.24.1 +accelerate==0.27.0 # via -r benchmarks/accelerate_opt/requirements.in -aiohttp==3.8.6 +aiohttp==3.9.3 # via # datasets # fsspec @@ -22,53 +22,57 @@ asttokens==2.4.1 # via giving async-timeout==4.0.3 # via aiohttp -attrs==23.1.0 +attrs==23.2.0 # via aiohttp cachetools==5.3.2 # via google-auth -certifi==2023.7.22 +certifi==2024.2.2 # via requests charset-normalizer==3.3.2 - # via - # aiohttp - # requests + # via requests codefind==0.1.3 # via ptera -datasets==2.14.6 +datasets==2.17.0 # via # -r benchmarks/accelerate_opt/requirements.in + # -r benchmarks/llama/requirements.in # evaluate -deepspeed==0.11.1 +deepspeed==0.13.1 # via # -r benchmarks/accelerate_opt/requirements.in # -r benchmarks/rwkv/requirements.in -dill==0.3.7 +dill==0.3.8 # via # datasets # evaluate # multiprocess -docker==6.1.3 +docker==7.0.0 # via torchx -docstring-parser==0.15 +docstring-parser==0.8.1 # via torchx evaluate==0.4.1 # via -r benchmarks/accelerate_opt/requirements.in executing==1.2.0 # via varname -fbgemm-gpu==0.5.0+cu118 +fairscale==0.4.13 + # via -r benchmarks/llama/requirements.in +fbgemm-gpu==0.6.0+cu118 # via torchrec filelock==3.13.1 # via + # datasets # huggingface-hub # torch # torchx # transformers # triton -frozenlist==1.4.0 +fire==0.5.0 + # via -r benchmarks/llama/requirements.in +frozenlist==1.4.1 # via # aiohttp # aiosignal -fsspec[http]==2023.1.0 +fsspec[http]==2023.10.0 # via # datasets # evaluate @@ -82,19 +86,19 @@ giving==0.4.2 # via # ptera # voir -google-auth==2.23.4 +google-auth==2.27.0 # via # google-auth-oauthlib # tensorboard -google-auth-oauthlib==1.1.0 +google-auth-oauthlib==1.2.0 # via tensorboard graphviz==0.20.1 # via torchviz -grpcio==1.59.2 +grpcio==1.60.1 # via tensorboard hjson==3.1.0 # via deepspeed -huggingface-hub==0.17.3 +huggingface-hub==0.20.3 # via # -r benchmarks/timm/requirements.in # accelerate @@ -102,25 +106,27 @@ huggingface-hub==0.17.3 # evaluate # tokenizers # transformers -idna==3.4 +idna==3.6 # via # requests # yarl -importlib-metadata==6.8.0 - # via torchx -jinja2==3.1.2 +importlib-metadata==7.0.1 + # via + # markdown + # torchx +jinja2==3.1.3 # via torch joblib==1.3.2 # via scikit-learn -lightning-utilities==0.9.0 +lightning-utilities==0.10.1 # via # pytorch-lightning # torchmetrics -markdown==3.5.1 +markdown==3.5.2 # via tensorboard markdown-it-py==3.0.0 # via rich -markupsafe==2.1.3 +markupsafe==2.1.5 # via # jinja2 # werkzeug @@ -128,11 +134,11 @@ mdurl==0.1.2 # via markdown-it-py mpmath==1.3.0 # via sympy -multidict==6.0.4 +multidict==6.0.5 # via # aiohttp # yarl -multiprocess==0.70.15 +multiprocess==0.70.16 # via # datasets # evaluate @@ -144,14 +150,18 @@ ninja==1.11.1.1 # via # -r benchmarks/rwkv/requirements.in # deepspeed -numpy==1.26.1 +numpy==1.26.4 # via + # -r benchmarks/dlrm/requirements.in + # -r benchmarks/rwkv/requirements.in # -r benchmarks/stargan/requirements.in # -r benchmarks/super-slomo/requirements.in # accelerate # datasets # deepspeed # evaluate + # fairscale + # fbgemm-gpu # onnx # opencv-python # pandas @@ -163,13 +173,38 @@ numpy==1.26.1 # torchmetrics # torchvision # transformers +nvidia-cublas-cu11==11.11.3.6 + # via + # nvidia-cudnn-cu11 + # nvidia-cusolver-cu11 + # torch +nvidia-cuda-cupti-cu11==11.8.87 + # via torch +nvidia-cuda-nvrtc-cu11==11.8.89 + # via torch +nvidia-cuda-runtime-cu11==11.8.89 + # via torch +nvidia-cudnn-cu11==8.7.0.84 + # via torch +nvidia-cufft-cu11==10.9.0.58 + # via torch +nvidia-curand-cu11==10.3.0.86 + # via torch +nvidia-cusolver-cu11==11.4.1.48 + # via torch +nvidia-cusparse-cu11==11.7.5.86 + # via torch +nvidia-nccl-cu11==2.19.3 + # via torch +nvidia-nvtx-cu11==11.8.86 + # via torch oauthlib==3.2.2 # via requests-oauthlib omegaconf==2.3.0 # via voir onnx==1.15.0 # via -r benchmarks/dlrm/requirements.in -opencv-python==4.8.1.78 +opencv-python==4.9.0.80 # via -r benchmarks/super-slomo/requirements.in ovld==0.3.2 # via voir @@ -185,17 +220,17 @@ packaging==23.2 # pytorch-lightning # torchmetrics # transformers -pandas==2.1.2 +pandas==2.2.0 # via # datasets # evaluate -pillow==10.1.0 +pillow==10.2.0 # via torchvision -protobuf==4.23.4 +protobuf==4.25.2 # via # onnx # tensorboard -psutil==5.9.6 +psutil==5.9.8 # via # accelerate # deepspeed @@ -203,24 +238,28 @@ ptera==1.4.1 # via voir py-cpuinfo==9.0.0 # via deepspeed -pyarrow==14.0.0 +pyarrow==15.0.0 + # via datasets +pyarrow-hotfix==0.6 # via datasets -pyasn1==0.5.0 +pyasn1==0.5.1 # via # pyasn1-modules # rsa pyasn1-modules==0.3.0 # via google-auth -pydantic==1.10.13 +pydantic==1.10.14 # via # -r benchmarks/rwkv/requirements.in # deepspeed -pydot==1.4.2 +pydot==2.0.0 # via -r benchmarks/dlrm/requirements.in -pygments==2.16.1 +pygments==2.17.2 # via rich pynvml==11.5.0 - # via voir + # via + # deepspeed + # voir pyparsing==3.1.1 # via pydot pyre-extensions==0.0.30 @@ -229,7 +268,7 @@ python-dateutil==2.8.2 # via pandas pytorch-lightning==1.9.5 # via -r benchmarks/rwkv/requirements.in -pytz==2023.3.post1 +pytz==2024.1 # via pandas pyyaml==6.0.1 # via @@ -243,7 +282,7 @@ pyyaml==6.0.1 # transformers reactivex==4.0.4 # via giving -regex==2023.10.3 +regex==2023.12.25 # via transformers requests==2.31.0 # via @@ -261,66 +300,87 @@ requests-oauthlib==1.3.1 # via google-auth-oauthlib responses==0.18.0 # via evaluate -rich==13.6.0 +rich==13.7.0 # via # -r benchmarks/accelerate_opt/requirements.in # voir rsa==4.9 # via google-auth -safetensors==0.4.0 +safetensors==0.4.2 # via # -r benchmarks/timm/requirements.in + # accelerate # transformers -scikit-learn==1.3.2 +scikit-learn==1.4.0 # via -r benchmarks/dlrm/requirements.in -scipy==1.11.3 +scipy==1.12.0 # via scikit-learn +sentencepiece==0.1.99 + # via -r benchmarks/llama/requirements.in six==1.16.0 # via # asttokens + # fire # python-dateutil # tensorboard sympy==1.12 # via torch tabulate==0.9.0 # via torchx -tensorboard==2.15.0 +tensorboard==2.15.2 # via -r benchmarks/dlrm/requirements.in tensorboard-data-server==0.7.2 # via tensorboard +termcolor==2.4.0 + # via fire threadpoolctl==3.2.0 # via scikit-learn -tokenizers==0.14.1 +tokenizers==0.15.1 # via transformers -torch==2.1.0+cu118 +torch==2.2.0+cu118 # via + # -r benchmarks/accelerate_opt/requirements.in + # -r benchmarks/dlrm/requirements.in + # -r benchmarks/flops/requirements.in + # -r benchmarks/huggingface/requirements.in + # -r benchmarks/llama/requirements.in + # -r benchmarks/rwkv/requirements.in # -r benchmarks/stargan/requirements.in # -r benchmarks/super-slomo/requirements.in + # -r benchmarks/timm/requirements.in + # -r benchmarks/torchvision/requirements.in # accelerate # deepspeed + # fairscale # pytorch-lightning # torchaudio # torchmetrics # torchvision # torchviz -torchaudio==2.1.0+cu118 +torchaudio==2.2.0+cu118 # via -r benchmarks/accelerate_opt/requirements.in torchmetrics==1.0.3 # via # pytorch-lightning # torchrec -torchrec==0.5.0+cu118 +torchrec==0.6.0+cu118 # via -r benchmarks/dlrm/requirements.in -torchvision==0.16.0+cu118 +torchvision==0.17.0+cu118 # via + # -r benchmarks/accelerate_opt/requirements.in + # -r benchmarks/flops/requirements.in # -r benchmarks/stargan/requirements.in # -r benchmarks/super-slomo/requirements.in + # -r benchmarks/timm/requirements.in + # -r benchmarks/torchvision/requirements.in torchviz==0.0.2 # via -r benchmarks/dlrm/requirements.in -torchx==0.6.0 +torchx==0.5.0 # via -r benchmarks/dlrm/requirements.in tqdm==4.66.1 # via + # -r benchmarks/dlrm/requirements.in + # -r benchmarks/flops/requirements.in # -r benchmarks/super-slomo/requirements.in # -r benchmarks/torchvision/requirements.in # datasets @@ -330,13 +390,14 @@ tqdm==4.66.1 # pytorch-lightning # torchrec # transformers -transformers==4.34.1 +transformers==4.37.2 # via # -r benchmarks/accelerate_opt/requirements.in # -r benchmarks/huggingface/requirements.in -triton==2.1.0 + # -r benchmarks/llama/requirements.in +triton==2.2.0 # via torch -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via # huggingface-hub # lightning-utilities @@ -348,7 +409,7 @@ typing-extensions==4.8.0 # typing-inspect typing-inspect==0.9.0 # via pyre-extensions -tzdata==2023.3 +tzdata==2023.4 # via pandas urllib3==1.26.18 # via @@ -358,19 +419,26 @@ urllib3==1.26.18 # torchx varname==0.10.0 # via giving -voir==0.2.11 +voir==0.2.12 # via + # -c .pin/../constraints/cuda.txt + # -r benchmarks/accelerate_opt/requirements.in + # -r benchmarks/dlrm/requirements.in + # -r benchmarks/flops/requirements.in + # -r benchmarks/huggingface/requirements.in + # -r benchmarks/llama/requirements.in + # -r benchmarks/rwkv/requirements.in # -r benchmarks/stargan/requirements.in # -r benchmarks/super-slomo/requirements.in -websocket-client==1.6.4 - # via docker + # -r benchmarks/timm/requirements.in + # -r benchmarks/torchvision/requirements.in werkzeug==3.0.1 # via tensorboard xxhash==3.4.1 # via # datasets # evaluate -yarl==1.9.2 +yarl==1.9.4 # via aiohttp zipp==3.17.0 # via importlib-metadata diff --git a/.pin/constraints-rocm-torch.txt b/.pin/constraints-rocm-torch.txt index a220c5c8d..00fdd63a4 100644 --- a/.pin/constraints-rocm-torch.txt +++ b/.pin/constraints-rocm-torch.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --config=pyproject.toml --output-file=.pin/constraints-rocm-torch.txt --resolver=backtracking .pin/tmp-constraints.txt benchmarks/accelerate_opt/requirements.in benchmarks/dlrm/requirements.in benchmarks/flops/requirements.in benchmarks/huggingface/requirements.in benchmarks/rwkv/requirements.in benchmarks/stargan/requirements.in benchmarks/super-slomo/requirements.in benchmarks/timm/requirements.in benchmarks/torchvision/requirements.in +# pip-compile --output-file=.pin/constraints-rocm-torch.txt .pin/tmp-constraints.txt benchmarks/accelerate_opt/requirements.in benchmarks/dlrm/requirements.in benchmarks/flops/requirements.in benchmarks/huggingface/requirements.in benchmarks/llama/requirements.in benchmarks/rwkv/requirements.in benchmarks/stargan/requirements.in benchmarks/super-slomo/requirements.in benchmarks/timm/requirements.in benchmarks/torchvision/requirements.in # --extra-index-url https://download.pytorch.org/whl/rocm5.6/ @@ -39,6 +39,7 @@ codefind==0.1.3 datasets==2.14.6 # via # -r benchmarks/accelerate_opt/requirements.in + # -r benchmarks/llama/requirements.in # evaluate deepspeed==0.12.2 # via @@ -57,6 +58,8 @@ evaluate==0.4.1 # via -r benchmarks/accelerate_opt/requirements.in executing==1.2.0 # via varname +fairscale==0.4.13 + # via -r benchmarks/llama/requirements.in fbgemm-gpu==0.5.0 # via torchrec filelock==3.13.1 @@ -66,6 +69,8 @@ filelock==3.13.1 # torch # torchx # transformers +fire==0.5.0 + # via -r benchmarks/llama/requirements.in frozenlist==1.4.0 # via # aiohttp @@ -109,7 +114,9 @@ idna==3.4 # requests # yarl importlib-metadata==6.8.0 - # via torchx + # via + # markdown + # torchx jinja2==3.1.2 # via torch joblib==1.3.2 @@ -152,10 +159,13 @@ numpy==1.26.1 # via # -r benchmarks/dlrm/requirements.in # -r benchmarks/rwkv/requirements.in + # -r benchmarks/stargan/requirements.in + # -r benchmarks/super-slomo/requirements.in # accelerate # datasets # deepspeed # evaluate + # fairscale # fbgemm-gpu # onnx # opencv-python @@ -284,9 +294,12 @@ scikit-learn==1.3.2 # via -r benchmarks/dlrm/requirements.in scipy==1.11.3 # via scikit-learn +sentencepiece==0.1.99 + # via -r benchmarks/llama/requirements.in six==1.16.0 # via # asttokens + # fire # python-dateutil # tensorboard sympy==1.12 @@ -297,6 +310,8 @@ tensorboard==2.15.1 # via -r benchmarks/dlrm/requirements.in tensorboard-data-server==0.7.2 # via tensorboard +termcolor==2.4.0 + # via fire threadpoolctl==3.2.0 # via scikit-learn tokenizers==0.14.1 @@ -304,9 +319,18 @@ tokenizers==0.14.1 torch==2.1.0+rocm5.6 # via # -r benchmarks/accelerate_opt/requirements.in + # -r benchmarks/dlrm/requirements.in + # -r benchmarks/flops/requirements.in + # -r benchmarks/huggingface/requirements.in + # -r benchmarks/llama/requirements.in # -r benchmarks/rwkv/requirements.in + # -r benchmarks/stargan/requirements.in + # -r benchmarks/super-slomo/requirements.in + # -r benchmarks/timm/requirements.in + # -r benchmarks/torchvision/requirements.in # accelerate # deepspeed + # fairscale # pytorch-lightning # pytorch-triton-rocm # torchaudio @@ -324,14 +348,20 @@ torchrec==0.5.0 torchvision==0.16.0+rocm5.6 # via # -r benchmarks/accelerate_opt/requirements.in + # -r benchmarks/flops/requirements.in # -r benchmarks/stargan/requirements.in + # -r benchmarks/super-slomo/requirements.in + # -r benchmarks/timm/requirements.in + # -r benchmarks/torchvision/requirements.in torchviz==0.0.2 # via -r benchmarks/dlrm/requirements.in torchx==0.5.0 # via -r benchmarks/dlrm/requirements.in tqdm==4.66.1 # via + # -r benchmarks/dlrm/requirements.in # -r benchmarks/flops/requirements.in + # -r benchmarks/super-slomo/requirements.in # -r benchmarks/torchvision/requirements.in # datasets # deepspeed @@ -344,6 +374,7 @@ transformers==4.35.0 # via # -r benchmarks/accelerate_opt/requirements.in # -r benchmarks/huggingface/requirements.in + # -r benchmarks/llama/requirements.in typing-extensions==4.8.0 # via # huggingface-hub @@ -368,8 +399,17 @@ varname==0.10.0 # via giving voir==0.2.11 # via + # -c .pin/../constraints/rocm.txt # -r benchmarks/accelerate_opt/requirements.in + # -r benchmarks/dlrm/requirements.in + # -r benchmarks/flops/requirements.in + # -r benchmarks/huggingface/requirements.in + # -r benchmarks/llama/requirements.in # -r benchmarks/rwkv/requirements.in + # -r benchmarks/stargan/requirements.in + # -r benchmarks/super-slomo/requirements.in + # -r benchmarks/timm/requirements.in + # -r benchmarks/torchvision/requirements.in websocket-client==1.6.4 # via docker werkzeug==3.0.1 diff --git a/benchmarks/accelerate_opt/requirements.cuda.txt b/benchmarks/accelerate_opt/requirements.cuda.txt new file mode 100644 index 000000000..f979c6993 --- /dev/null +++ b/benchmarks/accelerate_opt/requirements.cuda.txt @@ -0,0 +1,396 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --output-file=benchmarks/accelerate_opt/requirements.cuda.txt .pin/tmp-constraints-cuda-opt.txt benchmarks/accelerate_opt/requirements.in +# +--extra-index-url https://download.pytorch.org/whl/cu118 + +accelerate==0.27.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/accelerate_opt/requirements.in +aiohttp==3.9.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets + # fsspec +aiosignal==1.3.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # aiohttp +antlr4-python3-runtime==4.9.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # omegaconf +asttokens==2.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +async-timeout==4.0.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # aiohttp +attrs==23.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # aiohttp +certifi==2024.2.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +charset-normalizer==3.3.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +codefind==0.1.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # ptera +datasets==2.17.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/accelerate_opt/requirements.in + # evaluate +deepspeed==0.13.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/accelerate_opt/requirements.in +dill==0.3.8 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets + # evaluate + # multiprocess +evaluate==0.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/accelerate_opt/requirements.in +executing==1.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # varname +filelock==3.13.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets + # huggingface-hub + # torch + # transformers + # triton +frozenlist==1.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # aiohttp + # aiosignal +fsspec[http]==2023.10.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets + # evaluate + # huggingface-hub + # torch +giving==0.4.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # ptera + # voir +hjson==3.1.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # deepspeed +huggingface-hub==0.20.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # accelerate + # datasets + # evaluate + # tokenizers + # transformers +idna==3.6 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests + # yarl +jinja2==3.1.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +markdown-it-py==3.0.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # rich +markupsafe==2.1.5 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # jinja2 +mdurl==0.1.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # markdown-it-py +mpmath==1.3.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # sympy +multidict==6.0.5 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # aiohttp + # yarl +multiprocess==0.70.16 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets + # evaluate +networkx==3.2.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +ninja==1.11.1.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # deepspeed +numpy==1.26.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # accelerate + # datasets + # deepspeed + # evaluate + # pandas + # pyarrow + # torchvision + # transformers +nvidia-cublas-cu11==11.11.3.6 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # nvidia-cudnn-cu11 + # nvidia-cusolver-cu11 + # torch +nvidia-cuda-cupti-cu11==11.8.87 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cuda-nvrtc-cu11==11.8.89 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cuda-runtime-cu11==11.8.89 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cudnn-cu11==8.7.0.84 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cufft-cu11==10.9.0.58 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-curand-cu11==10.3.0.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cusolver-cu11==11.4.1.48 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cusparse-cu11==11.7.5.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-nccl-cu11==2.19.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-nvtx-cu11==11.8.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +omegaconf==2.3.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +ovld==0.3.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +packaging==23.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # accelerate + # datasets + # deepspeed + # evaluate + # huggingface-hub + # transformers +pandas==2.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets + # evaluate +pillow==10.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchvision +psutil==5.9.8 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # accelerate + # deepspeed +ptera==1.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +py-cpuinfo==9.0.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # deepspeed +pyarrow==15.0.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets +pyarrow-hotfix==0.6 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets +pydantic==1.10.14 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # deepspeed +pygments==2.17.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # rich +pynvml==11.5.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # deepspeed + # voir +python-dateutil==2.8.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # pandas +pytz==2024.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # pandas +pyyaml==6.0.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # accelerate + # datasets + # huggingface-hub + # omegaconf + # transformers +reactivex==4.0.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +regex==2023.12.25 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # transformers +requests==2.31.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets + # evaluate + # fsspec + # huggingface-hub + # responses + # torchvision + # transformers +responses==0.18.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # evaluate +rich==13.7.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/accelerate_opt/requirements.in + # voir +safetensors==0.4.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # accelerate + # transformers +six==1.16.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # asttokens + # python-dateutil +sympy==1.12 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +tokenizers==0.15.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # transformers +torch==2.2.0+cu118 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/accelerate_opt/requirements.in + # accelerate + # deepspeed + # torchaudio + # torchvision +torchaudio==2.2.0+cu118 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/accelerate_opt/requirements.in +torchvision==0.17.0+cu118 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/accelerate_opt/requirements.in +tqdm==4.66.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets + # deepspeed + # evaluate + # huggingface-hub + # transformers +transformers==4.37.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/accelerate_opt/requirements.in +triton==2.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +typing-extensions==4.9.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # huggingface-hub + # pydantic + # reactivex + # torch +tzdata==2023.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # pandas +urllib3==1.26.18 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests + # responses +varname==0.10.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +voir==0.2.12 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -c .pin/../constraints/cuda.txt + # -r benchmarks/accelerate_opt/requirements.in +xxhash==3.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets + # evaluate +yarl==1.9.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # aiohttp diff --git a/benchmarks/accelerate_opt/requirements.rocm.txt b/benchmarks/accelerate_opt/requirements.rocm.txt index b56ff798f..b1de81c3f 100644 --- a/benchmarks/accelerate_opt/requirements.rocm.txt +++ b/benchmarks/accelerate_opt/requirements.rocm.txt @@ -1,13 +1,15 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --config=pyproject.toml --output-file=benchmarks/accelerate_opt/requirements.rocm.txt --resolver=backtracking .pin/tmp-constraints-rocm-opt.txt benchmarks/accelerate_opt/requirements.in +# pip-compile --output-file=benchmarks/accelerate_opt/requirements.rocm.txt .pin/tmp-constraints-rocm-opt.txt benchmarks/accelerate_opt/requirements.in # --extra-index-url https://download.pytorch.org/whl/rocm5.6/ accelerate==0.24.1 - # via -r benchmarks/accelerate_opt/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/accelerate_opt/requirements.in aiohttp==3.8.6 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -52,10 +54,13 @@ codefind==0.1.3 # ptera datasets==2.14.6 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # -r benchmarks/accelerate_opt/requirements.in # evaluate deepspeed==0.12.2 - # via -r benchmarks/accelerate_opt/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/accelerate_opt/requirements.in dill==0.3.7 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -63,7 +68,9 @@ dill==0.3.7 # evaluate # multiprocess evaluate==0.4.1 - # via -r benchmarks/accelerate_opt/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/accelerate_opt/requirements.in executing==1.2.0 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -262,6 +269,7 @@ responses==0.18.0 # evaluate rich==13.6.0 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # -r benchmarks/accelerate_opt/requirements.in # voir safetensors==0.4.0 @@ -283,6 +291,7 @@ tokenizers==0.14.1 # transformers torch==2.1.0+rocm5.6 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # -r benchmarks/accelerate_opt/requirements.in # accelerate # deepspeed @@ -290,9 +299,13 @@ torch==2.1.0+rocm5.6 # torchaudio # torchvision torchaudio==2.1.0+rocm5.6 - # via -r benchmarks/accelerate_opt/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/accelerate_opt/requirements.in torchvision==0.16.0+rocm5.6 - # via -r benchmarks/accelerate_opt/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/accelerate_opt/requirements.in tqdm==4.66.1 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -302,7 +315,9 @@ tqdm==4.66.1 # huggingface-hub # transformers transformers==4.35.0 - # via -r benchmarks/accelerate_opt/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/accelerate_opt/requirements.in typing-extensions==4.8.0 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -324,7 +339,10 @@ varname==0.10.0 # -c .pin/../.pin/constraints-rocm-torch.txt # giving voir==0.2.11 - # via -r benchmarks/accelerate_opt/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -c .pin/../constraints/rocm.txt + # -r benchmarks/accelerate_opt/requirements.in xxhash==3.4.1 # via # -c .pin/../.pin/constraints-rocm-torch.txt diff --git a/benchmarks/dlrm/requirements.cuda.txt b/benchmarks/dlrm/requirements.cuda.txt new file mode 100644 index 000000000..f2c8e26c5 --- /dev/null +++ b/benchmarks/dlrm/requirements.cuda.txt @@ -0,0 +1,384 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --output-file=benchmarks/dlrm/requirements.cuda.txt .pin/tmp-constraints-cuda-dlrm.txt benchmarks/dlrm/requirements.in +# +--extra-index-url https://download.pytorch.org/whl/cu118 + +absl-py==2.1.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # tensorboard +antlr4-python3-runtime==4.9.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # omegaconf +asttokens==2.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +cachetools==5.3.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # google-auth +certifi==2024.2.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +charset-normalizer==3.3.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +codefind==0.1.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # ptera +docker==7.0.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchx +docstring-parser==0.8.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchx +executing==1.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # varname +fbgemm-gpu==0.6.0+cu118 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchrec +filelock==3.13.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch + # torchx + # triton +fsspec==2023.10.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch + # torchx +future==0.18.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/dlrm/requirements.in +giving==0.4.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # ptera + # voir +google-auth==2.27.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # google-auth-oauthlib + # tensorboard +google-auth-oauthlib==1.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # tensorboard +graphviz==0.20.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchviz +grpcio==1.60.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # tensorboard +idna==3.6 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +importlib-metadata==7.0.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # markdown + # torchx +jinja2==3.1.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +joblib==1.3.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # scikit-learn +lightning-utilities==0.10.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchmetrics +markdown==3.5.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # tensorboard +markdown-it-py==3.0.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # rich +markupsafe==2.1.5 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # jinja2 + # werkzeug +mdurl==0.1.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # markdown-it-py +mpmath==1.3.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # sympy +mypy-extensions==1.0.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # typing-inspect +networkx==3.2.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +numpy==1.26.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/dlrm/requirements.in + # fbgemm-gpu + # onnx + # scikit-learn + # scipy + # tensorboard + # torchmetrics +nvidia-cublas-cu11==11.11.3.6 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # nvidia-cudnn-cu11 + # nvidia-cusolver-cu11 + # torch +nvidia-cuda-cupti-cu11==11.8.87 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cuda-nvrtc-cu11==11.8.89 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cuda-runtime-cu11==11.8.89 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cudnn-cu11==8.7.0.84 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cufft-cu11==10.9.0.58 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-curand-cu11==10.3.0.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cusolver-cu11==11.4.1.48 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cusparse-cu11==11.7.5.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-nccl-cu11==2.19.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-nvtx-cu11==11.8.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +oauthlib==3.2.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests-oauthlib +omegaconf==2.3.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +onnx==1.15.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/dlrm/requirements.in +ovld==0.3.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +packaging==23.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # docker + # lightning-utilities + # torchmetrics +protobuf==4.25.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # onnx + # tensorboard +ptera==1.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +pyasn1==0.5.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # google-auth +pydot==2.0.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/dlrm/requirements.in +pygments==2.17.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # rich +pynvml==11.5.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +pyparsing==3.1.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # pydot +pyre-extensions==0.0.30 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchx +pyyaml==6.0.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # omegaconf + # torchx +reactivex==4.0.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +requests==2.31.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # docker + # requests-oauthlib + # tensorboard +requests-oauthlib==1.3.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # google-auth-oauthlib +rich==13.7.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +rsa==4.9 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # google-auth +scikit-learn==1.4.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/dlrm/requirements.in +scipy==1.12.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # scikit-learn +six==1.16.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # asttokens + # tensorboard +sympy==1.12 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +tabulate==0.9.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchx +tensorboard==2.15.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/dlrm/requirements.in +tensorboard-data-server==0.7.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # tensorboard +threadpoolctl==3.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # scikit-learn +torch==2.2.0+cu118 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/dlrm/requirements.in + # torchmetrics + # torchviz +torchmetrics==1.0.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchrec +torchrec==0.6.0+cu118 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/dlrm/requirements.in +torchviz==0.0.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/dlrm/requirements.in +torchx==0.5.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/dlrm/requirements.in +tqdm==4.66.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/dlrm/requirements.in + # torchrec +triton==2.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +typing-extensions==4.9.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # lightning-utilities + # pyre-extensions + # reactivex + # torch + # typing-inspect +typing-inspect==0.9.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # pyre-extensions +urllib3==1.26.18 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # docker + # requests + # torchx +varname==0.10.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +voir==0.2.12 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -c .pin/../constraints/cuda.txt + # -r benchmarks/dlrm/requirements.in +werkzeug==3.0.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # tensorboard +zipp==3.17.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/benchmarks/dlrm/requirements.rocm.txt b/benchmarks/dlrm/requirements.rocm.txt index a36f32986..8f7aea241 100644 --- a/benchmarks/dlrm/requirements.rocm.txt +++ b/benchmarks/dlrm/requirements.rocm.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --config=pyproject.toml --output-file=benchmarks/dlrm/requirements.rocm.txt --resolver=backtracking .pin/tmp-constraints-rocm-dlrm.txt benchmarks/dlrm/requirements.in +# pip-compile --output-file=benchmarks/dlrm/requirements.rocm.txt .pin/tmp-constraints-rocm-dlrm.txt benchmarks/dlrm/requirements.in # --extra-index-url https://download.pytorch.org/whl/rocm5.6/ @@ -66,7 +66,9 @@ fsspec==2023.10.0 # torch # torchx future==0.18.3 - # via -r benchmarks/dlrm/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/dlrm/requirements.in giving==0.4.2 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -96,6 +98,7 @@ idna==3.4 importlib-metadata==6.8.0 # via # -c .pin/../.pin/constraints-rocm-torch.txt + # markdown # torchx jinja2==3.1.2 # via @@ -144,6 +147,7 @@ networkx==3.2.1 # torch numpy==1.26.1 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # -r benchmarks/dlrm/requirements.in # fbgemm-gpu # onnx @@ -160,7 +164,9 @@ omegaconf==2.3.0 # -c .pin/../.pin/constraints-rocm-torch.txt # voir onnx==1.15.0 - # via -r benchmarks/dlrm/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/dlrm/requirements.in ovld==0.3.2 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -190,7 +196,9 @@ pyasn1-modules==0.3.0 # -c .pin/../.pin/constraints-rocm-torch.txt # google-auth pydot==1.4.2 - # via -r benchmarks/dlrm/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/dlrm/requirements.in pygments==2.16.1 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -239,7 +247,9 @@ rsa==4.9 # -c .pin/../.pin/constraints-rocm-torch.txt # google-auth scikit-learn==1.3.2 - # via -r benchmarks/dlrm/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/dlrm/requirements.in scipy==1.11.3 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -258,7 +268,9 @@ tabulate==0.9.0 # -c .pin/../.pin/constraints-rocm-torch.txt # torchx tensorboard==2.15.1 - # via -r benchmarks/dlrm/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/dlrm/requirements.in tensorboard-data-server==0.7.2 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -269,6 +281,7 @@ threadpoolctl==3.2.0 # scikit-learn torch==2.1.0+rocm5.6 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # -r benchmarks/dlrm/requirements.in # pytorch-triton-rocm # torchmetrics @@ -278,13 +291,20 @@ torchmetrics==1.0.3 # -c .pin/../.pin/constraints-rocm-torch.txt # torchrec torchrec==0.5.0 - # via -r benchmarks/dlrm/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/dlrm/requirements.in torchviz==0.0.2 - # via -r benchmarks/dlrm/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/dlrm/requirements.in torchx==0.5.0 - # via -r benchmarks/dlrm/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/dlrm/requirements.in tqdm==4.66.1 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # -r benchmarks/dlrm/requirements.in # torchrec typing-extensions==4.8.0 @@ -310,7 +330,10 @@ varname==0.10.0 # -c .pin/../.pin/constraints-rocm-torch.txt # giving voir==0.2.11 - # via -r benchmarks/dlrm/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -c .pin/../constraints/rocm.txt + # -r benchmarks/dlrm/requirements.in websocket-client==1.6.4 # via # -c .pin/../.pin/constraints-rocm-torch.txt diff --git a/benchmarks/flops/requirements.cuda.txt b/benchmarks/flops/requirements.cuda.txt new file mode 100644 index 000000000..e3be8c36c --- /dev/null +++ b/benchmarks/flops/requirements.cuda.txt @@ -0,0 +1,207 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --output-file=benchmarks/flops/requirements.cuda.txt .pin/tmp-constraints-cuda-flops.txt benchmarks/flops/requirements.in +# +--extra-index-url https://download.pytorch.org/whl/cu118 + +antlr4-python3-runtime==4.9.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # omegaconf +asttokens==2.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +certifi==2024.2.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +charset-normalizer==3.3.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +codefind==0.1.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # ptera +executing==1.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # varname +filelock==3.13.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch + # triton +fsspec==2023.10.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +giving==0.4.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # ptera + # voir +idna==3.6 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +jinja2==3.1.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +markdown-it-py==3.0.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # rich +markupsafe==2.1.5 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # jinja2 +mdurl==0.1.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # markdown-it-py +mpmath==1.3.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # sympy +networkx==3.2.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +numpy==1.26.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchvision +nvidia-cublas-cu11==11.11.3.6 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # nvidia-cudnn-cu11 + # nvidia-cusolver-cu11 + # torch +nvidia-cuda-cupti-cu11==11.8.87 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cuda-nvrtc-cu11==11.8.89 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cuda-runtime-cu11==11.8.89 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cudnn-cu11==8.7.0.84 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cufft-cu11==10.9.0.58 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-curand-cu11==10.3.0.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cusolver-cu11==11.4.1.48 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cusparse-cu11==11.7.5.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-nccl-cu11==2.19.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-nvtx-cu11==11.8.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +omegaconf==2.3.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +ovld==0.3.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +pillow==10.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchvision +ptera==1.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +pygments==2.17.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # rich +pynvml==11.5.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +pyyaml==6.0.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # omegaconf +reactivex==4.0.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +requests==2.31.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchvision +rich==13.7.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +six==1.16.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # asttokens +sympy==1.12 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +torch==2.2.0+cu118 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/flops/requirements.in + # torchvision +torchvision==0.17.0+cu118 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/flops/requirements.in +tqdm==4.66.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/flops/requirements.in +triton==2.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +typing-extensions==4.9.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # reactivex + # torch +urllib3==1.26.18 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +varname==0.10.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +voir==0.2.12 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -c .pin/../constraints/cuda.txt + # -r benchmarks/flops/requirements.in diff --git a/benchmarks/flops/requirements.rocm.txt b/benchmarks/flops/requirements.rocm.txt index 23d10b701..b7cfc21c9 100644 --- a/benchmarks/flops/requirements.rocm.txt +++ b/benchmarks/flops/requirements.rocm.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --config=pyproject.toml --output-file=benchmarks/flops/requirements.rocm.txt --resolver=backtracking .pin/tmp-constraints-rocm-flops.txt benchmarks/flops/requirements.in +# pip-compile --output-file=benchmarks/flops/requirements.rocm.txt .pin/tmp-constraints-rocm-flops.txt benchmarks/flops/requirements.in # --extra-index-url https://download.pytorch.org/whl/rocm5.6/ @@ -138,13 +138,18 @@ sympy==1.12 # torch torch==2.1.0+rocm5.6 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # -r benchmarks/flops/requirements.in # pytorch-triton-rocm # torchvision torchvision==0.16.0+rocm5.6 - # via -r benchmarks/flops/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/flops/requirements.in tqdm==4.66.1 - # via -r benchmarks/flops/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/flops/requirements.in typing-extensions==4.8.0 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -159,4 +164,7 @@ varname==0.10.0 # -c .pin/../.pin/constraints-rocm-torch.txt # giving voir==0.2.11 - # via -r benchmarks/flops/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -c .pin/../constraints/rocm.txt + # -r benchmarks/flops/requirements.in diff --git a/benchmarks/huggingface/requirements.cuda.txt b/benchmarks/huggingface/requirements.cuda.txt new file mode 100644 index 000000000..a223cfe9a --- /dev/null +++ b/benchmarks/huggingface/requirements.cuda.txt @@ -0,0 +1,232 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --output-file=benchmarks/huggingface/requirements.cuda.txt .pin/tmp-constraints-cuda-hf.txt benchmarks/huggingface/requirements.in +# +--extra-index-url https://download.pytorch.org/whl/cu118 + +antlr4-python3-runtime==4.9.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # omegaconf +asttokens==2.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +certifi==2024.2.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +charset-normalizer==3.3.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +codefind==0.1.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # ptera +executing==1.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # varname +filelock==3.13.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # huggingface-hub + # torch + # transformers + # triton +fsspec==2023.10.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # huggingface-hub + # torch +giving==0.4.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # ptera + # voir +huggingface-hub==0.20.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # tokenizers + # transformers +idna==3.6 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +jinja2==3.1.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +markdown-it-py==3.0.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # rich +markupsafe==2.1.5 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # jinja2 +mdurl==0.1.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # markdown-it-py +mpmath==1.3.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # sympy +networkx==3.2.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +numpy==1.26.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # transformers +nvidia-cublas-cu11==11.11.3.6 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # nvidia-cudnn-cu11 + # nvidia-cusolver-cu11 + # torch +nvidia-cuda-cupti-cu11==11.8.87 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cuda-nvrtc-cu11==11.8.89 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cuda-runtime-cu11==11.8.89 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cudnn-cu11==8.7.0.84 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cufft-cu11==10.9.0.58 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-curand-cu11==10.3.0.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cusolver-cu11==11.4.1.48 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cusparse-cu11==11.7.5.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-nccl-cu11==2.19.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-nvtx-cu11==11.8.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +omegaconf==2.3.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +ovld==0.3.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +packaging==23.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # huggingface-hub + # transformers +ptera==1.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +pygments==2.17.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # rich +pynvml==11.5.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +pyyaml==6.0.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # huggingface-hub + # omegaconf + # transformers +reactivex==4.0.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +regex==2023.12.25 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # transformers +requests==2.31.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # huggingface-hub + # transformers +rich==13.7.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +safetensors==0.4.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # transformers +six==1.16.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # asttokens +sympy==1.12 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +tokenizers==0.15.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # transformers +torch==2.2.0+cu118 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/huggingface/requirements.in +tqdm==4.66.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # huggingface-hub + # transformers +transformers==4.37.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/huggingface/requirements.in +triton==2.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +typing-extensions==4.9.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # huggingface-hub + # reactivex + # torch +urllib3==1.26.18 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +varname==0.10.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +voir==0.2.12 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -c .pin/../constraints/cuda.txt + # -r benchmarks/huggingface/requirements.in diff --git a/benchmarks/huggingface/requirements.rocm.txt b/benchmarks/huggingface/requirements.rocm.txt index 4e39b0c45..df77376ea 100644 --- a/benchmarks/huggingface/requirements.rocm.txt +++ b/benchmarks/huggingface/requirements.rocm.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --config=pyproject.toml --output-file=benchmarks/huggingface/requirements.rocm.txt --resolver=backtracking .pin/tmp-constraints-rocm-hf.txt benchmarks/huggingface/requirements.in +# pip-compile --output-file=benchmarks/huggingface/requirements.rocm.txt .pin/tmp-constraints-rocm-hf.txt benchmarks/huggingface/requirements.in # --extra-index-url https://download.pytorch.org/whl/rocm5.6/ @@ -162,6 +162,7 @@ tokenizers==0.14.1 # transformers torch==2.1.0+rocm5.6 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # -r benchmarks/huggingface/requirements.in # pytorch-triton-rocm tqdm==4.66.1 @@ -170,7 +171,9 @@ tqdm==4.66.1 # huggingface-hub # transformers transformers==4.35.0 - # via -r benchmarks/huggingface/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/huggingface/requirements.in typing-extensions==4.8.0 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -186,4 +189,7 @@ varname==0.10.0 # -c .pin/../.pin/constraints-rocm-torch.txt # giving voir==0.2.11 - # via -r benchmarks/huggingface/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -c .pin/../constraints/rocm.txt + # -r benchmarks/huggingface/requirements.in diff --git a/benchmarks/llama/requirements.cuda.txt b/benchmarks/llama/requirements.cuda.txt new file mode 100644 index 000000000..7601f7940 --- /dev/null +++ b/benchmarks/llama/requirements.cuda.txt @@ -0,0 +1,336 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --output-file=benchmarks/llama/requirements.cuda.txt .pin/tmp-constraints-cuda-llm.txt benchmarks/llama/requirements.in +# +--extra-index-url https://download.pytorch.org/whl/cu118 + +aiohttp==3.9.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets + # fsspec +aiosignal==1.3.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # aiohttp +antlr4-python3-runtime==4.9.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # omegaconf +asttokens==2.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +async-timeout==4.0.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # aiohttp +attrs==23.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # aiohttp +certifi==2024.2.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +charset-normalizer==3.3.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +codefind==0.1.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # ptera +datasets==2.17.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/llama/requirements.in +dill==0.3.8 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets + # multiprocess +executing==1.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # varname +fairscale==0.4.13 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/llama/requirements.in +filelock==3.13.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets + # huggingface-hub + # torch + # transformers + # triton +fire==0.5.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/llama/requirements.in +frozenlist==1.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # aiohttp + # aiosignal +fsspec[http]==2023.10.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets + # huggingface-hub + # torch +giving==0.4.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # ptera + # voir +huggingface-hub==0.20.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets + # tokenizers + # transformers +idna==3.6 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests + # yarl +jinja2==3.1.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +markdown-it-py==3.0.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # rich +markupsafe==2.1.5 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # jinja2 +mdurl==0.1.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # markdown-it-py +mpmath==1.3.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # sympy +multidict==6.0.5 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # aiohttp + # yarl +multiprocess==0.70.16 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets +networkx==3.2.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +numpy==1.26.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets + # fairscale + # pandas + # pyarrow + # transformers +nvidia-cublas-cu11==11.11.3.6 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # nvidia-cudnn-cu11 + # nvidia-cusolver-cu11 + # torch +nvidia-cuda-cupti-cu11==11.8.87 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cuda-nvrtc-cu11==11.8.89 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cuda-runtime-cu11==11.8.89 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cudnn-cu11==8.7.0.84 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cufft-cu11==10.9.0.58 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-curand-cu11==10.3.0.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cusolver-cu11==11.4.1.48 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cusparse-cu11==11.7.5.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-nccl-cu11==2.19.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-nvtx-cu11==11.8.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +omegaconf==2.3.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +ovld==0.3.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +packaging==23.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets + # huggingface-hub + # transformers +pandas==2.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets +ptera==1.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +pyarrow==15.0.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets +pyarrow-hotfix==0.6 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets +pygments==2.17.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # rich +pynvml==11.5.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +python-dateutil==2.8.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # pandas +pytz==2024.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # pandas +pyyaml==6.0.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets + # huggingface-hub + # omegaconf + # transformers +reactivex==4.0.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +regex==2023.12.25 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # transformers +requests==2.31.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets + # fsspec + # huggingface-hub + # transformers +rich==13.7.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +safetensors==0.4.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # transformers +sentencepiece==0.1.99 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/llama/requirements.in +six==1.16.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # asttokens + # fire + # python-dateutil +sympy==1.12 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +termcolor==2.4.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # fire +tokenizers==0.15.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # transformers +torch==2.2.0+cu118 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/llama/requirements.in + # fairscale +tqdm==4.66.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets + # huggingface-hub + # transformers +transformers==4.37.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/llama/requirements.in +triton==2.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +typing-extensions==4.9.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # huggingface-hub + # reactivex + # torch +tzdata==2023.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # pandas +urllib3==1.26.18 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +varname==0.10.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +voir==0.2.12 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -c .pin/../constraints/cuda.txt + # -r benchmarks/llama/requirements.in +xxhash==3.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # datasets +yarl==1.9.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # aiohttp diff --git a/benchmarks/llama/requirements.rocm.txt b/benchmarks/llama/requirements.rocm.txt index eb26e2fa9..bcd5391d8 100644 --- a/benchmarks/llama/requirements.rocm.txt +++ b/benchmarks/llama/requirements.rocm.txt @@ -1,186 +1,295 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --config=pyproject.toml --output-file=benchmarks/llama/requirements.rocm.txt --resolver=backtracking .pin/tmp-constraints-rocm-llm.txt benchmarks/llama/requirements.in +# pip-compile --output-file=benchmarks/llama/requirements.rocm.txt .pin/tmp-constraints-rocm-llm.txt benchmarks/llama/requirements.in # --extra-index-url https://download.pytorch.org/whl/rocm5.6/ aiohttp==3.8.6 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # datasets # fsspec aiosignal==1.3.1 - # via aiohttp + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # aiohttp antlr4-python3-runtime==4.9.3 - # via omegaconf + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # omegaconf asttokens==2.4.1 - # via giving + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # giving async-timeout==4.0.3 - # via aiohttp + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # aiohttp attrs==23.1.0 - # via aiohttp + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # aiohttp certifi==2023.7.22 - # via requests + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # requests charset-normalizer==3.3.2 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # aiohttp # requests cmake==3.27.7 - # via pytorch-triton-rocm + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # pytorch-triton-rocm codefind==0.1.3 - # via ptera + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # ptera datasets==2.14.6 - # via -r benchmarks/llama/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/llama/requirements.in dill==0.3.7 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # datasets # multiprocess executing==1.2.0 - # via varname + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # varname fairscale==0.4.13 - # via -r benchmarks/llama/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/llama/requirements.in filelock==3.13.1 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # huggingface-hub # pytorch-triton-rocm # torch # transformers fire==0.5.0 - # via -r benchmarks/llama/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/llama/requirements.in frozenlist==1.4.0 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # aiohttp # aiosignal fsspec[http]==2023.10.0 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # datasets # huggingface-hub # torch giving==0.4.2 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # ptera # voir huggingface-hub==0.17.3 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # datasets # tokenizers # transformers idna==3.4 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # requests # yarl jinja2==3.1.2 - # via torch + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # torch lit==17.0.4 - # via pytorch-triton-rocm + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # pytorch-triton-rocm markdown-it-py==3.0.0 - # via rich + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # rich markupsafe==2.1.3 - # via jinja2 + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # jinja2 mdurl==0.1.2 - # via markdown-it-py + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # markdown-it-py mpmath==1.3.0 - # via sympy + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # sympy multidict==6.0.4 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # aiohttp # yarl multiprocess==0.70.15 - # via datasets + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # datasets networkx==3.2.1 - # via torch + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # torch numpy==1.26.1 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # datasets # fairscale # pandas # pyarrow # transformers omegaconf==2.3.0 - # via voir + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # voir ovld==0.3.2 - # via voir + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # voir packaging==23.2 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # datasets # huggingface-hub # transformers pandas==2.1.2 - # via datasets + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # datasets ptera==1.4.1 - # via voir + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # voir pyarrow==14.0.0 - # via datasets + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # datasets pygments==2.16.1 - # via rich + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # rich pynvml==11.5.0 - # via voir + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # voir python-dateutil==2.8.2 - # via pandas + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # pandas pytorch-triton-rocm==2.1.0 - # via torch + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # torch pytz==2023.3.post1 - # via pandas + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # pandas pyyaml==6.0.1 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # datasets # huggingface-hub # omegaconf # transformers reactivex==4.0.4 - # via giving + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # giving regex==2023.10.3 - # via transformers + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # transformers requests==2.31.0 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # datasets # fsspec # huggingface-hub # transformers rich==13.6.0 - # via voir + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # voir safetensors==0.4.0 - # via transformers + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # transformers sentencepiece==0.1.99 - # via -r benchmarks/llama/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/llama/requirements.in six==1.16.0 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # asttokens # fire # python-dateutil sympy==1.12 - # via torch -termcolor==2.3.0 - # via fire + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # torch +termcolor==2.4.0 + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # fire tokenizers==0.14.1 - # via transformers + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # transformers torch==2.1.0+rocm5.6 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # -r benchmarks/llama/requirements.in # fairscale # pytorch-triton-rocm tqdm==4.66.1 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # datasets # huggingface-hub # transformers transformers==4.35.0 - # via -r benchmarks/llama/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/llama/requirements.in typing-extensions==4.8.0 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # huggingface-hub # reactivex # torch tzdata==2023.3 - # via pandas -urllib3==2.0.7 - # via requests + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # pandas +urllib3==1.26.18 + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # requests varname==0.10.0 - # via giving + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # giving voir==0.2.11 - # via -r benchmarks/llama/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -c .pin/../constraints/rocm.txt + # -r benchmarks/llama/requirements.in xxhash==3.4.1 - # via datasets + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # datasets yarl==1.9.2 - # via aiohttp + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # aiohttp diff --git a/benchmarks/rwkv/requirements.cuda.txt b/benchmarks/rwkv/requirements.cuda.txt new file mode 100644 index 000000000..9fcea0ccd --- /dev/null +++ b/benchmarks/rwkv/requirements.cuda.txt @@ -0,0 +1,291 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --output-file=benchmarks/rwkv/requirements.cuda.txt .pin/tmp-constraints-cuda-rwkv.txt benchmarks/rwkv/requirements.in +# +--extra-index-url https://download.pytorch.org/whl/cu118 + +aiohttp==3.9.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # fsspec +aiosignal==1.3.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # aiohttp +antlr4-python3-runtime==4.9.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # omegaconf +asttokens==2.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +async-timeout==4.0.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # aiohttp +attrs==23.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # aiohttp +certifi==2024.2.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +charset-normalizer==3.3.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +codefind==0.1.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # ptera +deepspeed==0.13.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/rwkv/requirements.in +executing==1.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # varname +filelock==3.13.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch + # triton +frozenlist==1.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # aiohttp + # aiosignal +fsspec[http]==2023.10.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # pytorch-lightning + # torch +giving==0.4.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # ptera + # voir +hjson==3.1.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # deepspeed +idna==3.6 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests + # yarl +jinja2==3.1.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +lightning-utilities==0.10.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # pytorch-lightning + # torchmetrics +markdown-it-py==3.0.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # rich +markupsafe==2.1.5 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # jinja2 +mdurl==0.1.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # markdown-it-py +mpmath==1.3.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # sympy +multidict==6.0.5 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # aiohttp + # yarl +networkx==3.2.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +ninja==1.11.1.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/rwkv/requirements.in + # deepspeed +numpy==1.26.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/rwkv/requirements.in + # deepspeed + # pytorch-lightning + # torchmetrics +nvidia-cublas-cu11==11.11.3.6 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # nvidia-cudnn-cu11 + # nvidia-cusolver-cu11 + # torch +nvidia-cuda-cupti-cu11==11.8.87 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cuda-nvrtc-cu11==11.8.89 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cuda-runtime-cu11==11.8.89 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cudnn-cu11==8.7.0.84 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cufft-cu11==10.9.0.58 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-curand-cu11==10.3.0.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cusolver-cu11==11.4.1.48 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cusparse-cu11==11.7.5.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-nccl-cu11==2.19.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-nvtx-cu11==11.8.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +omegaconf==2.3.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +ovld==0.3.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +packaging==23.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # deepspeed + # lightning-utilities + # pytorch-lightning + # torchmetrics +psutil==5.9.8 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # deepspeed +ptera==1.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +py-cpuinfo==9.0.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # deepspeed +pydantic==1.10.14 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/rwkv/requirements.in + # deepspeed +pygments==2.17.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # rich +pynvml==11.5.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # deepspeed + # voir +pytorch-lightning==1.9.5 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/rwkv/requirements.in +pyyaml==6.0.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # omegaconf + # pytorch-lightning +reactivex==4.0.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +requests==2.31.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # fsspec +rich==13.7.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +six==1.16.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # asttokens +sympy==1.12 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +torch==2.2.0+cu118 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/rwkv/requirements.in + # deepspeed + # pytorch-lightning + # torchmetrics +torchmetrics==1.0.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # pytorch-lightning +tqdm==4.66.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # deepspeed + # pytorch-lightning +triton==2.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +typing-extensions==4.9.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # lightning-utilities + # pydantic + # pytorch-lightning + # reactivex + # torch +urllib3==1.26.18 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +varname==0.10.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +voir==0.2.12 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -c .pin/../constraints/cuda.txt + # -r benchmarks/rwkv/requirements.in +yarl==1.9.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # aiohttp + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/benchmarks/rwkv/requirements.rocm.txt b/benchmarks/rwkv/requirements.rocm.txt index e97d63520..5cb2ecfad 100644 --- a/benchmarks/rwkv/requirements.rocm.txt +++ b/benchmarks/rwkv/requirements.rocm.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --config=pyproject.toml --output-file=benchmarks/rwkv/requirements.rocm.txt --resolver=backtracking .pin/tmp-constraints-rocm-rwkv.txt benchmarks/rwkv/requirements.in +# pip-compile --output-file=benchmarks/rwkv/requirements.rocm.txt .pin/tmp-constraints-rocm-rwkv.txt benchmarks/rwkv/requirements.in # --extra-index-url https://download.pytorch.org/whl/rocm5.6/ @@ -48,7 +48,9 @@ codefind==0.1.3 # -c .pin/../.pin/constraints-rocm-torch.txt # ptera deepspeed==0.12.2 - # via -r benchmarks/rwkv/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/rwkv/requirements.in executing==1.2.0 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -122,10 +124,12 @@ networkx==3.2.1 # torch ninja==1.11.1.1 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # -r benchmarks/rwkv/requirements.in # deepspeed numpy==1.26.1 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # -r benchmarks/rwkv/requirements.in # deepspeed # pytorch-lightning @@ -159,6 +163,7 @@ py-cpuinfo==9.0.0 # deepspeed pydantic==1.10.13 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # -r benchmarks/rwkv/requirements.in # deepspeed pygments==2.16.1 @@ -171,7 +176,9 @@ pynvml==11.5.0 # deepspeed # voir pytorch-lightning==1.9.5 - # via -r benchmarks/rwkv/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/rwkv/requirements.in pytorch-triton-rocm==2.1.0 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -203,6 +210,7 @@ sympy==1.12 # torch torch==2.1.0+rocm5.6 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # -r benchmarks/rwkv/requirements.in # deepspeed # pytorch-lightning @@ -234,7 +242,10 @@ varname==0.10.0 # -c .pin/../.pin/constraints-rocm-torch.txt # giving voir==0.2.11 - # via -r benchmarks/rwkv/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -c .pin/../constraints/rocm.txt + # -r benchmarks/rwkv/requirements.in yarl==1.9.2 # via # -c .pin/../.pin/constraints-rocm-torch.txt diff --git a/benchmarks/stargan/requirements.cuda.txt b/benchmarks/stargan/requirements.cuda.txt new file mode 100644 index 000000000..e8797a327 --- /dev/null +++ b/benchmarks/stargan/requirements.cuda.txt @@ -0,0 +1,204 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --output-file=benchmarks/stargan/requirements.cuda.txt .pin/tmp-constraints-cuda-stargan.txt benchmarks/stargan/requirements.in +# +--extra-index-url https://download.pytorch.org/whl/cu118 + +antlr4-python3-runtime==4.9.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # omegaconf +asttokens==2.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +certifi==2024.2.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +charset-normalizer==3.3.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +codefind==0.1.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # ptera +executing==1.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # varname +filelock==3.13.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch + # triton +fsspec==2023.10.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +giving==0.4.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # ptera + # voir +idna==3.6 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +jinja2==3.1.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +markdown-it-py==3.0.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # rich +markupsafe==2.1.5 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # jinja2 +mdurl==0.1.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # markdown-it-py +mpmath==1.3.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # sympy +networkx==3.2.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +numpy==1.26.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/stargan/requirements.in + # torchvision +nvidia-cublas-cu11==11.11.3.6 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # nvidia-cudnn-cu11 + # nvidia-cusolver-cu11 + # torch +nvidia-cuda-cupti-cu11==11.8.87 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cuda-nvrtc-cu11==11.8.89 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cuda-runtime-cu11==11.8.89 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cudnn-cu11==8.7.0.84 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cufft-cu11==10.9.0.58 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-curand-cu11==10.3.0.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cusolver-cu11==11.4.1.48 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cusparse-cu11==11.7.5.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-nccl-cu11==2.19.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-nvtx-cu11==11.8.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +omegaconf==2.3.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +ovld==0.3.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +pillow==10.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchvision +ptera==1.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +pygments==2.17.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # rich +pynvml==11.5.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +pyyaml==6.0.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # omegaconf +reactivex==4.0.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +requests==2.31.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchvision +rich==13.7.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +six==1.16.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # asttokens +sympy==1.12 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +torch==2.2.0+cu118 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/stargan/requirements.in + # torchvision +torchvision==0.17.0+cu118 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/stargan/requirements.in +triton==2.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +typing-extensions==4.9.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # reactivex + # torch +urllib3==1.26.18 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +varname==0.10.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +voir==0.2.12 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -c .pin/../constraints/cuda.txt + # -r benchmarks/stargan/requirements.in diff --git a/benchmarks/stargan/requirements.rocm.txt b/benchmarks/stargan/requirements.rocm.txt index d2b904c55..6f5938ac5 100644 --- a/benchmarks/stargan/requirements.rocm.txt +++ b/benchmarks/stargan/requirements.rocm.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --config=pyproject.toml --output-file=benchmarks/stargan/requirements.rocm.txt --resolver=backtracking .pin/tmp-constraints-rocm-stargan.txt benchmarks/stargan/requirements.in +# pip-compile --output-file=benchmarks/stargan/requirements.rocm.txt .pin/tmp-constraints-rocm-stargan.txt benchmarks/stargan/requirements.in # --extra-index-url https://download.pytorch.org/whl/rocm5.6/ @@ -82,6 +82,7 @@ networkx==3.2.1 # torch numpy==1.26.1 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # -r benchmarks/stargan/requirements.in # torchvision omegaconf==2.3.0 @@ -138,11 +139,14 @@ sympy==1.12 # torch torch==2.1.0+rocm5.6 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # -r benchmarks/stargan/requirements.in # pytorch-triton-rocm # torchvision torchvision==0.16.0+rocm5.6 - # via -r benchmarks/stargan/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/stargan/requirements.in typing-extensions==4.8.0 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -157,4 +161,7 @@ varname==0.10.0 # -c .pin/../.pin/constraints-rocm-torch.txt # giving voir==0.2.11 - # via -r benchmarks/stargan/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -c .pin/../constraints/rocm.txt + # -r benchmarks/stargan/requirements.in diff --git a/benchmarks/super-slomo/requirements.cuda.txt b/benchmarks/super-slomo/requirements.cuda.txt new file mode 100644 index 000000000..6f4b9805a --- /dev/null +++ b/benchmarks/super-slomo/requirements.cuda.txt @@ -0,0 +1,213 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --output-file=benchmarks/super-slomo/requirements.cuda.txt .pin/tmp-constraints-cuda-super-slomo.txt benchmarks/super-slomo/requirements.in +# +--extra-index-url https://download.pytorch.org/whl/cu118 + +antlr4-python3-runtime==4.9.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # omegaconf +asttokens==2.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +certifi==2024.2.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +charset-normalizer==3.3.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +codefind==0.1.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # ptera +executing==1.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # varname +filelock==3.13.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch + # triton +fsspec==2023.10.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +giving==0.4.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # ptera + # voir +idna==3.6 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +jinja2==3.1.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +markdown-it-py==3.0.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # rich +markupsafe==2.1.5 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # jinja2 +mdurl==0.1.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # markdown-it-py +mpmath==1.3.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # sympy +networkx==3.2.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +numpy==1.26.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/super-slomo/requirements.in + # opencv-python + # torchvision +nvidia-cublas-cu11==11.11.3.6 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # nvidia-cudnn-cu11 + # nvidia-cusolver-cu11 + # torch +nvidia-cuda-cupti-cu11==11.8.87 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cuda-nvrtc-cu11==11.8.89 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cuda-runtime-cu11==11.8.89 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cudnn-cu11==8.7.0.84 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cufft-cu11==10.9.0.58 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-curand-cu11==10.3.0.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cusolver-cu11==11.4.1.48 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cusparse-cu11==11.7.5.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-nccl-cu11==2.19.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-nvtx-cu11==11.8.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +omegaconf==2.3.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +opencv-python==4.9.0.80 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/super-slomo/requirements.in +ovld==0.3.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +pillow==10.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchvision +ptera==1.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +pygments==2.17.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # rich +pynvml==11.5.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +pyyaml==6.0.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # omegaconf +reactivex==4.0.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +requests==2.31.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchvision +rich==13.7.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +six==1.16.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # asttokens +sympy==1.12 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +torch==2.2.0+cu118 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/super-slomo/requirements.in + # torchvision +torchvision==0.17.0+cu118 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/super-slomo/requirements.in +tqdm==4.66.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/super-slomo/requirements.in +triton==2.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +typing-extensions==4.9.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # reactivex + # torch +urllib3==1.26.18 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +varname==0.10.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +voir==0.2.12 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -c .pin/../constraints/cuda.txt + # -r benchmarks/super-slomo/requirements.in diff --git a/benchmarks/super-slomo/requirements.rocm.txt b/benchmarks/super-slomo/requirements.rocm.txt index 02ff9f070..4ae1c575b 100644 --- a/benchmarks/super-slomo/requirements.rocm.txt +++ b/benchmarks/super-slomo/requirements.rocm.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --config=pyproject.toml --output-file=benchmarks/super-slomo/requirements.rocm.txt --resolver=backtracking .pin/tmp-constraints-rocm-super-slomo.txt benchmarks/super-slomo/requirements.in +# pip-compile --output-file=benchmarks/super-slomo/requirements.rocm.txt .pin/tmp-constraints-rocm-super-slomo.txt benchmarks/super-slomo/requirements.in # --extra-index-url https://download.pytorch.org/whl/rocm5.6/ @@ -82,6 +82,7 @@ networkx==3.2.1 # torch numpy==1.26.1 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # -r benchmarks/super-slomo/requirements.in # opencv-python # torchvision @@ -90,7 +91,9 @@ omegaconf==2.3.0 # -c .pin/../.pin/constraints-rocm-torch.txt # voir opencv-python==4.8.1.78 - # via -r benchmarks/super-slomo/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/super-slomo/requirements.in ovld==0.3.2 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -141,13 +144,18 @@ sympy==1.12 # torch torch==2.1.0+rocm5.6 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # -r benchmarks/super-slomo/requirements.in # pytorch-triton-rocm # torchvision torchvision==0.16.0+rocm5.6 - # via -r benchmarks/super-slomo/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/super-slomo/requirements.in tqdm==4.66.1 - # via -r benchmarks/super-slomo/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/super-slomo/requirements.in typing-extensions==4.8.0 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -162,4 +170,7 @@ varname==0.10.0 # -c .pin/../.pin/constraints-rocm-torch.txt # giving voir==0.2.11 - # via -r benchmarks/super-slomo/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -c .pin/../constraints/rocm.txt + # -r benchmarks/super-slomo/requirements.in diff --git a/benchmarks/timm/requirements.cuda.txt b/benchmarks/timm/requirements.cuda.txt new file mode 100644 index 000000000..00ec850b6 --- /dev/null +++ b/benchmarks/timm/requirements.cuda.txt @@ -0,0 +1,225 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --output-file=benchmarks/timm/requirements.cuda.txt .pin/tmp-constraints-cuda-timm.txt benchmarks/timm/requirements.in +# +--extra-index-url https://download.pytorch.org/whl/cu118 + +antlr4-python3-runtime==4.9.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # omegaconf +asttokens==2.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +certifi==2024.2.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +charset-normalizer==3.3.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +codefind==0.1.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # ptera +executing==1.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # varname +filelock==3.13.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # huggingface-hub + # torch + # triton +fsspec==2023.10.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # huggingface-hub + # torch +giving==0.4.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # ptera + # voir +huggingface-hub==0.20.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/timm/requirements.in +idna==3.6 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +jinja2==3.1.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +markdown-it-py==3.0.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # rich +markupsafe==2.1.5 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # jinja2 +mdurl==0.1.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # markdown-it-py +mpmath==1.3.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # sympy +networkx==3.2.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +numpy==1.26.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchvision +nvidia-cublas-cu11==11.11.3.6 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # nvidia-cudnn-cu11 + # nvidia-cusolver-cu11 + # torch +nvidia-cuda-cupti-cu11==11.8.87 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cuda-nvrtc-cu11==11.8.89 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cuda-runtime-cu11==11.8.89 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cudnn-cu11==8.7.0.84 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cufft-cu11==10.9.0.58 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-curand-cu11==10.3.0.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cusolver-cu11==11.4.1.48 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-cusparse-cu11==11.7.5.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-nccl-cu11==2.19.3 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +nvidia-nvtx-cu11==11.8.86 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +omegaconf==2.3.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +ovld==0.3.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +packaging==23.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # huggingface-hub +pillow==10.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchvision +ptera==1.4.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +pygments==2.17.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # rich +pynvml==11.5.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +pyyaml==6.0.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/timm/requirements.in + # huggingface-hub + # omegaconf +reactivex==4.0.4 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +requests==2.31.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # huggingface-hub + # torchvision +rich==13.7.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir +safetensors==0.4.2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/timm/requirements.in +six==1.16.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # asttokens +sympy==1.12 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +torch==2.2.0+cu118 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/timm/requirements.in + # torchvision +torchvision==0.17.0+cu118 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/timm/requirements.in +tqdm==4.66.1 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # huggingface-hub +triton==2.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch +typing-extensions==4.9.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # huggingface-hub + # reactivex + # torch +urllib3==1.26.18 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests +varname==0.10.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving +voir==0.2.12 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -c .pin/../constraints/cuda.txt + # -r benchmarks/timm/requirements.in diff --git a/benchmarks/timm/requirements.rocm.txt b/benchmarks/timm/requirements.rocm.txt index 6b15125f5..870523e20 100644 --- a/benchmarks/timm/requirements.rocm.txt +++ b/benchmarks/timm/requirements.rocm.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --config=pyproject.toml --output-file=benchmarks/timm/requirements.rocm.txt --resolver=backtracking .pin/tmp-constraints-rocm-timm.txt benchmarks/timm/requirements.in +# pip-compile --output-file=benchmarks/timm/requirements.rocm.txt .pin/tmp-constraints-rocm-timm.txt benchmarks/timm/requirements.in # --extra-index-url https://download.pytorch.org/whl/rocm5.6/ @@ -51,7 +51,9 @@ giving==0.4.2 # ptera # voir huggingface-hub==0.17.3 - # via -r benchmarks/timm/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/timm/requirements.in idna==3.4 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -122,6 +124,7 @@ pytorch-triton-rocm==2.1.0 # torch pyyaml==6.0.1 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # -r benchmarks/timm/requirements.in # huggingface-hub # omegaconf @@ -139,7 +142,9 @@ rich==13.6.0 # -c .pin/../.pin/constraints-rocm-torch.txt # voir safetensors==0.4.0 - # via -r benchmarks/timm/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/timm/requirements.in six==1.16.0 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -150,11 +155,14 @@ sympy==1.12 # torch torch==2.1.0+rocm5.6 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # -r benchmarks/timm/requirements.in # pytorch-triton-rocm # torchvision torchvision==0.16.0+rocm5.6 - # via -r benchmarks/timm/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/timm/requirements.in tqdm==4.66.1 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -174,4 +182,7 @@ varname==0.10.0 # -c .pin/../.pin/constraints-rocm-torch.txt # giving voir==0.2.11 - # via -r benchmarks/timm/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -c .pin/../constraints/rocm.txt + # -r benchmarks/timm/requirements.in diff --git a/benchmarks/torchvision/main.py b/benchmarks/torchvision/main.py index 1177d1953..d7518d179 100644 --- a/benchmarks/torchvision/main.py +++ b/benchmarks/torchvision/main.py @@ -16,23 +16,6 @@ normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) -class Stats: - def __init__(self): - self.count = 0 - self.epoch_count = 0 - - def newbatch(self, bs): - self.count += bs.shape[0] - self.epoch_count += bs.shape[0] - - def newepoch(self): - self.epoch_count = 0 - - - -stats = Stats() - - def is_tf32_allowed(args): return "tf32" in args.precision @@ -61,29 +44,17 @@ def scaling(enable): def train_epoch(model, criterion, optimizer, loader, device, scaler=None, timer=None): - global stats - - stats.newepoch() model.train() s = time.time() p = time.time() - def iterator(loader, timer): + def toiterator(loader, timer): with timer.timeit("loader"): - iterator = iter(loader) - - while True: - with timer.timeit("next"): - try: - batch = next(iterator) - except StopIteration: - return - - yield batch + return iter(loader) # this is what computes the batch size - for inp, target in voir.iterate("train", iterator(loader, timer), True): + for inp, target in timer.iterator(voir.iterate("train", toiterator(loader, timer), True)): with timer.timeit("batch"): inp = inp.to(device) @@ -127,6 +98,18 @@ def __len__(self): def main(): + from voir.phase import StopProgram + + try: + with timeit("main") as main_timer: + _main(main_timer) + + main_timer.show() + except StopProgram: + main_timer.show() + raise + +def _main(main_timer): parser = argparse.ArgumentParser(description="Torchvision models") parser.add_argument( "--batch-size", @@ -237,8 +220,11 @@ def main(): train_loader = torch.utils.data.DataLoader( train, batch_size=args.batch_size, - shuffle=True, num_workers=args.num_workers, + sampler=torch.utils.data.RandomSampler( + train, + replacement=True, + num_samples=len(train) * args.epochs) ) else: train_loader = SyntheticData( @@ -254,7 +240,7 @@ def main(): else: scaler = None - with timeit("train") as train_timer: + with main_timer.timeit("train") as train_timer: with given() as gv: if not args.no_stdout: gv.where("loss").display() @@ -272,7 +258,7 @@ def main(): model, criterion, optimizer, train_loader, device, scaler=scaler, timer=epoch_timer ) - train_timer.show() + break if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/benchmarks/torchvision/requirements.cuda.txt b/benchmarks/torchvision/requirements.cuda.txt index d35abd90a..281055fd8 100644 --- a/benchmarks/torchvision/requirements.cuda.txt +++ b/benchmarks/torchvision/requirements.cuda.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # pip-compile --output-file=benchmarks/torchvision/requirements.cuda.txt .pin/tmp-constraints-cuda-torchvision.txt benchmarks/torchvision/requirements.in @@ -7,107 +7,201 @@ --extra-index-url https://download.pytorch.org/whl/cu118 antlr4-python3-runtime==4.9.3 - # via omegaconf + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # omegaconf asttokens==2.4.1 - # via giving + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving certifi==2024.2.2 - # via requests + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests charset-normalizer==3.3.2 - # via requests + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests codefind==0.1.3 - # via ptera + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # ptera executing==1.2.0 - # via varname + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # varname filelock==3.13.1 - # via torch -fsspec==2024.2.0 - # via torch + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch + # triton +fsspec==2023.10.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch giving==0.4.2 # via + # -c .pin/../.pin/constraints-cuda-torch.txt # ptera # voir idna==3.6 - # via requests + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests jinja2==3.1.3 - # via torch + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch markdown-it-py==3.0.0 - # via rich + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # rich markupsafe==2.1.5 - # via jinja2 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # jinja2 mdurl==0.1.2 - # via markdown-it-py + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # markdown-it-py mpmath==1.3.0 - # via sympy + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # sympy networkx==3.2.1 - # via torch + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch numpy==1.26.4 - # via torchvision + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchvision nvidia-cublas-cu11==11.11.3.6 # via + # -c .pin/../.pin/constraints-cuda-torch.txt # nvidia-cudnn-cu11 # nvidia-cusolver-cu11 # torch nvidia-cuda-cupti-cu11==11.8.87 - # via torch + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch nvidia-cuda-nvrtc-cu11==11.8.89 - # via torch + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch nvidia-cuda-runtime-cu11==11.8.89 - # via torch + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch nvidia-cudnn-cu11==8.7.0.84 - # via torch + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch nvidia-cufft-cu11==10.9.0.58 - # via torch + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch nvidia-curand-cu11==10.3.0.86 - # via torch + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch nvidia-cusolver-cu11==11.4.1.48 - # via torch + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch nvidia-cusparse-cu11==11.7.5.86 - # via torch + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch nvidia-nccl-cu11==2.19.3 - # via torch + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch nvidia-nvtx-cu11==11.8.86 - # via torch + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch omegaconf==2.3.0 - # via voir + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir ovld==0.3.2 - # via voir + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir pillow==10.2.0 - # via torchvision + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchvision ptera==1.4.1 - # via voir + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir pygments==2.17.2 - # via rich + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # rich pynvml==11.5.0 - # via voir + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir pyyaml==6.0.1 - # via omegaconf + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # omegaconf reactivex==4.0.4 - # via giving + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving requests==2.31.0 - # via torchvision + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torchvision rich==13.7.0 - # via voir + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # voir six==1.16.0 - # via asttokens + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # asttokens sympy==1.12 - # via torch + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch torch==2.2.0+cu118 # via + # -c .pin/../.pin/constraints-cuda-torch.txt # -r benchmarks/torchvision/requirements.in # torchvision torchvision==0.17.0+cu118 - # via -r benchmarks/torchvision/requirements.in + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/torchvision/requirements.in tqdm==4.66.1 - # via -r benchmarks/torchvision/requirements.in + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # -r benchmarks/torchvision/requirements.in +triton==2.2.0 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # torch typing-extensions==4.9.0 # via + # -c .pin/../.pin/constraints-cuda-torch.txt # reactivex # torch -urllib3==2.2.0 - # via requests +urllib3==1.26.18 + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # requests varname==0.10.0 - # via giving + # via + # -c .pin/../.pin/constraints-cuda-torch.txt + # giving voir==0.2.12 # via + # -c .pin/../.pin/constraints-cuda-torch.txt # -c .pin/../constraints/cuda.txt # -r benchmarks/torchvision/requirements.in diff --git a/benchmarks/torchvision/requirements.rocm.txt b/benchmarks/torchvision/requirements.rocm.txt index 618dff8f8..4601266de 100644 --- a/benchmarks/torchvision/requirements.rocm.txt +++ b/benchmarks/torchvision/requirements.rocm.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --config=pyproject.toml --output-file=benchmarks/torchvision/requirements.rocm.txt --resolver=backtracking .pin/tmp-constraints-rocm-torchvision.txt benchmarks/torchvision/requirements.in +# pip-compile --output-file=benchmarks/torchvision/requirements.rocm.txt .pin/tmp-constraints-rocm-torchvision.txt benchmarks/torchvision/requirements.in # --extra-index-url https://download.pytorch.org/whl/rocm5.6/ @@ -138,13 +138,18 @@ sympy==1.12 # torch torch==2.1.0+rocm5.6 # via + # -c .pin/../.pin/constraints-rocm-torch.txt # -r benchmarks/torchvision/requirements.in # pytorch-triton-rocm # torchvision torchvision==0.16.0+rocm5.6 - # via -r benchmarks/torchvision/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/torchvision/requirements.in tqdm==4.66.1 - # via -r benchmarks/torchvision/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -r benchmarks/torchvision/requirements.in typing-extensions==4.8.0 # via # -c .pin/../.pin/constraints-rocm-torch.txt @@ -159,4 +164,7 @@ varname==0.10.0 # -c .pin/../.pin/constraints-rocm-torch.txt # giving voir==0.2.11 - # via -r benchmarks/torchvision/requirements.in + # via + # -c .pin/../.pin/constraints-rocm-torch.txt + # -c .pin/../constraints/rocm.txt + # -r benchmarks/torchvision/requirements.in diff --git a/commands.sh b/commands.sh index ce296f888..999583de3 100644 --- a/commands.sh +++ b/commands.sh @@ -1,26 +1,30 @@ -# --- -# Virtual Env -# =========== -export VIRTUAL_ENV="/Tmp/slurm.4123709.0/base/venv/torch" +#!/bin/sh - -# --- -# Milabench -# ========= -export MILABENCH_DIR_BASE="/Tmp/slurm.4123709.0/base" -export MILABENCH_DIR_VENV="/Tmp/slurm.4123709.0/base/venv/torch" -export MILABENCH_DIR_DATA="/Tmp/slurm.4123709.0/base/data" -export MILABENCH_DIR_RUNS="/Tmp/slurm.4123709.0/base/runs" -export MILABENCH_DIR_EXTRA="/Tmp/slurm.4123709.0/base/extra/torchvision" -export MILABENCH_DIR_CACHE="/Tmp/slurm.4123709.0/base/cache" -export MILABENCH_CONFIG='{"system": {"arch": "cuda", "sshkey": null, "nodes": [{"ip": "127.0.0.1", "main": true, "name": "0", "port": 22, "user": "username", "hostname": "localhost", "aliaslist": [], "ipaddrlist": ["70:b5:e8:f0:5a:08", "fe80::1270:fd03:cd:a394%ibp161s0", "::1", "172.16.9.28", "fe80::72b5:e8ff:fef0:5a08%eno8303", "00:00:00:00:00:00", "00:00:02:5d:fe:80:00:00:00:00:00:00:10:70:fd:03:00:cd:a3:94", "10.20.9.28", "00:00:00:bf:fe:80:00:00:00:00:00:00:10:70:fd:03:00:e6:1b:38", "fe80::1270:fd03:e6:1b38%ibp37s0", "127.0.0.1", "10.20.137.28"], "local": true}], "gpu": {"capacity": "0 MiB"}, "self": {"ip": "127.0.0.1", "main": true, "name": "0", "port": 22, "user": "username", "hostname": "localhost", "aliaslist": [], "ipaddrlist": ["70:b5:e8:f0:5a:08", "fe80::1270:fd03:cd:a394%ibp161s0", "::1", "172.16.9.28", "fe80::72b5:e8ff:fef0:5a08%eno8303", "00:00:00:00:00:00", "00:00:02:5d:fe:80:00:00:00:00:00:00:10:70:fd:03:00:cd:a3:94", "10.20.9.28", "00:00:00:bf:fe:80:00:00:00:00:00:00:10:70:fd:03:00:e6:1b:38", "fe80::1270:fd03:e6:1b38%ibp37s0", "127.0.0.1", "10.20.137.28"], "local": true}}, "dirs": {"base": "/Tmp/slurm.4123709.0/base", "venv": "/Tmp/slurm.4123709.0/base/venv/torch", "data": "/Tmp/slurm.4123709.0/base/data", "runs": "/Tmp/slurm.4123709.0/base/runs", "extra": "/Tmp/slurm.4123709.0/base/extra/torchvision", "cache": "/Tmp/slurm.4123709.0/base/cache"}, "group": "torchvision", "install_group": "torch", "install_variant": "cuda", "run_name": "dev", "enabled": true, "capabilities": {"nodes": 1}, "max_duration": 600, "voir": {"options": {"stop": 60, "interval": "1s"}}, "validation": {"usage": {"gpu_load_threshold": 0.5, "gpu_mem_threshold": 0.5}}, "config_base": "/home/mila/d/delaunap/milabench/config", "config_file": "/home/mila/d/delaunap/milabench/config/standard.yaml", "definition": "/home/mila/d/delaunap/milabench/benchmarks/torchvision", "plan": {"method": "per_gpu"}, "argv": {"--precision": "tf32-fp16", "--lr": 0.01, "--no-stdout": true, "--epochs": 50, "--model": "resnet50", "--batch-size": 64}, "tags": ["classification", "convnet", "resnet", "vision"], "weight": 1.0, "name": "resnet50", "tag": ["resnet50"]}' +echo "---" +echo "Virtual Env" +echo "===========" +export VIRTUAL_ENV=/Tmp/slurm.4130792.0/base/venv/torch source $VIRTUAL_ENV/bin/activate +echo "---" +echo "Milabench" +echo "=========" +export MILABENCH_DIR_BASE=/Tmp/slurm.4130792.0/base +export MILABENCH_DIR_VENV=/Tmp/slurm.4130792.0/base/venv/torch +export MILABENCH_DIR_DATA=/Tmp/slurm.4130792.0/base/data +export MILABENCH_DIR_RUNS=/Tmp/slurm.4130792.0/base/runs +export MILABENCH_DIR_EXTRA=/Tmp/slurm.4130792.0/base/extra/torchvision +export MILABENCH_DIR_CACHE=/Tmp/slurm.4130792.0/base/cache +export MILABENCH_CONFIG='{"system": {"arch": "cuda", "sshkey": null, "nodes": [{"ip": "127.0.0.1", "main": true, "name": "0", "port": 22, "user": "username", "hostname": "localhost", "aliaslist": [], "ipaddrlist": ["10.20.9.29", "fe80::1270:fd03:cd:a6a8%ibp37s0", "fe80::1270:fd03:cd:a454%ibp161s0", "10.20.137.29", "00:00:04:cc:fe:80:00:00:00:00:00:00:10:70:fd:03:00:cd:a4:54", "00:00:04:57:fe:80:00:00:00:00:00:00:10:70:fd:03:00:cd:a6:a8", "::1", "cc:48:3a:1e:8d:7e", "127.0.0.1", "fe80::ce48:3aff:fe1e:8d7e%eno8303", "00:00:00:00:00:00", "172.16.9.29"], "local": true}, {"ip": "192.168.0.11", "main": false, "name": "1", "port": 22, "user": "username", "hostname": "192.168.0.11", "aliaslist": [], "ipaddrlist": [], "local": false}], "gpu": {"capacity": "0 MiB"}, "self": {"ip": "127.0.0.1", "main": true, "name": "0", "port": 22, "user": "username", "hostname": "localhost", "aliaslist": [], "ipaddrlist": ["10.20.9.29", "fe80::1270:fd03:cd:a6a8%ibp37s0", "fe80::1270:fd03:cd:a454%ibp161s0", "10.20.137.29", "00:00:04:cc:fe:80:00:00:00:00:00:00:10:70:fd:03:00:cd:a4:54", "00:00:04:57:fe:80:00:00:00:00:00:00:10:70:fd:03:00:cd:a6:a8", "::1", "cc:48:3a:1e:8d:7e", "127.0.0.1", "fe80::ce48:3aff:fe1e:8d7e%eno8303", "00:00:00:00:00:00", "172.16.9.29"], "local": true}}, "dirs": {"base": "/Tmp/slurm.4130792.0/base", "venv": "/Tmp/slurm.4130792.0/base/venv/torch", "data": "/Tmp/slurm.4130792.0/base/data", "runs": "/Tmp/slurm.4130792.0/base/runs", "extra": "/Tmp/slurm.4130792.0/base/extra/torchvision", "cache": "/Tmp/slurm.4130792.0/base/cache"}, "group": "torchvision", "install_group": "torch", "install_variant": "cuda", "run_name": "dev", "enabled": true, "capabilities": {"nodes": 1}, "max_duration": 600, "voir": {"options": {"stop": 60, "interval": "1s"}}, "validation": {"usage": {"gpu_load_threshold": 0.5, "gpu_mem_threshold": 0.5}}, "config_base": "/home/mila/d/delaunap/milabench/config", "config_file": "/home/mila/d/delaunap/milabench/config/standard.yaml", "definition": "/home/mila/d/delaunap/milabench/benchmarks/torchvision", "plan": {"method": "per_gpu"}, "argv": {"--precision": "tf32-fp16", "--lr": 0.01, "--no-stdout": true, "--epochs": 50, "--model": "resnet50", "--batch-size": 64}, "tags": ["classification", "convnet", "resnet", "vision"], "weight": 1.0, "name": "resnet50", "tag": ["resnet50"]}' -# --- -# resnet50 -# ======== -( - python /home/mila/d/delaunap/milabench/benchmarks/torchvision/main.py --precision tf32-fp16 --lr 0.01 --no-stdout --epochs 10 --model resnet50 --batch-size 64 +echo "---" +echo "resnet50" +echo "========" +time ( + CUDA_VISIBLE_DEVICES=0 voir --config /Tmp/slurm.4130792.0/base/extra/torchvision/voirconf-resnet50.D0-0efae956f1553a76c1e03985181900f5.json /home/mila/d/delaunap/milabench/benchmarks/torchvision/main.py --precision tf32-fp16 --lr 0.01 --no-stdout --epochs 15 --model resnet50 --batch-size 64 ) +# time ( +# CUDA_VISIBLE_DEVICES=0 voir --config /Tmp/slurm.4130792.0/base/extra/torchvision/voirconf-resnet50.D0-0efae956f1553a76c1e03985181900f5.json /home/mila/d/delaunap/milabench/benchmarks/torchvision/main.py --precision tf32-fp16 --lr 0.01 --no-stdout --epochs 15 --model resnet50 --batch-size 64 +# ) + diff --git a/milabench/_version.py b/milabench/_version.py index 3b9311daf..bc1e09c71 100644 --- a/milabench/_version.py +++ b/milabench/_version.py @@ -1,5 +1,5 @@ """This file is generated, do not modify""" -__tag__ = "v0.0.6-45-gac2ebf69" -__commit__ = "ac2ebf69ce2d44242a726b81531894f5b3049522" -__date__ = "2024-02-05 12:05:56 -0500" +__tag__ = "v0.0.6-54-ge75f56f1" +__commit__ = "e75f56f1a743da6ca5c46baac352519028da53d9" +__date__ = "2024-02-08 15:45:29 -0500" diff --git a/milabench/cli/dry.py b/milabench/cli/dry.py index d2ec5b528..9e81c192a 100644 --- a/milabench/cli/dry.py +++ b/milabench/cli/dry.py @@ -83,16 +83,16 @@ def section(self, title): self.echo("---") self.echo(title) self.echo("=" * len(title)) - + def echo(self, msg): - self.print(f"echo \"{msg}\"") + self.print(f'echo "{msg}"') def comment(self, cmt): self.print(f"# {cmt}") def env(self, env): for k, v in env.items(): - self.print(f'export {k}={shlex.quote(v)}') + self.print(f"export {k}={shlex.quote(v)}") self.print() @contextmanager @@ -120,9 +120,9 @@ def command(self, *args, env=None, **kwargs): sufix = "" if True: sufix = "&" - + frags = [prefix] + [str(a) for a in args] + [sufix] - + self.print(" ".join(filter(lambda x: x != "", frags))) @@ -142,10 +142,10 @@ def arguments(): ngpu: Option & int = 8 capacity: Option & int = 80000 nnodes: Option & int = 2 - + # [negate] withenv: Option & bool = True - + # [negate] usevoir: Option & bool = True return Arguments(nnodes, ngpu, capacity, withenv, usevoir) @@ -204,11 +204,11 @@ def cli_dry(args=None): if first_pack and args.withenv: first_pack = False gen.section("Virtual Env") - + venv = pack.core._nox_session.env["VIRTUAL_ENV"] - gen.env(VIRTUAL_ENV=VIRTUAL_ENV) + gen.env({"VIRTUAL_ENV": venv}) gen.print("source $VIRTUAL_ENV/bin/activate") - + gen.section("Milabench") gen.env(pack.make_env()) diff --git a/milabench/commands/__init__.py b/milabench/commands/__init__.py index e97404758..cf1997c67 100644 --- a/milabench/commands/__init__.py +++ b/milabench/commands/__init__.py @@ -453,18 +453,18 @@ def __init__(self, executor: SingleCmdCommand, *torchrun_argv, **kwargs) -> None def _argv(self, **kwargs): devices = self.pack.config.get("devices", []) nproc = len(devices) - + if nproc > 1: argv = [*super()._argv(**kwargs), f"--nproc_per_node={nproc}"] - + # Check if the sub-executor targets a module or not cmd = next(iter(self.exec.argv()), None) - + if cmd: # python or voir; tell it to not prepend python since we are doing it if cmd in ("python", "voir"): argv.append("--no-python") - + # if the command exists and it is not a path assume it is a module # script is not a file, maybe it is a module elif not XPath(cmd).exists(): diff --git a/milabench/log.py b/milabench/log.py index 5826d309b..03e0b75fb 100644 --- a/milabench/log.py +++ b/milabench/log.py @@ -5,6 +5,7 @@ import time from collections import defaultdict from datetime import datetime +from io import StringIO from blessed import Terminal from rich.console import Console @@ -167,21 +168,42 @@ def __init__(self, pipe): self.pipe = pipe self.files = {} + def _buffer_open(self, entry): + self.files[entry.tag] = StringIO() + + def _file_open(self, entry): + file = entry.pack.logfile(self.pipe) + os.makedirs(XPath(file).parent, exist_ok=True) + self.files[entry.tag] = open(file, "w").__enter__() + def file(self, entry): if entry.tag not in self.files: - file = entry.pack.logfile(self.pipe) - os.makedirs(XPath(file).parent, exist_ok=True) - self.files[entry.tag] = open(file, "w").__enter__() + self._file_open(entry) + return self.files[entry.tag] def log(self, entry): pass + def _buffer_cleanup(self, entry): + buffer = self.files[entry.tag] + + file = entry.pack.logfile(self.pipe) + os.makedirs(XPath(file).parent, exist_ok=True) + + with open(file, "w") as fp: + fp.write(buffer.getvalue()) + + del self.files[entry.tag] + + def _file_cleanup(self, entry): + self.files[entry.tag].__exit__(None, None, None) + del self.files[entry.tag] + def cleanup(self, entry): if entry.event == "end": if entry.tag in self.files: - self.files[entry.tag].__exit__(None, None, None) - del self.files[entry.tag] + self._file_cleanup(entry) def __call__(self, entry): self.log(entry) @@ -300,9 +322,9 @@ def on_data(self, entry, data, row): load = int(data.get("load", 0) * 100) currm, totalm = data.get("memory", [0, 0]) temp = int(data.get("temperature", 0)) - row[ - f"gpu:{gpuid}" - ] = f"{load}% load | {currm:.0f}/{totalm:.0f} MB | {temp}C" + row[f"gpu:{gpuid}"] = ( + f"{load}% load | {currm:.0f}/{totalm:.0f} MB | {temp}C" + ) row["gpu_load"] = f"{load}%" row["gpu_mem"] = f"{currm:.0f}/{totalm:.0f} MB" row["gpu_temp"] = f"{temp}C" @@ -376,9 +398,9 @@ def on_data(self, entry, data, row): load = int(data.get("load", 0) * 100) currm, totalm = data.get("memory", [0, 0]) temp = int(data.get("temperature", 0)) - row[ - f"gpu:{gpuid}" - ] = f"{load}% load | {currm:.0f}/{totalm:.0f} MB | {temp}C" + row[f"gpu:{gpuid}"] = ( + f"{load}% load | {currm:.0f}/{totalm:.0f} MB | {temp}C" + ) else: task = data.pop("task", "") units = data.pop("units", "") diff --git a/milabench/main.py b/milabench/main.py new file mode 100644 index 000000000..059c7fbf0 --- /dev/null +++ b/milabench/main.py @@ -0,0 +1,4 @@ +from milabench.cli import main + +if __name__ == "__main__": + main() diff --git a/milabench/merge.py b/milabench/merge.py index e5010c629..a9efa4cec 100644 --- a/milabench/merge.py +++ b/milabench/merge.py @@ -1,6 +1,5 @@ """Utilities to merge dictionaries and other data structures.""" - from collections import deque from functools import reduce from typing import Union diff --git a/milabench/scripts/vcs.py b/milabench/scripts/vcs.py index f1a8c4ddf..0f895f886 100644 --- a/milabench/scripts/vcs.py +++ b/milabench/scripts/vcs.py @@ -1,5 +1,6 @@ """Use to retrieve GIT version info, this file cannot import milabench modules as it is executed as part of the installation process""" + import os import subprocess import warnings diff --git a/milabench/utils.py b/milabench/utils.py index 63cba6b2d..42d0adbef 100644 --- a/milabench/utils.py +++ b/milabench/utils.py @@ -118,7 +118,7 @@ def relativize(pth): pth = XPath(pth) if pth.is_absolute(): - return pth.relative_to(XPath(here.parent).absolute()) + return pth.relative_to(XPath(here.parent).absolute()) else: return pth diff --git a/scripts/interactive.sh b/scripts/interactive.sh index 6d25f6f86..60eea1fe5 100644 --- a/scripts/interactive.sh +++ b/scripts/interactive.sh @@ -7,6 +7,7 @@ export ENV="$SLURM_TMPDIR/env" export MILABENCH_SOURCE="$HOME/milabench" export BASE="$SLURM_TMPDIR/base" export ARCH="cuda" +export PYTHON=3.9 if [ ! -d "$ENV" ] && [ "$ENV" != "base" ] && [ ! -d "$CONDA_ENVS/$ENV" ]; then conda create --prefix $ENV python=$PYTHON -y