From 236049ea5767401cd0cc3594c3e9dadbd1bc205c Mon Sep 17 00:00:00 2001 From: Gregory Shtrasberg Date: Wed, 11 Dec 2024 11:53:56 -0600 Subject: [PATCH] Storing the installed commit hashes and customizations in a file --- Dockerfile.base | 64 +++++++++++++++++++++++++++++++++++--------- Dockerfile.base_navi | 58 ++++++++++++++++++++++++++++++--------- 2 files changed, 98 insertions(+), 24 deletions(-) diff --git a/Dockerfile.base b/Dockerfile.base index fa7200b947fd0..c32677ed80b42 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -1,4 +1,17 @@ ARG BASE_IMAGE=rocm/dev-ubuntu-22.04:6.3-complete +ARG HIPBLASLT_BRANCH="507a649" +ARG LEGACY_HIPBLASLT_OPTION= +ARG RCCL_BRANCH="dfe4a3e" +ARG RCCL_REPO="https://github.com/ROCm/rccl" +ARG TRITON_BRANCH="e192dba" +ARG TRITON_REPO="https://github.com/triton-lang/triton.git" +ARG PYTORCH_BRANCH="8bc4033" +ARG PYTORCH_VISION_BRANCH="v0.19.1" +ARG PYTORCH_REPO="https://github.com/pytorch/pytorch.git" +ARG PYTORCH_VISION_REPO="https://github.com/pytorch/vision.git" +ARG FA_BRANCH="c555642" +ARG FA_REPO="https://github.com/ROCm/flash-attention.git" + FROM ${BASE_IMAGE} AS base ENV PATH=/opt/rocm/llvm/bin:$PATH @@ -29,9 +42,9 @@ RUN apt-get update -y \ RUN pip install -U packaging cmake ninja wheel setuptools pybind11 Cython FROM base AS build_hipblaslt -ARG HIPBLASLT_BRANCH="507a649" +ARG HIPBLASLT_BRANCH # Set to "--legacy_hipblas_direct" for ROCm<=6.2 -ARG LEGACY_HIPBLASLT_OPTION= +ARG LEGACY_HIPBLASLT_OPTION RUN git clone https://github.com/ROCm/hipBLASLt RUN cd hipBLASLt \ && git checkout ${HIPBLASLT_BRANCH} \ @@ -41,8 +54,8 @@ RUN cd hipBLASLt \ RUN mkdir -p /app/install && cp /app/hipBLASLt/build/release/*.deb /app/install FROM base AS build_rccl -ARG RCCL_BRANCH="dfe4a3e" -ARG RCCL_REPO="https://github.com/ROCm/rccl" +ARG RCCL_BRANCH +ARG RCCL_REPO RUN git clone ${RCCL_REPO} RUN cd rccl \ && git checkout ${RCCL_BRANCH} \ @@ -50,8 +63,8 @@ RUN cd rccl \ RUN mkdir -p /app/install && cp /app/rccl/build/release/*.deb /app/install FROM base AS build_triton -ARG TRITON_BRANCH="e192dba" -ARG TRITON_REPO="https://github.com/triton-lang/triton.git" +ARG TRITON_BRANCH +ARG TRITON_REPO RUN git clone ${TRITON_REPO} RUN cd triton \ && git checkout ${TRITON_BRANCH} \ @@ -65,12 +78,12 @@ RUN cd /opt/rocm/share/amd_smi \ RUN mkdir -p /app/install && cp /opt/rocm/share/amd_smi/dist/*.whl /app/install FROM base AS build_pytorch -ARG PYTORCH_BRANCH="8bc4033" -ARG PYTORCH_VISION_BRANCH="v0.19.1" -ARG PYTORCH_REPO="https://github.com/pytorch/pytorch.git" -ARG PYTORCH_VISION_REPO="https://github.com/pytorch/vision.git" -ARG FA_BRANCH="c555642" -ARG FA_REPO="https://github.com/ROCm/flash-attention.git" +ARG PYTORCH_BRANCH +ARG PYTORCH_VISION_BRANCH +ARG PYTORCH_REPO +ARG PYTORCH_VISION_REPO +ARG FA_BRANCH +ARG FA_REPO RUN git clone ${PYTORCH_REPO} pytorch RUN cd pytorch && git checkout ${PYTORCH_BRANCH} && \ pip install -r requirements.txt && git submodule update --init --recursive \ @@ -105,3 +118,30 @@ RUN --mount=type=bind,from=build_amdsmi,src=/app/install/,target=/install \ pip install /install/*.whl RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \ pip install /install/*.whl + +ARG BASE_IMAGE +ARG HIPBLASLT_BRANCH +ARG LEGACY_HIPBLASLT_OPTION +ARG RCCL_BRANCH +ARG RCCL_REPO +ARG TRITON_BRANCH +ARG TRITON_REPO +ARG PYTORCH_BRANCH +ARG PYTORCH_VISION_BRANCH +ARG PYTORCH_REPO +ARG PYTORCH_VISION_REPO +ARG FA_BRANCH +ARG FA_REPO +RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \ + && echo "HIPBLASLT_BRANCH: ${HIPBLASLT_BRANCH}" >> /app/versions.txt \ + && echo "LEGACY_HIPBLASLT_OPTION: ${LEGACY_HIPBLASLT_OPTION}" >> /app/versions.txt \ + && echo "RCCL_BRANCH: ${RCCL_BRANCH}" >> /app/versions.txt \ + && echo "RCCL_REPO: ${RCCL_REPO}" >> /app/versions.txt \ + && echo "TRITON_BRANCH: ${TRITON_BRANCH}" >> /app/versions.txt \ + && echo "TRITON_REPO: ${TRITON_REPO}" >> /app/versions.txt \ + && echo "PYTORCH_BRANCH: ${PYTORCH_BRANCH}" >> /app/versions.txt \ + && echo "PYTORCH_VISION_BRANCH: ${PYTORCH_VISION_BRANCH}" >> /app/versions.txt \ + && echo "PYTORCH_REPO: ${PYTORCH_REPO}" >> /app/versions.txt \ + && echo "PYTORCH_VISION_REPO: ${PYTORCH_VISION_REPO}" >> /app/versions.txt \ + && echo "FA_BRANCH: ${FA_BRANCH}" >> /app/versions.txt \ + && echo "FA_REPO: ${FA_REPO}" >> /app/versions.txt \ No newline at end of file diff --git a/Dockerfile.base_navi b/Dockerfile.base_navi index d9449843f1bfd..4041024189f5d 100644 --- a/Dockerfile.base_navi +++ b/Dockerfile.base_navi @@ -1,4 +1,15 @@ ARG BASE_IMAGE=rocm/dev-ubuntu-22.04:6.3-complete +ARG HIPBLASLT_BRANCH="d43d84a" +ARG LEGACY_HIPBLASLT_OPTION= +ARG RCCL_BRANCH="dfe4a3e" +ARG RCCL_REPO="https://github.com/ROCm/rccl" +ARG TRITON_BRANCH="e192dba" +ARG TRITON_REPO="https://github.com/triton-lang/triton.git" +ARG PYTORCH_BRANCH="8bc4033" +ARG PYTORCH_VISION_BRANCH="v0.19.1" +ARG PYTORCH_REPO="https://github.com/pytorch/pytorch.git" +ARG PYTORCH_VISION_REPO="https://github.com/pytorch/vision.git" + FROM ${BASE_IMAGE} AS base ENV PATH=/opt/rocm/llvm/bin:$PATH @@ -26,12 +37,12 @@ RUN apt-get update -y \ && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \ && python3 --version && python3 -m pip --version -RUN pip install -U packaging cmake ninja wheel setuptools Cython +RUN pip install -U packaging cmake ninja wheel setuptools Cython pybind11 FROM base AS build_hipblaslt -ARG HIPBLASLT_BRANCH="d43d84a" +ARG HIPBLASLT_BRANCH # Set to "--legacy_hipblas_direct" for ROCm<=6.2 -ARG LEGACY_HIPBLASLT_OPTION= +ARG LEGACY_HIPBLASLT_OPTION RUN git clone https://github.com/ROCm/hipBLASLt RUN cd hipBLASLt \ && git checkout ${HIPBLASLT_BRANCH} \ @@ -41,8 +52,8 @@ RUN cd hipBLASLt \ RUN mkdir -p /app/install && cp /app/hipBLASLt/build/release/*.deb /app/install FROM base AS build_rccl -ARG RCCL_BRANCH="dfe4a3e" -ARG RCCL_REPO="https://github.com/ROCm/rccl" +ARG RCCL_BRANCH +ARG RCCL_REPO RUN git clone ${RCCL_REPO} RUN cd rccl \ && git checkout ${RCCL_BRANCH} \ @@ -50,8 +61,8 @@ RUN cd rccl \ RUN mkdir -p /app/install && cp /app/rccl/build/release/*.deb /app/install FROM base AS build_triton -ARG TRITON_BRANCH="release/3.1.x" -ARG TRITON_REPO="https://github.com/triton-lang/triton.git" +ARG TRITON_BRANCH +ARG TRITON_REPO RUN git clone ${TRITON_REPO} RUN cd triton \ && git checkout ${TRITON_BRANCH} \ @@ -65,10 +76,10 @@ RUN cd /opt/rocm/share/amd_smi \ RUN mkdir -p /app/install && cp /opt/rocm/share/amd_smi/dist/*.whl /app/install FROM base AS build_pytorch -ARG PYTORCH_BRANCH="8bc4033" -ARG PYTORCH_VISION_BRANCH="v0.19.1" -ARG PYTORCH_REPO="https://github.com/pytorch/pytorch.git" -ARG PYTORCH_VISION_REPO="https://github.com/pytorch/vision.git" +ARG PYTORCH_BRANCH +ARG PYTORCH_VISION_BRANCH +ARG PYTORCH_REPO +ARG PYTORCH_VISION_REPO RUN git clone ${PYTORCH_REPO} pytorch RUN cd pytorch && git checkout ${PYTORCH_BRANCH} && \ pip install -r requirements.txt && git submodule update --init --recursive \ @@ -96,4 +107,27 @@ RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \ RUN --mount=type=bind,from=build_amdsmi,src=/app/install/,target=/install \ pip install /install/*.whl RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \ - pip install /install/*.whl \ No newline at end of file + pip install /install/*.whl + +ARG BASE_IMAGE +ARG HIPBLASLT_BRANCH +ARG LEGACY_HIPBLASLT_OPTION +ARG RCCL_BRANCH +ARG RCCL_REPO +ARG TRITON_BRANCH +ARG TRITON_REPO +ARG PYTORCH_BRANCH +ARG PYTORCH_VISION_BRANCH +ARG PYTORCH_REPO +ARG PYTORCH_VISION_REPO +RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \ + && echo "HIPBLASLT_BRANCH: ${HIPBLASLT_BRANCH}" >> /app/versions.txt \ + && echo "LEGACY_HIPBLASLT_OPTION: ${LEGACY_HIPBLASLT_OPTION}" >> /app/versions.txt \ + && echo "RCCL_BRANCH: ${RCCL_BRANCH}" >> /app/versions.txt \ + && echo "RCCL_REPO: ${RCCL_REPO}" >> /app/versions.txt \ + && echo "TRITON_BRANCH: ${TRITON_BRANCH}" >> /app/versions.txt \ + && echo "TRITON_REPO: ${TRITON_REPO}" >> /app/versions.txt \ + && echo "PYTORCH_BRANCH: ${PYTORCH_BRANCH}" >> /app/versions.txt \ + && echo "PYTORCH_VISION_BRANCH: ${PYTORCH_VISION_BRANCH}" >> /app/versions.txt \ + && echo "PYTORCH_REPO: ${PYTORCH_REPO}" >> /app/versions.txt \ + && echo "PYTORCH_VISION_REPO: ${PYTORCH_VISION_REPO}" >> /app/versions.txt \ No newline at end of file