From a2c71c5405fdd8822956bcd785e72149c1cfb655 Mon Sep 17 00:00:00 2001
From: Daniele <36171005+dtrifiro@users.noreply.github.com>
Date: Thu, 17 Oct 2024 19:25:06 +0200
Subject: [PATCH] [CI/Build] remove .github from .dockerignore, add dirty repo
 check (#9375)

---
 .buildkite/release-pipeline.yaml   |  4 ++--
 .dockerignore                      |  1 -
 .github/workflows/scripts/build.sh |  4 ++++
 Dockerfile                         |  4 +++-
 Dockerfile.cpu                     |  5 ++++-
 Dockerfile.neuron                  | 14 +++++++-------
 Dockerfile.openvino                |  3 +++
 Dockerfile.ppc64le                 |  3 +++
 Dockerfile.rocm                    |  3 +++
 Dockerfile.tpu                     | 11 +++++++----
 Dockerfile.xpu                     |  5 ++++-
 tools/check_repo.sh                | 14 ++++++++++++++
 12 files changed, 54 insertions(+), 17 deletions(-)
 create mode 100644 tools/check_repo.sh

diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml
index 98592ea7948f2..3b7fa0f2d94b3 100644
--- a/.buildkite/release-pipeline.yaml
+++ b/.buildkite/release-pipeline.yaml
@@ -3,7 +3,7 @@ steps:
     agents:
       queue: cpu_queue
     commands:
-      - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg CUDA_VERSION=12.1.0 --tag vllm-ci:build-image --target build --progress plain ."
+      - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.1.0 --tag vllm-ci:build-image --target build --progress plain ."
       - "mkdir artifacts"
       - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
       # rename the files to change linux -> manylinux1
@@ -22,7 +22,7 @@ steps:
     agents:
       queue: cpu_queue
     commands:
-      - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg CUDA_VERSION=11.8.0 --tag vllm-ci:build-image --target build --progress plain ."
+      - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=11.8.0 --tag vllm-ci:build-image --target build --progress plain ."
       - "mkdir artifacts"
       - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
       # rename the files to change linux -> manylinux1
diff --git a/.dockerignore b/.dockerignore
index 575f087f3ef6f..3863656915d03 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,4 +1,3 @@
-/.github/
 /.venv
 /build
 dist
diff --git a/.github/workflows/scripts/build.sh b/.github/workflows/scripts/build.sh
index 9e0a698990b3b..122e4e101e201 100644
--- a/.github/workflows/scripts/build.sh
+++ b/.github/workflows/scripts/build.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+set -eux
 
 python_executable=python$1
 cuda_home=/usr/local/cuda-$2
@@ -15,5 +16,8 @@ export MAX_JOBS=1
 # Make sure release wheels are built for the following architectures
 export TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6 8.9 9.0+PTX"
 export VLLM_FA_CMAKE_GPU_ARCHES="80-real;90-real"
+
+bash tools/check_repo.sh
+
 # Build
 $python_executable setup.py bdist_wheel --dist-dir=dist
diff --git a/Dockerfile b/Dockerfile
index d527868bc4c2f..0a562253c537b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -70,8 +70,10 @@ COPY requirements-build.txt requirements-build.txt
 RUN --mount=type=cache,target=/root/.cache/pip \
     python3 -m pip install -r requirements-build.txt
 
-# files and directories related to build wheels
 COPY . .
+ARG GIT_REPO_CHECK=0
+RUN --mount=type=bind,source=.git,target=.git \
+    if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
 
 # max jobs used by Ninja to build extensions
 ARG max_jobs=2
diff --git a/Dockerfile.cpu b/Dockerfile.cpu
index 2e7d66e7d8ffa..f1a21d6bd13fc 100644
--- a/Dockerfile.cpu
+++ b/Dockerfile.cpu
@@ -42,7 +42,10 @@ RUN --mount=type=cache,target=/root/.cache/pip \
     --mount=type=bind,src=requirements-cpu.txt,target=requirements-cpu.txt \
     pip install -v -r requirements-cpu.txt
 
-COPY ./ ./
+COPY . .
+ARG GIT_REPO_CHECK=0
+RUN --mount=type=bind,source=.git,target=.git \
+    if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
 
 # Support for building with non-AVX512 vLLM: docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" ...
 ARG VLLM_CPU_DISABLE_AVX512
diff --git a/Dockerfile.neuron b/Dockerfile.neuron
index adae6db87ba87..3d9d8e7da487c 100644
--- a/Dockerfile.neuron
+++ b/Dockerfile.neuron
@@ -17,7 +17,7 @@ RUN apt-get update && \
 # When launching the container, mount the code directory to /app
 ARG APP_MOUNT=/app
 VOLUME [ ${APP_MOUNT} ]
-WORKDIR ${APP_MOUNT}
+WORKDIR ${APP_MOUNT}/vllm
 
 RUN python3 -m pip install --upgrade pip
 RUN python3 -m pip install --no-cache-dir fastapi ninja tokenizers pandas
@@ -25,17 +25,17 @@ RUN python3 -m pip install sentencepiece transformers==4.36.2 -U
 RUN python3 -m pip install transformers-neuronx --extra-index-url=https://pip.repos.neuron.amazonaws.com -U
 RUN python3 -m pip install --pre neuronx-cc==2.15.* --extra-index-url=https://pip.repos.neuron.amazonaws.com -U
 
-COPY . /app/vllm
+COPY . .
+ARG GIT_REPO_CHECK=0
+RUN --mount=type=bind,source=.git,target=.git \
+    if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
 
-RUN cd /app/vllm \
-    && python3 -m pip install -U \
+RUN python3 -m pip install -U \
         cmake>=3.26 ninja packaging setuptools-scm>=8 wheel jinja2 \
         -r requirements-neuron.txt
 
 ENV VLLM_TARGET_DEVICE neuron
 RUN --mount=type=bind,source=.git,target=.git \
-    cd /app/vllm \
-    && pip install --no-build-isolation -v -e . \
-    && cd ..
+    pip install --no-build-isolation -v -e . \
 
 CMD ["/bin/bash"]
diff --git a/Dockerfile.openvino b/Dockerfile.openvino
index d65bfa08ccd90..c89864da91180 100644
--- a/Dockerfile.openvino
+++ b/Dockerfile.openvino
@@ -10,6 +10,9 @@ RUN apt-get update -y && \
 WORKDIR /workspace
 
 COPY . .
+ARG GIT_REPO_CHECK=0
+RUN --mount=type=bind,source=.git,target=.git \
+    if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
 
 # install build requirements
 RUN PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu" python3 -m pip install -r /workspace/vllm/requirements-build.txt
diff --git a/Dockerfile.ppc64le b/Dockerfile.ppc64le
index 1f374b01b9bc0..a84e00fd5677f 100644
--- a/Dockerfile.ppc64le
+++ b/Dockerfile.ppc64le
@@ -14,6 +14,9 @@ RUN micromamba install -y -n base -c https://ftp.osuosl.org/pub/open-ce/1.11.0-p
 COPY ./ /workspace/vllm
 
 WORKDIR /workspace/vllm
+ARG GIT_REPO_CHECK=0
+RUN --mount=type=bind,source=.git,target=.git \
+    if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh; fi
 
 # These packages will be in rocketce eventually
 RUN --mount=type=cache,target=/root/.cache/pip  \
diff --git a/Dockerfile.rocm b/Dockerfile.rocm
index 496e6bed7c022..d35889f053e27 100644
--- a/Dockerfile.rocm
+++ b/Dockerfile.rocm
@@ -117,6 +117,9 @@ RUN --mount=type=cache,target=${CCACHE_DIR} \
 FROM base AS final
 # Import the vLLM development directory from the build context
 COPY . .
+ARG GIT_REPO_CHECK=0
+RUN --mount=type=bind,source=.git,target=.git \
+    if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
 
 # Package upgrades for useful functionality or to avoid dependency issues
 RUN --mount=type=cache,target=/root/.cache/pip \
diff --git a/Dockerfile.tpu b/Dockerfile.tpu
index d8f1a42c45177..bdfab3f61910f 100644
--- a/Dockerfile.tpu
+++ b/Dockerfile.tpu
@@ -2,7 +2,7 @@ ARG NIGHTLY_DATE="20240828"
 ARG BASE_IMAGE="us-central1-docker.pkg.dev/tpu-pytorch-releases/docker/xla:nightly_3.10_tpuvm_$NIGHTLY_DATE"
 
 FROM $BASE_IMAGE
-WORKDIR /workspace
+WORKDIR /workspace/vllm
 
 # Install some basic utilities
 RUN apt-get update && apt-get install -y \
@@ -16,14 +16,17 @@ RUN --mount=type=cache,target=/root/.cache/pip \
     python3 -m pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
 
 # Build vLLM.
-COPY . /workspace/vllm
+COPY . .
+ARG GIT_REPO_CHECK=0
+RUN --mount=type=bind,source=.git,target=.git \
+    if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh; fi
+
 ENV VLLM_TARGET_DEVICE="tpu"
 RUN --mount=type=cache,target=/root/.cache/pip \
     --mount=type=bind,source=.git,target=.git \
-     cd /workspace/vllm && \
     python3 -m pip install \
         cmake>=3.26 ninja packaging setuptools-scm>=8 wheel jinja2 \
         -r requirements-tpu.txt
-RUN cd /workspace/vllm && python3 setup.py develop
+RUN python3 setup.py develop
 
 CMD ["/bin/bash"]
diff --git a/Dockerfile.xpu b/Dockerfile.xpu
index 83db341556eaf..0ecb46df6256c 100644
--- a/Dockerfile.xpu
+++ b/Dockerfile.xpu
@@ -33,7 +33,10 @@ RUN --mount=type=cache,target=/root/.cache/pip \
     --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ \
     -r requirements-xpu.txt
 
-COPY ./ /workspace/vllm
+COPY . .
+ARG GIT_REPO_CHECK
+RUN --mount=type=bind,source=.git,target=.git \
+    if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh; fi
 
 ENV VLLM_TARGET_DEVICE=xpu
 
diff --git a/tools/check_repo.sh b/tools/check_repo.sh
new file mode 100644
index 0000000000000..48eba5bea836f
--- /dev/null
+++ b/tools/check_repo.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+# Checks whether the repo is clean and whether tags are available (necessary to correctly produce vllm version at build time)
+
+if ! git diff --quiet; then
+	echo "Repo is dirty" >&2
+
+	exit 1
+fi
+
+if ! git describe --tags; then
+	echo "No tags are present. Is this a shallow clone? git fetch --unshallow --tags" >&2
+
+	exit 1
+fi