From 75280395efc6f38d6b79e19c186acc137be962a9 Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Wed, 30 Oct 2024 15:38:52 -0400 Subject: [PATCH] chore(hermetic-build): use secure base image hosted in docker.com (#3324) This PR switches the base images of all the Hermetic Build Docker image stages to publicly hosted docker images. The chosen images are mirrored by Airlock or are at least in process of incorporation. The choice of OS family is `alpine` for its security-oriented setup. 0 vulnerabilities were found in the public image scan reports ([example](https://hub.docker.com/layers/library/python/3.12.7-alpine3.20/images/sha256-f498302457ec11162f872199b92239c34e1fbcdbc391ff37a4959e820224aa98?context=explore)). Although several optimizations are possible (see https://github.com/googleapis/sdk-platform-java/pull/3196 and its intermediate commits), this PR is restricted to the minimal changes to have a secure base image. _Notes:_ * Since the UNIX tools are from FreeBSD, the `-d` flag in the `rm` command is not supported. This is why we removed its usage in the scripts * A few downstream checks are failing in other PRs as well. I raised https://github.com/googleapis/sdk-platform-java/issues/3325 to track this. --- .../library_generation.Dockerfile | 71 ++++++++++++------- .../library_generation/generate_library.sh | 4 +- .../owlbot/bin/entrypoint.sh | 2 +- .../tests/generate_library_unit_tests.py | 2 +- .../tests/generate_library_unit_tests.sh | 8 +-- 5 files changed, 54 insertions(+), 33 deletions(-) diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 9a5434bb33..e52b938c19 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -14,22 +14,44 @@ # install gapic-generator-java in a separate layer so we don't overload the image # with the transferred source code and jars -FROM gcr.io/cloud-devrel-public-resources/java21@sha256:2ceff5eeea72260258df56d42e44ed413e52ee421c1b77393c5f2c9c4d7c41da AS ggj-build + +# 3.9.9-eclipse-temurin-11-alpine +FROM docker.io/library/maven@sha256:006d25558f9d5244ed55b5d2bd8eaf34d883e447d0c4b940e67b9f44d21167bf AS ggj-build WORKDIR /sdk-platform-java COPY . . # {x-version-update-start:gapic-generator-java:current} -ENV DOCKER_GAPIC_GENERATOR_VERSION="2.49.1-SNAPSHOT" +ENV DOCKER_GAPIC_GENERATOR_VERSION="2.49.1-SNAPSHOT" # {x-version-update-end} RUN mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip RUN cp "/root/.m2/repository/com/google/api/gapic-generator-java/${DOCKER_GAPIC_GENERATOR_VERSION}/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar" \ "./gapic-generator-java.jar" -# build from the root of this repo: -FROM gcr.io/cloud-devrel-public-resources/python@sha256:9c5ea427632f195ad164054831968389d86fdde4a15abca651f3fcb2a71268cb +# alpine:3.20.3 +FROM docker.io/library/alpine@sha256:beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367effd16dc0d06d as glibc-compat + +RUN apk add git sudo +# This SHA is the latest known-to-work version of this binary compatibility tool +ARG GLIB_MUS_SHA=7717dd4dc26377dd9cedcc92b72ebf35f9e68a2d +WORKDIR /home + +# Install compatibility layer to run glibc-based programs (such as the +# grpc plugin). +# Alpine, by default, only supports musl-based binaries, and there is no public +# downloadable distribution of the grpc plugin that is Alpine (musl) compatible. +# This is one of the recommended approaches to ensure glibc-compatibility +# as per https://wiki.alpinelinux.org/wiki/Running_glibc_programs +RUN git clone https://gitlab.com/manoel-linux1/GlibMus-HQ.git +WORKDIR /home/GlibMus-HQ +# We lock the tool to the latest known-to-work version +RUN git checkout "${GLIB_MUS_SHA}" +RUN chmod a+x compile-x86_64-alpine-linux.sh +RUN sh compile-x86_64-alpine-linux.sh + +# python:3.12.7-alpine3.20 +FROM docker.io/library/python@sha256:38e179a0f0436c97ecc76bcd378d7293ab3ee79e4b8c440fdc7113670cb6e204 as final -SHELL [ "/bin/bash", "-c" ] ARG OWLBOT_CLI_COMMITTISH=38fe6f89a2339ee75c77739b31b371f601b01bb3 @@ -40,9 +62,24 @@ ENV HOME=/home ENV OS_ARCHITECTURE="linux-x86_64" # install OS tools -RUN apt-get update && apt-get install -y \ - unzip openjdk-17-jdk rsync maven jq \ - && apt-get clean +RUN apk update && apk add unzip curl rsync openjdk11 jq bash nodejs npm git + +SHELL [ "/bin/bash", "-c" ] + +# Copy glibc shared objects to enable execution of the grpc plugin. +# This list was obtained via `libtree -pvvv /grpc/*` in the final container as +# well as inspecting the modifications done by compile-x86_64-alpine-linux.sh +# in the glibc-compat stage using the `dive` command. +COPY --from=glibc-compat /etc/libgcc* /etc/ +COPY --from=glibc-compat /lib64/ld-linux-x86-64.so.2 /lib64/ +COPY --from=glibc-compat /lib/GLIBCFAKE.so.0 /lib/ +COPY --from=glibc-compat /lib/ld-linux-x86-64.so.2 /lib/ +COPY --from=glibc-compat /lib/libpthread* /lib/ +COPY --from=glibc-compat /lib/libucontext* /lib/ +COPY --from=glibc-compat /lib/libc.* /lib/ +COPY --from=glibc-compat /usr/lib/libgcc* /usr/lib/ +COPY --from=glibc-compat /usr/lib/libstdc* /usr/lib/ +COPY --from=glibc-compat /usr/lib/libobstack* /usr/lib/ # copy source code COPY hermetic_build/common /src/common @@ -72,10 +109,6 @@ ENV DOCKER_GRPC_VERSION="${GRPC_VERSION}" COPY --from=ggj-build "/sdk-platform-java/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar" RUN chmod 755 "${HOME}/.library_generation/gapic-generator-java.jar" -# use python 3.12 (the base image has several python versions; here we define the default one) -RUN rm $(which python3) -RUN ln -s $(which python3.12) /usr/local/bin/python -RUN ln -s $(which python3.12) /usr/local/bin/python3 RUN python -m pip install --upgrade pip # install main scripts as a python package @@ -85,16 +118,6 @@ RUN python -m pip install src/common RUN python -m pip install --require-hashes -r src/library_generation/requirements.txt RUN python -m pip install src/library_generation -# Install nvm with node and npm -ENV NODE_VERSION 20.12.0 -WORKDIR /home -RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash -RUN chmod o+rx /home/.nvm -ENV NODE_PATH=/home/.nvm/versions/node/v${NODE_VERSION}/bin -ENV PATH=${PATH}:${NODE_PATH} -RUN node --version -RUN npm --version - # install the owl-bot CLI WORKDIR /tools RUN git clone https://github.com/googleapis/repo-automation-bots @@ -102,8 +125,7 @@ WORKDIR /tools/repo-automation-bots/packages/owl-bot RUN git checkout "${OWLBOT_CLI_COMMITTISH}" RUN npm i && npm run compile && npm link RUN owl-bot copy-code --version -RUN chmod -R o+rx ${NODE_PATH} -RUN ln -sf ${NODE_PATH}/* /usr/local/bin +RUN chmod o+rx $(which owl-bot) # download the Java formatter ADD https://maven-central.storage-download.googleapis.com/maven2/com/google/googlejavaformat/google-java-format/${JAVA_FORMAT_VERSION}/google-java-format-${JAVA_FORMAT_VERSION}-all-deps.jar \ @@ -120,7 +142,6 @@ RUN git config --system user.name "Cloud Java Bot" # allow read-write for /home and execution for binaries in /home/.nvm RUN chmod -R a+rw /home -RUN chmod -R a+rx /home/.nvm WORKDIR /workspace ENTRYPOINT [ "python", "/src/library_generation/cli/entry_point.py", "generate" ] diff --git a/hermetic_build/library_generation/generate_library.sh b/hermetic_build/library_generation/generate_library.sh index b8d22aca54..313b8243f8 100755 --- a/hermetic_build/library_generation/generate_library.sh +++ b/hermetic_build/library_generation/generate_library.sh @@ -119,7 +119,7 @@ temp_destination_path="${output_folder}/temp_preprocessed" mkdir -p "${output_folder}/${destination_path}" if [ -d "${temp_destination_path}" ]; then # we don't want the preprocessed sources of a previous run - rm -rd "${temp_destination_path}" + rm -r "${temp_destination_path}" fi mkdir -p "${temp_destination_path}" ##################### Section 0 ##################### @@ -274,5 +274,5 @@ rm -rf java_gapic_srcjar java_gapic_srcjar_raw.srcjar.zip java_grpc.jar java_pro popd # destination path cp -r ${temp_destination_path}/* "${output_folder}/${destination_path}" -rm -rdf "${temp_destination_path}" +rm -rf "${temp_destination_path}" exit 0 diff --git a/hermetic_build/library_generation/owlbot/bin/entrypoint.sh b/hermetic_build/library_generation/owlbot/bin/entrypoint.sh index 3d38677678..3118f32a2b 100755 --- a/hermetic_build/library_generation/owlbot/bin/entrypoint.sh +++ b/hermetic_build/library_generation/owlbot/bin/entrypoint.sh @@ -36,7 +36,7 @@ library_version=$5 if [[ "${is_monorepo}" == "true" ]]; then mv owl-bot-staging/* temp - rm -rd owl-bot-staging/ + rm -rf owl-bot-staging/ mv temp owl-bot-staging fi diff --git a/hermetic_build/library_generation/tests/generate_library_unit_tests.py b/hermetic_build/library_generation/tests/generate_library_unit_tests.py index 7bc14e3e20..eb2249908c 100644 --- a/hermetic_build/library_generation/tests/generate_library_unit_tests.py +++ b/hermetic_build/library_generation/tests/generate_library_unit_tests.py @@ -53,7 +53,7 @@ def setUp(self): bash_call(f"mkdir {self.output_folder}") def tearDown(self): - bash_call(f"rm -rdf {self.simulated_home}") + bash_call(f"rm -rf {self.simulated_home}") def _run_command(self, command, **kwargs): env = os.environ.copy() diff --git a/hermetic_build/library_generation/tests/generate_library_unit_tests.sh b/hermetic_build/library_generation/tests/generate_library_unit_tests.sh index 639abd8677..a3d95c1be7 100755 --- a/hermetic_build/library_generation/tests/generate_library_unit_tests.sh +++ b/hermetic_build/library_generation/tests/generate_library_unit_tests.sh @@ -138,7 +138,7 @@ download_tools_succeed_with_baked_protoc() { download_tools "99.99" "${test_grpc_version}" "linux-x86_64" assertEquals "${protoc_bin_folder}" "${protoc_path}" - rm -rdf "${output_folder}" + rm -rf "${output_folder}" unset DOCKER_PROTOC_LOCATION unset DOCKER_PROTOC_VERSION unset output_folder @@ -159,7 +159,7 @@ download_tools_succeed_with_baked_grpc() { download_tools "${test_protoc_version}" "99.99" "linux-x86_64" assertEquals "${DOCKER_GRPC_LOCATION}" "${grpc_path}" - rm -rdf "${output_folder}" + rm -rf "${output_folder}" unset DOCKER_GRPC_LOCATION unset DOCKER_GRPC_VERSION unset output_folder @@ -243,7 +243,7 @@ copy_directory_if_exists_valid_folder_succeeds() { mkdir -p "${destination}" copy_directory_if_exists "${source_folder}" "gapic" "${destination}/copied-folder" n_matching_folders=$(ls "${destination}" | grep -e 'copied-folder' | wc -l) - rm -rdf "${destination}" + rm -rf "${destination}" assertEquals 1 ${n_matching_folders} } @@ -253,7 +253,7 @@ copy_directory_if_exists_invalid_folder_does_not_copy() { mkdir -p "${destination}" copy_directory_if_exists "${source_folder}" "gapic" "${destination}/copied-folder" n_matching_folders=$(ls "${destination}" | grep -e 'copied-folder' | wc -l) || res=$? - rm -rdf "${destination}" + rm -rf "${destination}" assertEquals 0 ${n_matching_folders} }