Merge branch 'main' into ctk-cuda-11-only

.github/workflows/build-and-publish-images.yml → .github/workflows/build-and-publish-images.yaml

@@ -32,7 +32,7 @@ jobs:
       matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
       fail-fast: false
     secrets: inherit
-    uses: ./.github/workflows/build-image.yml
+    uses: ./.github/workflows/build-image.yaml
     with:
       ARCHES: ${{ toJSON(matrix.ARCHES) }}
       CUDA_VER: ${{ matrix.CUDA_VER }}

.github/workflows/pr.yml → .github/workflows/pr.yaml

@@ -11,7 +11,7 @@ concurrency:
 
 jobs:
   build-images:
-    uses: ./.github/workflows/build-and-publish-images.yml
+    uses: ./.github/workflows/build-and-publish-images.yaml
     with:
       build_type: pull-request
     secrets: inherit

.github/workflows/push.yml → .github/workflows/push.yaml

@@ -20,7 +20,7 @@ concurrency:
 
 jobs:
   build-images:
-    uses: ./.github/workflows/build-and-publish-images.yml
+    uses: ./.github/workflows/build-and-publish-images.yaml
     with:
       build_type: branch
     secrets: inherit

ci-conda.Dockerfile

@@ -28,7 +28,8 @@ ENV SCCACHE_S3_NO_CREDENTIALS=false
 RUN \
     case "${LINUX_VER}" in \
       "ubuntu"*) \
-        apt-get update \
+        echo 'APT::Update::Error-Mode "any";' > /etc/apt/apt.conf.d/warnings-as-errors \
+        && apt-get update \
         && apt-get upgrade -y \
         && apt-get install -y --no-install-recommends \
           file \
@@ -109,6 +110,7 @@ RUN rapids-mamba-retry install -y \
     git \
     jq \
     "sccache==0.4.2" \
+    "python=${PYTHON_VERSION}.*=*_cpython" \
   && conda clean -aipty
 
 # Install codecov binary
@@ -142,7 +144,7 @@ RUN cat /tmp/condarc.tmpl | envsubst | tee /opt/conda/.condarc; \
 RUN /opt/conda/bin/git config --system --add safe.directory '*'
 
 # Install CI tools using pip
-RUN pip install "rapids-dependency-file-generator==1.*" \
+RUN pip install dunamai "rapids-dependency-file-generator==1.*" \
     && pip cache purge
 
 COPY --from=mikefarah/yq:4.35.1 /usr/bin/yq /usr/local/bin/yq

ci-wheel.Dockerfile

@@ -27,13 +27,34 @@ ENV PATH="/pyenv/bin:/pyenv/shims:$PATH"
 
 RUN case "${LINUX_VER}" in \
     "ubuntu"*) \
-        apt update -y && apt install -y jq build-essential software-properties-common wget gcc zlib1g-dev libbz2-dev libssl-dev libreadline-dev libsqlite3-dev libffi-dev curl git libncurses5-dev libnuma-dev openssh-client libcudnn8-dev zip libopenblas-dev liblapack-dev protobuf-compiler autoconf automake libtool cmake && rm -rf /var/lib/apt/lists/* \
-        && add-apt-repository ppa:git-core/ppa && add-apt-repository ppa:ubuntu-toolchain-r/test && apt update -y && apt install -y git gcc-9 g++-9 && add-apt-repository -r ppa:git-core/ppa && add-apt-repository -r ppa:ubuntu-toolchain-r/test \
+        echo 'APT::Update::Error-Mode "any";' > /etc/apt/apt.conf.d/warnings-as-errors \
+        && apt update -y \
+        && apt install -y \
+          jq build-essential software-properties-common wget gcc zlib1g-dev libbz2-dev \
+          libssl-dev libreadline-dev libsqlite3-dev libffi-dev curl git libncurses5-dev \
+          libnuma-dev openssh-client libcudnn8-dev zip libopenblas-dev liblapack-dev \
+          protobuf-compiler autoconf automake libtool cmake \
+        && add-apt-repository ppa:git-core/ppa \
+        && add-apt-repository ppa:ubuntu-toolchain-r/test \
+        && apt update -y \
+        && apt install -y git gcc-9 g++-9 \
+        && add-apt-repository -r ppa:git-core/ppa \
+        && add-apt-repository -r ppa:ubuntu-toolchain-r/test \
         && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90 --slave /usr/bin/g++ g++ /usr/bin/g++-9 --slave /usr/bin/gcov gcov /usr/bin/gcov-9 \
+        && rm -rf /var/lib/apt/lists/* \
       ;; \
     "centos"*) \
-        yum update --exclude=libnccl* -y && yum install -y epel-release wget gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel xz xz-devel libffi-devel curl git ncurses-devel numactl numactl-devel openssh-clients libcudnn8-devel zip blas-devel lapack-devel protobuf-compiler autoconf automake libtool centos-release-scl scl-utils cmake && yum clean all \
-        && yum remove -y git && yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm && yum install -y git jq devtoolset-11 && yum remove -y endpoint-repo \
+        yum update --exclude=libnccl* -y \
+        && yum install -y \
+          epel-release wget gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite \
+          sqlite-devel xz xz-devel libffi-devel curl git ncurses-devel numactl \
+          numactl-devel openssh-clients libcudnn8-devel zip blas-devel lapack-devel \
+          protobuf-compiler autoconf automake libtool centos-release-scl scl-utils cmake \
+        && yum remove -y git \
+        && yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm \
+        && yum install -y git jq devtoolset-11 \
+        && yum remove -y endpoint-repo \
+        && yum clean all \
         && echo -e ' \
         #!/bin/bash\n \
         source scl_source enable devtoolset-11\n \

citestwheel.Dockerfile

@@ -19,14 +19,23 @@ ARG DEBIAN_FRONTEND=noninteractive
 ENV PYENV_ROOT="/pyenv"
 ENV PATH="/pyenv/bin:/pyenv/shims:$PATH"
 
-RUN apt-get update \
-        && apt-get upgrade -y \
-        && apt-get install -y --no-install-recommends \
-        wget curl git jq ssh \
-        make build-essential libssl-dev zlib1g-dev \
-        libbz2-dev libreadline-dev libsqlite3-dev wget \
-        curl llvm libncursesw5-dev xz-utils tk-dev unzip \
-        libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev
+RUN <<EOF
+set -e
+echo 'APT::Update::Error-Mode "any";' > /etc/apt/apt.conf.d/warnings-as-errors
+apt-get update
+apt-get install -y software-properties-common
+# update git > 2.17
+add-apt-repository ppa:git-core/ppa -y
+apt-get update
+apt-get upgrade -y
+apt-get install -y --no-install-recommends \
+  wget curl git jq ssh \
+  make build-essential libssl-dev zlib1g-dev \
+  libbz2-dev libreadline-dev libsqlite3-dev wget \
+  curl llvm libncursesw5-dev xz-utils tk-dev unzip \
+  libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev
+rm -rf /var/cache/apt/archives /var/lib/apt/lists/*
+EOF
 
 # Install pyenv
 RUN curl https://pyenv.run | bash
@@ -49,9 +58,6 @@ RUN mkdir -p /aws_install && cd /aws_install && \
     cd / && \
     rm -rf /aws_install
 
-# update git > 2.17
-RUN grep '18.04' /etc/issue && bash -c "apt-get install -y software-properties-common && add-apt-repository ppa:git-core/ppa -y && apt-get update && apt-get install --upgrade -y git" || true;
-
 # Install latest gha-tools
 RUN wget https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - \
   | tar -xz -C /usr/local/bin