Skip to content

Commit

Permalink
Use latest Miniforge; specify versions as build args
Browse files Browse the repository at this point in the history
  • Loading branch information
hcho3 committed Dec 8, 2024
1 parent 8f111db commit 38e0c8e
Show file tree
Hide file tree
Showing 12 changed files with 120 additions and 111 deletions.
20 changes: 10 additions & 10 deletions containers/dockerfile/Dockerfile.aarch64
Original file line number Diff line number Diff line change
@@ -1,23 +1,23 @@
FROM quay.io/pypa/manylinux_2_28_aarch64
ARG MINIFORGE_VERSION=24.9.2-0

SHELL ["/bin/bash", "-c"] # Use Bash as shell
SHELL ["/bin/bash", "-c"]

ENV PATH=/opt/miniforge/bin:$PATH
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
ENV GOSU_VERSION=1.10

# Install all basic requirements
RUN \
dnf -y update && \
dnf -y install dnf-plugins-core && \
dnf config-manager --set-enabled powertools && \
dnf install -y tar unzip wget xz git which ninja-build gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ && \
# Python
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-aarch64.sh && \
# Miniforge
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/$MINIFORGE_VERSION/Mambaforge-$MINIFORGE_VERSION-Linux-aarch64.sh && \
bash conda.sh -b -p /opt/miniforge

ENV PATH=/opt/miniforge/bin:$PATH
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
ENV GOSU_VERSION=1.10

# Create new Conda environment
COPY conda_env/aarch64_test.yml /scripts/
RUN mamba create -n aarch64_test && \
Expand Down
16 changes: 4 additions & 12 deletions containers/dockerfile/Dockerfile.clang_tidy
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,15 @@ ARG CUDA_VERSION=notset
FROM nvidia/cuda:$CUDA_VERSION-devel-ubuntu22.04
ARG CUDA_VERSION

# Environment
ENV DEBIAN_FRONTEND=noninteractive
ENV GOSU_VERSION=1.10

# Install all basic requirements
RUN \
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub && \
apt-get update && \
apt-get install -y wget git python3 python3-pip software-properties-common \
apt-transport-https ca-certificates gnupg-agent && \
apt-get install -y ninja-build
apt-get install -y ninja-build libgtest-dev libgmock-dev

# Install clang-tidy: https://apt.llvm.org/
RUN \
Expand All @@ -23,18 +22,11 @@ RUN \
apt-get install -y clang-tidy-19 clang-19 libomp-19-dev

# Set default clang-tidy version
RUN \
update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-19 100 && \
RUN update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-19 100 && \
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-19 100

RUN \
apt-get install libgtest-dev libgmock-dev -y

# Install Python packages
RUN \
pip3 install cmake

ENV GOSU_VERSION=1.10
RUN pip3 install cmake

# Install lightweight sudo (not bound to TTY)
RUN set -ex; \
Expand Down
23 changes: 11 additions & 12 deletions containers/dockerfile/Dockerfile.cpu
Original file line number Diff line number Diff line change
@@ -1,28 +1,27 @@
FROM ubuntu:22.04
ARG MINIFORGE_VERSION=24.9.2-0

# Environment
ENV DEBIAN_FRONTEND=noninteractive
SHELL ["/bin/bash", "-c"]

ENV DEBIAN_FRONTEND=noninteractive
ENV PATH=/opt/miniforge/bin:$PATH
ENV CC=gcc-10
ENV CXX=g++-10
ENV CPP=cpp-10
ENV GOSU_VERSION=1.10
ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/

# Install all basic requirements
RUN \
apt-get update && \
apt-get install -y software-properties-common && \
add-apt-repository ppa:ubuntu-toolchain-r/test && \
apt-get update && \
apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libidn12 cmake ninja-build gcc-10 g++-10 openjdk-8-jdk-headless && \
# Python
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
# Miniforge
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/$MINIFORGE_VERSION/Mambaforge-$MINIFORGE_VERSION-Linux-x86_64.sh && \
bash conda.sh -b -p /opt/miniforge

ENV PATH=/opt/miniforge/bin:$PATH
ENV CC=gcc-10
ENV CXX=g++-10
ENV CPP=cpp-10

ENV GOSU_VERSION=1.10
ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/

# Install gRPC
# Patch Abseil to apply https://github.com/abseil/abseil-cpp/issues/1629
RUN git clone -b v1.65.4 https://github.com/grpc/grpc.git \
Expand Down
17 changes: 8 additions & 9 deletions containers/dockerfile/Dockerfile.gpu
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,25 @@ ARG RAPIDS_VERSION
# Should be first 4 digits (e.g. 24.06)
ARG NCCL_VERSION
ARG RAPIDSAI_CONDA_CHANNEL="rapidsai"
ARG MINIFORGE_VERSION=24.9.2-0

# Environment
ENV DEBIAN_FRONTEND=noninteractive
SHELL ["/bin/bash", "-c"]

ENV DEBIAN_FRONTEND=noninteractive
ENV PATH=/opt/miniforge/bin:$PATH
ENV GOSU_VERSION=1.10
ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/

# Install all basic requirements
RUN \
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub && \
apt-get update && \
apt-get install -y wget unzip bzip2 libgomp1 build-essential openjdk-8-jdk-headless && \
apt-get install libnccl2 libnccl-dev -y --allow-change-held-packages && \
# Python
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
# Miniforge
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/$MINIFORGE_VERSION/Mambaforge-$MINIFORGE_VERSION-Linux-x86_64.sh && \
bash conda.sh -b -p /opt/miniforge

ENV PATH=/opt/miniforge/bin:$PATH

# Create new Conda environment with cuDF, Dask, and cuPy
RUN \
export NCCL_SHORT_VER=$(echo "$NCCL_VERSION" | cut -d "-" -f 1) && \
Expand All @@ -37,9 +39,6 @@ RUN \
"pyspark>=3.4.0" cloudpickle cuda-python && \
mamba clean --all --yes

ENV GOSU_VERSION=1.10
ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/

# Install lightweight sudo (not bound to TTY)
RUN set -ex; \
wget -nv -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
Expand Down
40 changes: 20 additions & 20 deletions containers/dockerfile/Dockerfile.gpu_build_r_rockylinux8
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,19 @@ ARG CUDA_VERSION=notset
FROM nvcr.io/nvidia/cuda:$CUDA_VERSION-devel-rockylinux8
ARG CUDA_VERSION
ARG R_VERSION
ARG MINIFORGE_VERSION=24.9.2-0
ARG CMAKE_VERSION=3.31.2

SHELL ["/bin/bash", "-c"]

ENV PATH=/opt/miniforge/bin:/usr/local/ninja:/opt/software/packages/bin:/opt/R/$R_VERSION/bin:$PATH
ENV LD_LIBRARY_PATH=/opt/software/packages/lib:/opt/R/$R_VERSION/lib64:$LD_LIBRARY_PATH
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
ENV F77=/opt/rh/gcc-toolset-10/root/usr/bin/gfortran
ENV FC=/opt/rh/gcc-toolset-10/root/usr/bin/gfortran
ENV GOSU_VERSION=1.10

# Install all basic requirements
RUN \
Expand All @@ -15,15 +28,13 @@ RUN \
pcre2-devel libcurl-devel texlive-* \
gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ \
gcc-toolset-10-gcc-gfortran gcc-toolset-10-libquadmath-devel \
gcc-toolset-10-runtime gcc-toolset-10-libstdc++-devel

ENV PATH=/opt/miniforge/bin:/usr/local/ninja:/opt/software/packages/bin:/opt/R/$R_VERSION/bin:$PATH
ENV LD_LIBRARY_PATH=/opt/software/packages/lib:/opt/R/$R_VERSION/lib64:$LD_LIBRARY_PATH
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
ENV F77=/opt/rh/gcc-toolset-10/root/usr/bin/gfortran
ENV FC=/opt/rh/gcc-toolset-10/root/usr/bin/gfortran
gcc-toolset-10-runtime gcc-toolset-10-libstdc++-devel && \
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/$MINIFORGE_VERSION/Mambaforge-$MINIFORGE_VERSION-Linux-x86_64.sh && \
bash conda.sh -b -p /opt/miniforge && \
/opt/miniforge/bin/python -m pip install awscli && \
# CMake
wget -nv -O cmake.sh https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh && \
bash cmake.sh --skip-license --prefix=/usr

RUN \
wget -nv -nc https://cran.r-project.org/src/base/R-4/R-$R_VERSION.tar.gz && \
Expand All @@ -33,17 +44,6 @@ RUN \
make -j$(nproc) && \
make install

run \
# Python
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
bash conda.sh -b -p /opt/miniforge && \
/opt/miniforge/bin/python -m pip install auditwheel awscli && \
# CMake
wget -nv -nc https://cmake.org/files/v3.29/cmake-3.29.5-linux-x86_64.sh --no-check-certificate && \
bash cmake-3.29.5-linux-x86_64.sh --skip-license --prefix=/usr

ENV GOSU_VERSION=1.10

# Install lightweight sudo (not bound to TTY)
RUN set -ex; \
wget -nv -nc -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
Expand Down
27 changes: 15 additions & 12 deletions containers/dockerfile/Dockerfile.gpu_build_rockylinux8
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,17 @@ FROM nvcr.io/nvidia/cuda:$CUDA_VERSION-devel-rockylinux8
ARG CUDA_VERSION
ARG NCCL_VERSION
ARG RAPIDS_VERSION
ARG MINIFORGE_VERSION=24.9.2-0
ARG CMAKE_VERSION=3.31.2

SHELL ["/bin/bash", "-c"]

ENV PATH=/opt/miniforge/bin:/usr/local/ninja:$PATH
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
ENV CUDAHOSTCXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
ENV GOSU_VERSION=1.10

# Install all basic requirements
RUN \
Expand All @@ -12,13 +23,13 @@ RUN \
dnf -y install dnf-plugins-core && \
dnf config-manager --set-enabled powertools && \
dnf install -y tar unzip wget xz git which ninja-build gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ && \
# Python
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
# Miniforge
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/$MINIFORGE_VERSION/Mambaforge-$MINIFORGE_VERSION-Linux-x86_64.sh && \
bash conda.sh -b -p /opt/miniforge && \
/opt/miniforge/bin/python -m pip install awscli && \
# CMake
wget -nv -nc https://cmake.org/files/v3.29/cmake-3.29.5-linux-x86_64.sh --no-check-certificate && \
bash cmake-3.29.5-linux-x86_64.sh --skip-license --prefix=/usr
wget -nv -O cmake.sh https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh && \
bash cmake.sh --skip-license --prefix=/usr

# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
RUN \
Expand All @@ -28,14 +39,6 @@ RUN \
dnf -y update && \
dnf install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT}

ENV PATH=/opt/miniforge/bin:/usr/local/ninja:$PATH
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
ENV CUDAHOSTCXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++

ENV GOSU_VERSION=1.10

# Install gRPC
# Patch Abseil to apply https://github.com/abseil/abseil-cpp/issues/1629
RUN git clone -b v1.65.4 https://github.com/grpc/grpc.git \
Expand Down
6 changes: 3 additions & 3 deletions containers/dockerfile/Dockerfile.i386
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
FROM i386/debian:sid

ENV DEBIAN_FRONTEND=noninteractive
SHELL ["/bin/bash", "-c"]

RUN \
apt-get update && \
ENV DEBIAN_FRONTEND=noninteractive

RUN apt-get update && \
apt-get install -y tar unzip wget git build-essential ninja-build cmake
35 changes: 19 additions & 16 deletions containers/dockerfile/Dockerfile.jvm
Original file line number Diff line number Diff line change
@@ -1,34 +1,37 @@
FROM rockylinux:8
ARG MINIFORGE_VERSION=24.9.2-0
ARG CMAKE_VERSION=3.31.2
ARG MAVEN_VERSION=3.9.9

SHELL ["/bin/bash", "-c"]

ENV PATH=/opt/miniforge/bin:/opt/maven/bin:$PATH
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
ENV GOSU_VERSION=1.10

# Install all basic requirements
RUN \
dnf -y update && \
RUN dnf -y update && \
dnf -y install dnf-plugins-core && \
dnf config-manager --set-enabled powertools && \
dnf install -y tar unzip make bzip2 wget xz git which ninja-build java-1.8.0-openjdk-devel \
gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ \
gcc-toolset-10-runtime gcc-toolset-10-libstdc++-devel && \
# Python
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
# Miniforge
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/$MINIFORGE_VERSION/Mambaforge-$MINIFORGE_VERSION-Linux-x86_64.sh && \
bash conda.sh -b -p /opt/miniforge && \
# CMake
wget -nv -nc https://cmake.org/files/v3.29/cmake-3.29.5-linux-x86_64.sh --no-check-certificate && \
bash cmake-3.29.5-linux-x86_64.sh --skip-license --prefix=/usr && \
wget -nv -O cmake.sh https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh && \
bash cmake.sh --skip-license --prefix=/usr && \
# Maven
wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.9.7/binaries/apache-maven-3.9.7-bin.tar.gz && \
tar xvf apache-maven-3.9.7-bin.tar.gz -C /opt && \
ln -s /opt/apache-maven-3.9.7/ /opt/maven

ENV PATH=/opt/miniforge/bin:/opt/maven/bin:$PATH
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
wget -nv -O apache-maven.tar.gz https://dlcdn.apache.org/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz && \
tar xvf apache-maven.tar.gz -C /opt && \
ln -s /opt/apache-maven-${MAVEN_VERSION}/ /opt/maven

# Install Python packages
RUN pip install numpy pytest scipy scikit-learn wheel kubernetes awscli

ENV GOSU_VERSION=1.10

# Install lightweight sudo (not bound to TTY)
RUN set -ex; \
wget -nv -nc -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
Expand Down
32 changes: 18 additions & 14 deletions containers/dockerfile/Dockerfile.jvm_gpu_build
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,17 @@ ARG CUDA_VERSION=notset
FROM nvcr.io/nvidia/cuda:$CUDA_VERSION-devel-rockylinux8
ARG CUDA_VERSION
ARG NCCL_VERSION
ARG MINIFORGE_VERSION=24.9.2-0
ARG CMAKE_VERSION=3.31.2
ARG MAVEN_VERSION=3.9.9

SHELL ["/bin/bash", "-c"]

ENV PATH=/opt/miniforge/bin:/opt/maven/bin:$PATH
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp
ENV GOSU_VERSION=1.10

# Install all basic requirements
RUN \
Expand All @@ -11,16 +22,16 @@ RUN \
dnf -y install dnf-plugins-core && \
dnf config-manager --set-enabled powertools && \
dnf install -y tar unzip wget xz git which ninja-build java-1.8.0-openjdk-devel gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ && \
# Python
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-x86_64.sh && \
# Miniforge
wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/$MINIFORGE_VERSION/Mambaforge-$MINIFORGE_VERSION-Linux-x86_64.sh && \
bash conda.sh -b -p /opt/miniforge && \
# CMake
wget -nv -nc https://cmake.org/files/v3.29/cmake-3.29.5-linux-x86_64.sh --no-check-certificate && \
bash cmake-3.29.5-linux-x86_64.sh --skip-license --prefix=/usr && \
wget -nv -O cmake.sh https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh && \
bash cmake.sh --skip-license --prefix=/usr && \
# Maven
wget -nv -nc https://archive.apache.org/dist/maven/maven-3/3.9.7/binaries/apache-maven-3.9.7-bin.tar.gz && \
tar xvf apache-maven-3.9.7-bin.tar.gz -C /opt && \
ln -s /opt/apache-maven-3.9.7/ /opt/maven
wget -nv -O apache-maven.tar.gz https://dlcdn.apache.org/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz && \
tar xvf apache-maven.tar.gz -C /opt && \
ln -s /opt/apache-maven-${MAVEN_VERSION}/ /opt/maven

# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
RUN \
Expand All @@ -30,16 +41,9 @@ RUN \
dnf -y update && \
dnf install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-static-${NCCL_VERSION}+cuda${CUDA_SHORT}

ENV PATH=/opt/miniforge/bin:/opt/maven/bin:$PATH
ENV CC=/opt/rh/gcc-toolset-10/root/usr/bin/gcc
ENV CXX=/opt/rh/gcc-toolset-10/root/usr/bin/c++
ENV CPP=/opt/rh/gcc-toolset-10/root/usr/bin/cpp

# Install Python packages
RUN pip install numpy pytest scipy scikit-learn wheel kubernetes awscli

ENV GOSU_VERSION=1.10

# Install lightweight sudo (not bound to TTY)
RUN set -ex; \
wget -nv -nc -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
Expand Down
Loading

0 comments on commit 38e0c8e

Please sign in to comment.