From 3ef7dd54ea92578e7af26c8684812cea8cffcf27 Mon Sep 17 00:00:00 2001 From: anakinxc Date: Mon, 11 Dec 2023 16:30:13 +0800 Subject: [PATCH 1/6] Multiarch release ci --- dockerfiles/build_llvm.sh | 21 -------- dockerfiles/release-ci-aarch64.DockerFile | 65 ----------------------- dockerfiles/release-ci.DockerFile | 36 ++++++++----- 3 files changed, 24 insertions(+), 98 deletions(-) delete mode 100644 dockerfiles/build_llvm.sh delete mode 100644 dockerfiles/release-ci-aarch64.DockerFile diff --git a/dockerfiles/build_llvm.sh b/dockerfiles/build_llvm.sh deleted file mode 100644 index 75a7c57..0000000 --- a/dockerfiles/build_llvm.sh +++ /dev/null @@ -1,21 +0,0 @@ -#! /usr/bin/bash - -mkdir -pv build -cd build -cmake -DCMAKE_INSTALL_PREFIX=/usr/local \ - -DLLVM_ENABLE_PROJECTS="lld" \ - -DLLVM_INCLUDE_UTILS=OFF \ - -DLLVM_INCLUDE_DOCS=OFF \ - -DLLVM_INCLUDE_TESTS=OFF \ - -DLLVM_ENABLE_EH=ON \ - -DLLVM_ENABLE_Z3_SOLVER=OFF \ - -DLLVM_OPTIMIZED_TABLEGEN=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_TARGETS_TO_BUILD="host" \ - -DLLVM_INSTALL_TOOLCHAIN_ONLY=ON \ - -DLLVM_ENABLE_RTTI=ON \ - -DPython3_EXECUTABLE=/root/miniconda3/bin/python3 \ - -Wno-dev -G Ninja .. && - ninja install - -rm -r /usr/local/bin/llvm-* \ No newline at end of file diff --git a/dockerfiles/release-ci-aarch64.DockerFile b/dockerfiles/release-ci-aarch64.DockerFile deleted file mode 100644 index 773be59..0000000 --- a/dockerfiles/release-ci-aarch64.DockerFile +++ /dev/null @@ -1,65 +0,0 @@ -FROM centos:centos8 - -# GCC version -ARG DEVTOOLSET_VERSION=11 - -RUN cd /etc/yum.repos.d/ \ - && sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* \ - && sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - -RUN yum update -y \ - && yum install -y dnf-plugins-core \ - && yum clean all - -RUN yum config-manager --set-enabled powertools - -# install devtools and [enable it](https://access.redhat.com/solutions/527703) -RUN yum install -y \ - gcc-toolset-${DEVTOOLSET_VERSION}-gcc \ - gcc-toolset-${DEVTOOLSET_VERSION}-gcc-c++ \ - gcc-toolset-${DEVTOOLSET_VERSION}-binutils \ - git vim-common wget unzip which java-11-openjdk-devel.aarch64 \ - libtool autoconf make ninja-build \ - && yum clean all \ - && echo "source scl_source enable gcc-toolset-${DEVTOOLSET_VERSION}" > /etc/profile.d/enable_gcc_toolset.sh - -ENV PATH="/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/bin:${PATH}" - -RUN wget https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-aarch64.sh \ - && sh cmake-3.26.3-linux-aarch64.sh --prefix=/usr --skip-license \ - && rm -f cmake-3.26.3-linux-aarch64.sh - -# install conda -RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh \ - && bash Miniconda3-latest-Linux-aarch64.sh -b \ - && rm -f Miniconda3-latest-Linux-aarch64.sh \ - && /root/miniconda3/bin/conda init bash - -# install go -ARG GO_VERSION=1.19.10 -ARG GO_SHA256SUM="df98698821211c819e8b2420c77a0f802d989e377718578a31b1f91f6be2c5b4" - -# install go -RUN set -eux; \ - url="https://golang.google.cn/dl/go${GO_VERSION}.linux-arm64.tar.gz"; \ - wget --no-check-certificate -O go.tgz "$url"; \ - echo "${GO_SHA256SUM} *go.tgz" | sha256sum -c -; \ - tar -C /usr/local -xzf go.tgz; \ - rm go.tgz; - -ENV GOPATH="/usr/local" -ENV PATH="/usr/local/go/bin:${GOPATH}/bin:${PATH}" - -# install bazel -# RUN wget https://github.com/bazelbuild/bazel/releases/download/5.4.1/bazel-5.4.1-linux-arm64 \ -RUN wget https://mirrors.huaweicloud.com/bazel/5.4.1/bazel-5.4.1-linux-arm64 \ - && mv bazel-5.4.1-linux-arm64 /usr/bin/bazel \ - && chmod +x /usr/bin/bazel - -# Add conda to path -ENV PATH="/root/miniconda3/bin:${PATH}" - -# run as root for now -WORKDIR /home/admin/ - -ENTRYPOINT [ "/bin/bash", "-l" ] diff --git a/dockerfiles/release-ci.DockerFile b/dockerfiles/release-ci.DockerFile index f44829e..ce0872d 100644 --- a/dockerfiles/release-ci.DockerFile +++ b/dockerfiles/release-ci.DockerFile @@ -1,20 +1,34 @@ -FROM centos:centos7 +FROM --platform=linux/amd64 centos:centos7 +FROM --platform=linux/arm64 centos:centos8 -RUN yum install -y centos-release-scl epel-release \ +ARG TARGETPLATFORM + +RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then yum install -y centos-release-scl epel-release \ && yum update -y \ - && yum clean all + && yum clean all \ +; fi + +RUN if [ "$TARGETPLATFORM" = "linux/arm64" ] ; then cd /etc/yum.repos.d/ \ + && sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* \ + && sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* \ + && yum update -y \ + && yum install -y dnf-plugins-core \ + && yum config-manager --set-enabled powertools \ + && yum clean all \ +; f # install devtools and [enable it](https://access.redhat.com/solutions/527703) RUN yum install -y \ - git vim-common wget unzip which java-11-openjdk-devel.x86_64 \ + git vim-common wget unzip which java-11-openjdk-devel \ libtool autoconf make ninja-build \ && yum clean all # install conda -RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ - && bash Miniconda3-latest-Linux-x86_64.sh -b \ - && rm -f Miniconda3-latest-Linux-x86_64.sh \ - && /root/miniconda3/bin/conda init bash +RUN if [ "$TARGETPLATFORM" = "linux/arm64" ] ; then CONDA_ARCH=aarch64 ; else CONDA_ARCH=x86_64 ; fi \ + && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-$CONDA_ARCH.sh \ + && bash Miniconda3-latest-Linux-$CONDA_ARCH.sh -b \ + && rm -f Miniconda3-latest-Linux-$CONDA_ARCH.sh \ + && /root/miniconda3/bin/conda init # Install lld RUN /root/miniconda3/bin/conda install -c conda-forge lld nasm cmake gxx==11.4.0 -y @@ -24,13 +38,11 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- -y # install go ARG GO_VERSION=1.21.4 -ARG GO_SHA256SUM="73cac0215254d0c7d1241fa40837851f3b9a8a742d0b54714cbdfb3feaf8f0af" # install go -RUN set -eux; \ - url="https://golang.google.cn/dl/go${GO_VERSION}.linux-amd64.tar.gz"; \ +RUN if [ "$TARGETPLATFORM" = "linux/arm64" ] ; then GO_ARCH=arm64 ; else GO_ARCH=amd64 ; fi \ + url="https://golang.google.cn/dl/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz"; \ wget --no-check-certificate -O go.tgz "$url"; \ - echo "${GO_SHA256SUM} *go.tgz" | sha256sum -c -; \ tar -C /usr/local -xzf go.tgz; \ rm go.tgz; From 97355d6e8bfb09abf18f4f1982db55f6fb6835d6 Mon Sep 17 00:00:00 2001 From: anakinxc Date: Mon, 11 Dec 2023 16:53:09 +0800 Subject: [PATCH 2/6] split --- dockerfiles/release-ci-aarch64.DockerFile | 53 +++++++++++++++++++++++ dockerfiles/release-ci.DockerFile | 38 +++++----------- 2 files changed, 65 insertions(+), 26 deletions(-) create mode 100644 dockerfiles/release-ci-aarch64.DockerFile diff --git a/dockerfiles/release-ci-aarch64.DockerFile b/dockerfiles/release-ci-aarch64.DockerFile new file mode 100644 index 0000000..1209ffb --- /dev/null +++ b/dockerfiles/release-ci-aarch64.DockerFile @@ -0,0 +1,53 @@ +FROM centos:centos8 + +RUN cd /etc/yum.repos.d/ \ + && sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* \ + && sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* \ + && yum update -y \ + && yum install -y dnf-plugins-core \ + && yum config-manager --set-enabled powertools \ + && yum clean all + +RUN yum install -y \ + git vim-common wget unzip which java-11-openjdk-devel \ + libtool autoconf make ninja-build \ + && yum clean all + +# install conda +RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh \ + && bash Miniconda3-latest-Linux-aarch64.sh -b \ + && rm -f Miniconda3-latest-Linux-aarch64.sh \ + && /root/miniconda3/bin/conda init + +# Install lld +RUN /root/miniconda3/bin/conda install -c conda-forge lld nasm cmake gxx==11.4.0 -y + +# install Rust +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y + +# install go +ARG GO_VERSION=1.21.4 +ARG GO_SHA256SUM=ce1983a7289856c3a918e1fd26d41e072cc39f928adfb11ba1896440849b95da + +# install go +RUN url="https://golang.google.cn/dl/go${GO_VERSION}.linux-arm64.tar.gz"; \ + wget --no-check-certificate -O go.tgz "$url"; \ + echo "${GO_SHA256SUM} *go.tgz" | sha256sum -c -; \ + tar -C /usr/local -xzf go.tgz; \ + rm go.tgz; + +ENV GOPATH="/usr/local" +ENV PATH="/usr/local/go/bin:${GOPATH}/bin:${PATH}" + +# install bazel +RUN wget https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-arm64 \ + && mv bazelisk-linux-arm64 /usr/bin/bazel \ + && chmod +x /usr/bin/bazel + +# Add conda to path +ENV PATH="/root/miniconda3/bin:${PATH}" + +# run as root for now +WORKDIR /home/admin/ + +ENTRYPOINT [ "/bin/bash", "-l" ] diff --git a/dockerfiles/release-ci.DockerFile b/dockerfiles/release-ci.DockerFile index ce0872d..939eb87 100644 --- a/dockerfiles/release-ci.DockerFile +++ b/dockerfiles/release-ci.DockerFile @@ -1,48 +1,34 @@ -FROM --platform=linux/amd64 centos:centos7 -FROM --platform=linux/arm64 centos:centos8 +FROM centos:centos7 -ARG TARGETPLATFORM - -RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then yum install -y centos-release-scl epel-release \ - && yum update -y \ - && yum clean all \ -; fi - -RUN if [ "$TARGETPLATFORM" = "linux/arm64" ] ; then cd /etc/yum.repos.d/ \ - && sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* \ - && sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* \ +RUN yum install -y centos-release-scl epel-release \ && yum update -y \ - && yum install -y dnf-plugins-core \ - && yum config-manager --set-enabled powertools \ - && yum clean all \ -; f + && yum clean all -# install devtools and [enable it](https://access.redhat.com/solutions/527703) RUN yum install -y \ git vim-common wget unzip which java-11-openjdk-devel \ libtool autoconf make ninja-build \ && yum clean all # install conda -RUN if [ "$TARGETPLATFORM" = "linux/arm64" ] ; then CONDA_ARCH=aarch64 ; else CONDA_ARCH=x86_64 ; fi \ - && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-$CONDA_ARCH.sh \ - && bash Miniconda3-latest-Linux-$CONDA_ARCH.sh -b \ - && rm -f Miniconda3-latest-Linux-$CONDA_ARCH.sh \ +RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && bash Miniconda3-latest-Linux-x86_64.sh -b \ + && rm -f Miniconda3-latest-Linux-x86_64.sh \ && /root/miniconda3/bin/conda init # Install lld -RUN /root/miniconda3/bin/conda install -c conda-forge lld nasm cmake gxx==11.4.0 -y +RUN /root/miniconda3/bin/conda install -c conda-forge lld nasm cmake gxx==11.4.0 -y \ + conda clean -afy # install Rust RUN curl https://sh.rustup.rs -sSf | sh -s -- -y # install go ARG GO_VERSION=1.21.4 - -# install go -RUN if [ "$TARGETPLATFORM" = "linux/arm64" ] ; then GO_ARCH=arm64 ; else GO_ARCH=amd64 ; fi \ - url="https://golang.google.cn/dl/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz"; \ +ARG GO_SHA256SUM="73cac0215254d0c7d1241fa40837851f3b9a8a742d0b54714cbdfb3feaf8f0af" +RUN set -eux; \ + url="https://golang.google.cn/dl/go${GO_VERSION}.linux-amd64.tar.gz"; \ wget --no-check-certificate -O go.tgz "$url"; \ + echo "${GO_SHA256SUM} *go.tgz" | sha256sum -c -; \ tar -C /usr/local -xzf go.tgz; \ rm go.tgz; From 770560f390ed53d36e0ecade782b1608b848c67d Mon Sep 17 00:00:00 2001 From: anakinxc Date: Mon, 11 Dec 2023 16:59:51 +0800 Subject: [PATCH 3/6] update --- dockerfiles/release-ci.DockerFile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/release-ci.DockerFile b/dockerfiles/release-ci.DockerFile index 939eb87..62ea8ba 100644 --- a/dockerfiles/release-ci.DockerFile +++ b/dockerfiles/release-ci.DockerFile @@ -17,7 +17,7 @@ RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ # Install lld RUN /root/miniconda3/bin/conda install -c conda-forge lld nasm cmake gxx==11.4.0 -y \ - conda clean -afy + /root/miniconda3/bin/conda clean -afy # install Rust RUN curl https://sh.rustup.rs -sSf | sh -s -- -y From ff8a44b3f58eb486522259a86c49185fe2f538ec Mon Sep 17 00:00:00 2001 From: anakinxc Date: Mon, 11 Dec 2023 17:00:27 +0800 Subject: [PATCH 4/6] fix --- dockerfiles/release-ci.DockerFile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/release-ci.DockerFile b/dockerfiles/release-ci.DockerFile index 62ea8ba..38b17b0 100644 --- a/dockerfiles/release-ci.DockerFile +++ b/dockerfiles/release-ci.DockerFile @@ -17,7 +17,7 @@ RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ # Install lld RUN /root/miniconda3/bin/conda install -c conda-forge lld nasm cmake gxx==11.4.0 -y \ - /root/miniconda3/bin/conda clean -afy + && /root/miniconda3/bin/conda clean -afy # install Rust RUN curl https://sh.rustup.rs -sSf | sh -s -- -y From 2dfbefa384fc85efd72bd958f44b18056335f1f0 Mon Sep 17 00:00:00 2001 From: anakinxc Date: Mon, 11 Dec 2023 17:01:05 +0800 Subject: [PATCH 5/6] update --- dockerfiles/release-ci-aarch64.DockerFile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dockerfiles/release-ci-aarch64.DockerFile b/dockerfiles/release-ci-aarch64.DockerFile index 1209ffb..855567b 100644 --- a/dockerfiles/release-ci-aarch64.DockerFile +++ b/dockerfiles/release-ci-aarch64.DockerFile @@ -20,7 +20,8 @@ RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh && /root/miniconda3/bin/conda init # Install lld -RUN /root/miniconda3/bin/conda install -c conda-forge lld nasm cmake gxx==11.4.0 -y +RUN /root/miniconda3/bin/conda install -c conda-forge lld nasm cmake gxx==11.4.0 -y \ + && /root/miniconda3/bin/conda clean -afy # install Rust RUN curl https://sh.rustup.rs -sSf | sh -s -- -y From 8776367e00c3ccca6c5e0ac3a9c01aae5f0f8d95 Mon Sep 17 00:00:00 2001 From: anakinxc Date: Mon, 11 Dec 2023 17:06:38 +0800 Subject: [PATCH 6/6] Update release-ci --- dockerfiles/release-ci-aarch64.DockerFile | 8 +++----- dockerfiles/release-ci.DockerFile | 9 ++++----- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/dockerfiles/release-ci-aarch64.DockerFile b/dockerfiles/release-ci-aarch64.DockerFile index 855567b..3ce5c6b 100644 --- a/dockerfiles/release-ci-aarch64.DockerFile +++ b/dockerfiles/release-ci-aarch64.DockerFile @@ -19,6 +19,9 @@ RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh && rm -f Miniconda3-latest-Linux-aarch64.sh \ && /root/miniconda3/bin/conda init +# Add conda to path +ENV PATH="/root/miniconda3/bin:${PATH}" + # Install lld RUN /root/miniconda3/bin/conda install -c conda-forge lld nasm cmake gxx==11.4.0 -y \ && /root/miniconda3/bin/conda clean -afy @@ -29,8 +32,6 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- -y # install go ARG GO_VERSION=1.21.4 ARG GO_SHA256SUM=ce1983a7289856c3a918e1fd26d41e072cc39f928adfb11ba1896440849b95da - -# install go RUN url="https://golang.google.cn/dl/go${GO_VERSION}.linux-arm64.tar.gz"; \ wget --no-check-certificate -O go.tgz "$url"; \ echo "${GO_SHA256SUM} *go.tgz" | sha256sum -c -; \ @@ -45,9 +46,6 @@ RUN wget https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazeli && mv bazelisk-linux-arm64 /usr/bin/bazel \ && chmod +x /usr/bin/bazel -# Add conda to path -ENV PATH="/root/miniconda3/bin:${PATH}" - # run as root for now WORKDIR /home/admin/ diff --git a/dockerfiles/release-ci.DockerFile b/dockerfiles/release-ci.DockerFile index 38b17b0..11716ff 100644 --- a/dockerfiles/release-ci.DockerFile +++ b/dockerfiles/release-ci.DockerFile @@ -15,6 +15,9 @@ RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ && rm -f Miniconda3-latest-Linux-x86_64.sh \ && /root/miniconda3/bin/conda init +# Add conda to path +ENV PATH="/root/miniconda3/bin:${PATH}" + # Install lld RUN /root/miniconda3/bin/conda install -c conda-forge lld nasm cmake gxx==11.4.0 -y \ && /root/miniconda3/bin/conda clean -afy @@ -25,8 +28,7 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- -y # install go ARG GO_VERSION=1.21.4 ARG GO_SHA256SUM="73cac0215254d0c7d1241fa40837851f3b9a8a742d0b54714cbdfb3feaf8f0af" -RUN set -eux; \ - url="https://golang.google.cn/dl/go${GO_VERSION}.linux-amd64.tar.gz"; \ +RUN url="https://golang.google.cn/dl/go${GO_VERSION}.linux-amd64.tar.gz"; \ wget --no-check-certificate -O go.tgz "$url"; \ echo "${GO_SHA256SUM} *go.tgz" | sha256sum -c -; \ tar -C /usr/local -xzf go.tgz; \ @@ -40,9 +42,6 @@ RUN wget https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazeli && mv bazelisk-linux-amd64 /usr/bin/bazel \ && chmod +x /usr/bin/bazel -# Add conda to path -ENV PATH="/root/miniconda3/bin:${PATH}" - # run as root for now WORKDIR /home/admin/