From b929941ebe17d74855d9dd582db0c36e290b3031 Mon Sep 17 00:00:00 2001 From: Ian Davis Date: Wed, 18 Oct 2023 15:16:38 -0700 Subject: [PATCH] Remove manylinux container --- .github/actions/install-llvm/action.yml | 5 +- .github/workflows/ci.yml | 8 +-- .github/workflows/llvm.yml | 9 +-- docs/building.md | 3 - eng/Dockerfile.examples | 11 ++-- eng/Dockerfile.manylinux | 64 ------------------- eng/psakefile.ps1 | 83 ++----------------------- 7 files changed, 15 insertions(+), 168 deletions(-) delete mode 100644 eng/Dockerfile.manylinux diff --git a/.github/actions/install-llvm/action.yml b/.github/actions/install-llvm/action.yml index c836d3f3..53aeb0b9 100644 --- a/.github/actions/install-llvm/action.yml +++ b/.github/actions/install-llvm/action.yml @@ -31,7 +31,7 @@ runs: shell: pwsh if: ${{ (inputs.os == 'ubuntu-20.04') && (steps.cache-llvm.outputs.cache-hit != 'true') }} - name: Windows - Install build dependencies, ninja - run: | + run: | choco install --accept-license -y ninja choco uninstall -y llvm shell: pwsh @@ -55,7 +55,6 @@ runs: - name: Install LLVM run: | - $target_prefix = if("${{ inputs.target }}" -eq "manylinux") {"manylinux-"} else {""} - ./build.ps1 -t "$($target_prefix)install-llvm-from-source" + ./build.ps1 -t install-llvm-from-source shell: pwsh if: ${{ steps.cache-llvm.outputs.cache-hit != 'true' }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ab99a3d..f91e7ee2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,12 +29,6 @@ jobs: target: "checks", env: {}, } - - { - os: "ubuntu-20.04", - arch: "amd64", - target: "manylinux", - env: {}, - } - { os: "ubuntu-20.04", arch: "amd64", @@ -83,7 +77,7 @@ jobs: uses: actions/upload-artifact@v2 with: name: manylinux-artifacts - path: target/wheels/*manylinux_2_17*.whl + path: target/wheels/*manylinux_2_*.whl if: ${{ matrix.config.os == 'ubuntu-20.04' }} - name: Artifacts uses: actions/upload-artifact@v2 diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index 9fcf30a0..a500fc54 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -19,7 +19,7 @@ jobs: matrix: config: - { - os: "ubuntu-22.04", + os: "ubuntu-20.04", arch: "amd64", env: {}, } @@ -39,7 +39,7 @@ jobs: submodules: 'recursive' - name: Linux - Install build dependencies, ccache, ninja run: sudo apt-get install -y ccache ninja-build - if: ${{ matrix.config.os == 'ubuntu-22.04' }} + if: ${{ matrix.config.os == 'ubuntu-20.04' }} - name: Windows - Install build dependencies, sccache, ninja run: choco install --accept-license -y sccache ninja if: ${{ matrix.config.os == 'windows-2019' }} @@ -52,11 +52,6 @@ jobs: - name: "Build and Package LLVM" run: ./build.ps1 -t package-llvm shell: pwsh - if: ${{ matrix.config.os != 'ubuntu-22.04' }} - - name: "Build and Package LLVM for manylinux" - run: ./build.ps1 -t package-manylinux-llvm - shell: pwsh - if: ${{ matrix.config.os == 'ubuntu-22.04' }} - name: Artifacts uses: actions/upload-artifact@v3 with: diff --git a/docs/building.md b/docs/building.md index 15f7a9d7..617cdb64 100644 --- a/docs/building.md +++ b/docs/building.md @@ -152,6 +152,3 @@ be linked into the rest of the build. The `build.ps1`, `maturin` builds all generate Python wheels to the `target/wheels` folder. The default Python3 installation will be used targeting Python ABI 3.7. - -The manylinux support uses a Docker image in the build scripts to run the builds -in the CI environment. diff --git a/eng/Dockerfile.examples b/eng/Dockerfile.examples index 3e909ce7..91c67663 100644 --- a/eng/Dockerfile.examples +++ b/eng/Dockerfile.examples @@ -3,19 +3,18 @@ ARG RELEASE=jammy -FROM buildpack-deps:buster-scm AS buster +FROM buildpack-deps:bullseye-scm AS bullseye -RUN wget https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb \ - && dpkg -i packages-microsoft-prod.deb \ - && rm packages-microsoft-prod.deb \ +RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \ + && echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-debian-bullseye-prod bullseye main" > /etc/apt/sources.list.d/microsoft.list \ && apt-get update \ && apt-get install -y --no-install-recommends powershell python3-minimal python3-pip python3-setuptools sudo \ && rm -rf /var/lib/apt/lists/* -FROM buildpack-deps:bullseye-scm AS bullseye +FROM buildpack-deps:bookworm-scm AS bookworm RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \ - && echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-debian-bullseye-prod bullseye main" > /etc/apt/sources.list.d/microsoft.list \ + && echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-debian-bookworm-prod bookworm main" > /etc/apt/sources.list.d/microsoft.list \ && apt-get update \ && apt-get install -y --no-install-recommends powershell python3-minimal python3-pip python3-setuptools sudo \ && rm -rf /var/lib/apt/lists/* diff --git a/eng/Dockerfile.manylinux b/eng/Dockerfile.manylinux deleted file mode 100644 index c5fb2adb..00000000 --- a/eng/Dockerfile.manylinux +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -FROM quay.io/pypa/manylinux2014_x86_64 - -ARG USERNAME=runner -ARG USER_UID=1000 -ARG USER_GID=${USER_UID} -ARG RUST_TOOLCHAIN=1.64.0 - -# libcgroup created a new group that conflicts with the agent group id. -# To avoid this issue in the future, -o allows for non-unique group ids. -RUN groupadd --gid ${USER_GID} -o ${USERNAME} -RUN useradd --uid ${USER_UID} --gid ${USER_GID} -m ${USERNAME} -RUN yum install -y sudo -RUN echo ${USERNAME} ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/${USERNAME} -RUN chmod 0440 /etc/sudoers.d/${USERNAME} - -ENV RUSTUP_HOME=/usr/local/rustup \ - CARGO_HOME=/usr/local/cargo \ - PATH=/usr/local/cargo/bin:${PATH} - -RUN curl --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --profile minimal --default-toolchain none -y -RUN chmod -R a+w ${RUSTUP_HOME} ${CARGO_HOME}; - -RUN rustup override set ${RUST_TOOLCHAIN} -RUN rustup --version; \ - cargo --version; \ - rustc --version; - -WORKDIR /io -RUN chown ${USER_UID}:${USER_GID} /io - -USER $USERNAME - -# Add all supported python versions -ENV PATH /opt/python/cp38-cp38/bin:/opt/python/cp39-cp39/bin:/opt/python/cp310-cp310/bin:/opt/python/cp311-cp311/bin:/opt/python/cp312-cp312/bin:$PATH - -RUN python3.8 -m pip install --no-cache-dir cffi \ - && python3.9 -m pip install --no-cache-dir cffi \ - && python3.10 -m pip install --no-cache-dir cffi \ - && python3.11 -m pip install --no-cache-dir cffi \ - && python3.12 -m pip install --no-cache-dir cffi - -USER root - -RUN yum install -y libffi-devel ninja-build ccache - -ADD https://repo.anaconda.com/miniconda/Miniconda3-py39_4.10.3-Linux-x86_64.sh /tmp/Miniconda3.sh - -RUN /bin/bash /tmp/Miniconda3.sh -b -p /usr/local/miniconda3 - -RUN chown -R ${USER_UID}:${USER_GID} /usr/local/miniconda3 - -RUN curl https://packages.microsoft.com/config/rhel/7/prod.repo | tee /etc/yum.repos.d/microsoft.repo && \ - yum install -y powershell - -USER $USERNAME - -ENV PATH="/usr/local/miniconda3/bin:${PATH}" - -RUN conda init && \ - conda install -y -c conda-forge clang-11 libstdcxx-devel_linux-64 libgcc-devel_linux-64 && \ - cp /usr/local/miniconda3/bin/clang-11 /usr/local/miniconda3/bin/clang++-11 diff --git a/eng/psakefile.ps1 b/eng/psakefile.ps1 index aedb5d20..e74a6041 100644 --- a/eng/psakefile.ps1 +++ b/eng/psakefile.ps1 @@ -14,33 +14,12 @@ Properties { $VscodeSettingsJson = Join-Path $Root .vscode settings.json $DocsRoot = Join-Path $Root docs $DocsBuild = Join-Path $DocsRoot _build - $ManylinuxTag = "manylinux2014_x86_64_maturin" - $ManylinuxRoot = "/io" $Python = Resolve-Python } task default -depends build, run-examples task build -depends qirlib, pyqir task checks -depends cargo-fmt, cargo-clippy, black, mypy -task manylinux -depends build-manylinux-container-image, run-manylinux-container-image, run-examples-in-containers - -task run-manylinux-container-image -preaction { Write-CacheStats } -postaction { Write-CacheStats } { - $llvmDir = Resolve-InstallationDirectory - $llvmMount = @("-v", "$($llvmDir):/tmp/llvm") - Write-BuildLog "Running container image: $ManylinuxTag" - $ioVolume = "${Root}:$ManylinuxRoot" - $userName = Get-LinuxContainerUserName - - Invoke-LoggedCommand { - docker run --rm ` - --user $userName ` - --volume $ioVolume @llvmMount ` - --env QIRLIB_CACHE_DIR=/tmp/llvm ` - --workdir $ManylinuxRoot ` - $ManylinuxTag ` - conda run --no-capture-output pwsh build.ps1 -t default - } -} task cargo-fmt { Invoke-LoggedCommand -workingDirectory $Root -errorMessage "Please run 'cargo fmt --all' before pushing" { @@ -77,8 +56,11 @@ task qirlib -depends init { task pyqir -depends init { $env:MATURIN_PEP517_ARGS = (Get-CargoArgs) -Join " " Get-Wheels pyqir | Remove-Item - Invoke-LoggedCommand { pip --verbose wheel --wheel-dir $Wheels $Pyqir } + Invoke-LoggedCommand { & $Python -m pip --verbose wheel --wheel-dir $Wheels $Pyqir } + if ($IsLinux) { + Invoke-LoggedCommand { & $Python -m pip install auditwheel } + } if (Test-CommandExists auditwheel) { $unauditedWheels = Get-Wheels pyqir Invoke-LoggedCommand { auditwheel repair --wheel-dir $Wheels $unauditedWheels } @@ -179,41 +161,6 @@ task install-llvm-from-source -depends configure-sccache -postaction { Write-Cac Assert (Test-LlvmConfig $installationDirectory) "install-llvm-from-source failed to install a usable LLVM installation" } -task manylinux-install-llvm-from-source -depends build-manylinux-container-image -preaction { Write-CacheStats } -postaction { Write-CacheStats } { - $llvmDir = Resolve-InstallationDirectory - $llvmMount = @("-v", "$($llvmDir):/tmp/llvm") - Write-BuildLog "Running container image: $ManylinuxTag" - $ioVolume = "${Root}:$ManylinuxRoot" - $userName = Get-LinuxContainerUserName - - Invoke-LoggedCommand { - docker run --rm ` - --user $userName ` - --volume $ioVolume @llvmMount ` - --workdir $ManylinuxRoot ` - $ManylinuxTag ` - conda run --no-capture-output pwsh build.ps1 -t install-llvm-from-source - } -} - -task package-manylinux-llvm -depends build-manylinux-container-image -preaction { Write-CacheStats } -postaction { Write-CacheStats } { - $llvmDir = Resolve-InstallationDirectory - $llvmMount = @("-v", "$($llvmDir):/tmp/llvm") - Write-BuildLog "Running container image: $ManylinuxTag" - $ioVolume = "${Root}:$ManylinuxRoot" - $userName = Get-LinuxContainerUserName - - Invoke-LoggedCommand { - docker run --rm ` - --user $userName ` - --volume $ioVolume @llvmMount ` - --workdir $ManylinuxRoot ` - --env QIRLIB_PKG_DEST=$ManylinuxRoot/target/manylinux ` - $ManylinuxTag ` - conda run --no-capture-output pwsh build.ps1 -t package-llvm - } -} - task package-llvm { if ($IsWindows) { Include vcvars.ps1 @@ -236,32 +183,12 @@ task package-llvm { } } -task build-manylinux-container-image { - Write-BuildLog "Building container image manylinux-llvm-builder" - $RustVersion = (rustc --version) -split " " -match "^(\d+\.)?(\d+\.)?(\*|\d+)$" - Write-BuildLog "Found rustc version $RustVersion" - Invoke-LoggedCommand -workingDirectory (Join-Path $Root eng) { - $user = Get-LinuxContainerUserName - $uid = Get-LinuxContainerUserId - $gid = Get-LinuxContainerGroupId - docker build ` - --build-arg USERNAME=$user ` - --build-arg USER_UID=$uid ` - --build-arg USER_GID=$gid ` - --build-arg RUST_TOOLCHAIN=$RustVersion ` - --tag $ManylinuxTag ` - -f Dockerfile.manylinux ` - $Root - } -} - -# This is only usable if building for manylinux task run-examples-in-containers { $user = Get-LinuxContainerUserName $uid = Get-LinuxContainerUserId $gid = Get-LinuxContainerGroupId - foreach ($release in @("buster", "bullseye", "focal", "jammy")) { + foreach ($release in @("bullseye", "bookworm", "focal", "jammy")) { exec -workingDirectory (Join-Path $Root eng) { Get-Content Dockerfile.examples | docker build ` --build-arg RELEASE=$release `