From 9ed414c9ac00229c66103dc9c52ec4b53de6b4ff Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 29 Sep 2023 23:11:09 -0400 Subject: [PATCH] Use MUSL cross-build for ARM64 & dockers (#909) --- .github/workflows/ci.yml | 158 +++++++++++++++++++++++--------------- multi-platform.Dockerfile | 2 +- 2 files changed, 98 insertions(+), 62 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7a8c6d94..e11fef128 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,9 +34,6 @@ jobs: - target: aarch64-apple-darwin os: macOS-latest cross: 'true' - - target: aarch64-unknown-linux-gnu - os: ubuntu-latest - cross: 'true' - target: debian-x86_64 os: ubuntu-latest cross: 'true' @@ -52,6 +49,7 @@ jobs: uses: actions/checkout@v4 - name: Rust Versions run: rustc --version && cargo --version + - uses: Swatinem/rust-cache@v2 - name: Lint (Linux) if: matrix.target == 'x86_64-unknown-linux-gnu' run: | @@ -69,49 +67,74 @@ jobs: run: | echo "VCPKG_ROOT=$env:VCPKG_INSTALLATION_ROOT" | Out-File -FilePath $env:GITHUB_ENV -Append vcpkg install openssl:x64-windows-static-md - - name: Build (native) - if: matrix.cross != 'true' - run: | - cargo build --release --target ${{ matrix.target }} --package martin --features=ssl --package martin - cargo build --release --target ${{ matrix.target }} --package martin-mbtiles - - name: Build (cross - aarch64-apple-darwin) - if: matrix.target == 'aarch64-apple-darwin' - run: | - rustup target add "${{ matrix.target }}" - # compile without debug symbols because stripping them with `strip` does not work cross-platform - export RUSTFLAGS='-C link-arg=-s' - cargo build --release --target ${{ matrix.target }} --package martin --features=vendored-openssl - cargo build --release --target ${{ matrix.target }} --package martin-mbtiles - - name: Build (cross - aarch64-unknown-linux-gnu) - if: matrix.target == 'aarch64-unknown-linux-gnu' - run: | - sudo apt-get install -y gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu - rustup target add "${{ matrix.target }}" - # compile without debug symbols because stripping them with `strip` does not work cross-platform - export RUSTFLAGS='-C link-arg=-s -C linker=aarch64-linux-gnu-gcc' - cargo build --release --target ${{ matrix.target }} --package martin --features=vendored-openssl - cargo build --release --target ${{ matrix.target }} --package martin-mbtiles - - name: Build (debian package) + - name: Build (.deb) if: matrix.target == 'debian-x86_64' run: | sudo apt-get install -y dpkg dpkg-dev liblzma-dev cargo install cargo-deb cargo deb -v -p martin --output target/debian/debian-x86_64.deb - - name: Move build artifacts - run: | mkdir -p target_releases - if [[ "${{ matrix.target }}" == "debian-x86_64" ]]; then - mv target/debian/debian-x86_64.deb target_releases + mv target/debian/debian-x86_64.deb target_releases/ + - name: Build + if: matrix.target != 'debian-x86_64' + run: | + rustup target add "${{ matrix.target }}" + if [[ "${{ runner.os }}" == "Windows" ]]; then + FEATURES="ssl" else - mv target/${{ matrix.target }}/release/martin${{ matrix.ext }} target_releases - mv target/${{ matrix.target }}/release/mbtiles${{ matrix.ext }} target_releases + FEATURES="vendored-openssl" fi + + set -x + export RUSTFLAGS='-C strip=debuginfo' + cargo build --release --target ${{ matrix.target }} --package martin-mbtiles + cargo build --release --target ${{ matrix.target }} --package martin --features=$FEATURES + + mkdir -p target_releases + mv target/${{ matrix.target }}/release/mbtiles${{ matrix.ext }} target_releases/ + mv target/${{ matrix.target }}/release/martin${{ matrix.ext }} target_releases/ - name: Save build artifacts to build-${{ matrix.target }} uses: actions/upload-artifact@v3 with: name: build-${{ matrix.target }} path: target_releases/* + build-cross: + name: Cross-platform builds + runs-on: ubuntu-latest + env: + TARGETS: "aarch64-unknown-linux-musl x86_64-unknown-linux-musl" + # TODO: aarch64-unknown-linux-gnu + steps: + - name: Checkout sources + uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + - name: Install cross + run: | + cargo install cross + # Install latest cross version from git (disabled as it is probably less stable) + # cargo install cross --git https://github.com/cross-rs/cross + cross --version + - name: Build targets + run: | + for target in $TARGETS; do + echo -e "\n----------------------------------------------" + echo "Building $target" + + export "CARGO_TARGET_$(echo $target | tr 'a-z-' 'A-Z_')_RUSTFLAGS"='-C strip=debuginfo' + cross build --release --target $target --package martin-mbtiles + cross build --release --target $target --package martin --features=vendored-openssl + + mkdir -p target_releases/$target + mv target/$target/release/mbtiles target_releases/$target + mv target/$target/release/martin target_releases/$target + done + - name: Save build artifacts to build-${{ matrix.target }} + uses: actions/upload-artifact@v3 + with: + name: build-cross + path: target_releases/* + test: name: Test ${{ matrix.target }} runs-on: ${{ matrix.os }} @@ -130,6 +153,7 @@ jobs: steps: - name: Checkout sources uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 - name: Start postgres uses: nyurik/action-setup-postgis@v1 id: pg @@ -255,6 +279,7 @@ jobs: steps: - name: Checkout sources uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 - name: Setup database run: tests/fixtures/initdb.sh env: @@ -313,7 +338,7 @@ jobs: docker: name: Build docker images runs-on: ubuntu-latest - needs: [ build ] + needs: [ build, build-cross ] env: # PG_* variables are used by psql PGDATABASE: test @@ -345,6 +370,7 @@ jobs: steps: - name: Checkout sources uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 - name: Setup database run: tests/fixtures/initdb.sh env: @@ -362,19 +388,20 @@ jobs: install: true platforms: linux/amd64,linux/arm64 - - run: rm -rf target_releases - - name: Download build artifact build-aarch64-unknown-linux-gnu + - run: rm -rf target_releases2 + - name: Download build-cross artifacts uses: actions/download-artifact@v3 with: - name: build-aarch64-unknown-linux-gnu - path: target_releases/linux/arm64 - - name: Download build artifact build-x86_64-unknown-linux-gnu - uses: actions/download-artifact@v3 - with: - name: build-x86_64-unknown-linux-gnu - path: target_releases/linux/amd64 - - name: Reset permissions - run: chmod -R +x target_releases/ + name: build-cross + path: target_releases2 + - name: Reorganize build artifacts + run: | + chmod -R +x target_releases2/ + mkdir -p target_releases/linux/arm64 + mv target_releases2/aarch64-unknown-linux-musl/* target_releases/linux/arm64/ + mkdir -p target_releases/linux/amd64 + mv target_releases2/x86_64-unknown-linux-musl/* target_releases/linux/amd64/ + rm -rf target_releases2 - name: Build linux/arm64 Docker image id: docker_aarch64-unknown-linux-gnu @@ -457,12 +484,13 @@ jobs: - target: aarch64-apple-darwin os: ubuntu-latest name: martin-Darwin-aarch64.tar.gz - cross: 'true' sha: 'true' - - target: aarch64-unknown-linux-gnu + - target: debian-x86_64 os: ubuntu-latest - name: martin-Linux-aarch64.tar.gz - cross: 'true' + name: martin-Debian-x86_64.deb + # - target: aarch64-unknown-linux-gnu + # os: ubuntu-latest + # name: martin-Linux-aarch64.tar.gz - target: x86_64-apple-darwin os: macOS-latest name: martin-Darwin-x86_64.tar.gz @@ -474,26 +502,30 @@ jobs: - target: x86_64-unknown-linux-gnu os: ubuntu-latest name: martin-Linux-x86_64.tar.gz - - target: debian-x86_64 + # From the cross build + - target: aarch64-unknown-linux-musl os: ubuntu-latest - name: martin-Debian-x86_64.deb cross: 'true' + name: martin-Linux-aarch64-musl.tar.gz + - target: x86_64-unknown-linux-musl + os: ubuntu-latest + cross: 'true' + name: martin-Linux-x86_64-musl.tar.gz steps: - name: Checkout sources uses: actions/checkout@v4 - name: Download build artifact build-${{ matrix.target }} + if: matrix.cross != 'true' uses: actions/download-artifact@v3 with: name: build-${{ matrix.target }} path: target/ - - name: Strip symbols - # Symbol stripping does not work cross-platform - # For cross, symbols were already removed during build - if: matrix.cross != 'true' - run: | - cd target/ - strip martin${{ matrix.ext }} - strip mbtiles${{ matrix.ext }} + - name: Download cross-build artifact build-${{ matrix.target }} + if: matrix.cross == 'true' + uses: actions/download-artifact@v3 + with: + name: build-cross + path: target/ - name: Package run: | cd target/ @@ -502,11 +534,15 @@ jobs: elif [[ "${{ matrix.target }}" == "debian-x86_64" ]]; then mv debian-x86_64.deb ../${{ matrix.name }} else + if [[ "${{ matrix.cross }}" == "true" ]]; then + mv ${{ matrix.target }}/* . + fi tar czvf ../${{ matrix.name }} martin${{ matrix.ext }} mbtiles${{ matrix.ext }} fi - - name: Generate SHA-256 (MacOS) - if: matrix.sha == 'true' - run: shasum -a 256 ${{ matrix.name }} + # TODO: why is this needed and where should the result go? + # - name: Generate SHA-256 (MacOS) + # if: matrix.sha == 'true' + # run: shasum -a 256 ${{ matrix.name }} - name: Publish if: startsWith(github.ref, 'refs/tags/') uses: softprops/action-gh-release@v1 diff --git a/multi-platform.Dockerfile b/multi-platform.Dockerfile index e1fe2f4ee..b7420a3da 100644 --- a/multi-platform.Dockerfile +++ b/multi-platform.Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:latest +FROM alpine ARG TARGETPLATFORM LABEL org.opencontainers.image.description="Blazing fast and lightweight tile server with PostGIS, MBTiles, and PMTiles support"