Use MUSL cross-build for ARM64 & dockers #2006

Workflow file for this run

	name: CI

	on:
	push:
	branches: [ main ]
	paths-ignore:
	- '**.md'
	- 'demo/**'
	- 'docs/**'
	- 'homebrew-formula/**'
	pull_request:
	branches: [ main ]
	paths-ignore:
	- '**.md'
	- 'demo/**'
	- 'docs/**'
	- 'homebrew-formula/**'
	release:
	types: [ published ]
	workflow_dispatch:

	defaults:
	run:
	shell: bash

	jobs:
	build:
	name: Build ${{ matrix.target }}
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: true
	matrix:
	include:
	- target: aarch64-apple-darwin
	os: macOS-latest
	cross: 'true'
	- target: debian-x86_64
	os: ubuntu-latest
	cross: 'true'
	- target: x86_64-apple-darwin
	os: macOS-latest
	- target: x86_64-pc-windows-msvc
	os: windows-latest
	ext: '.exe'
	- target: x86_64-unknown-linux-gnu
	os: ubuntu-latest
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: Rust Versions
	run: rustc --version && cargo --version
	- uses: Swatinem/rust-cache@v2
	- name: Lint (Linux)
	if: matrix.target == 'x86_64-unknown-linux-gnu'
	run: \|
	set -x
	cargo fmt --all -- --check
	cargo clippy --package martin-tile-utils -- -D warnings
	cargo clippy --package martin-mbtiles --no-default-features -- -D warnings
	cargo clippy --package martin-mbtiles -- -D warnings
	cargo clippy --package martin -- -D warnings
	cargo clippy --package martin --features vendored-openssl -- -D warnings
	cargo clippy --package martin --features bless-tests -- -D warnings
	- name: Install OpenSSL (Windows)
	if: runner.os == 'Windows'
	shell: powershell
	run: \|
	echo "VCPKG_ROOT=$env:VCPKG_INSTALLATION_ROOT" \| Out-File -FilePath $env:GITHUB_ENV -Append
	vcpkg install openssl:x64-windows-static-md
	- name: Build (.deb)
	if: matrix.target == 'debian-x86_64'
	run: \|
	sudo apt-get install -y dpkg dpkg-dev liblzma-dev
	cargo install cargo-deb
	cargo deb -v -p martin --output target/debian/debian-x86_64.deb
	mkdir -p target_releases
	mv target/debian/debian-x86_64.deb target_releases/
	- name: Build
	if: matrix.target != 'debian-x86_64'
	run: \|
	rustup target add "${{ matrix.target }}"
	if [[ "${{ runner.os }}" == "Windows" ]]; then
	FEATURES="ssl"
	else
	FEATURES="vendored-openssl"
	fi

	set -x
	export RUSTFLAGS='-C strip=debuginfo'
	cargo build --release --target ${{ matrix.target }} --package martin-mbtiles
	cargo build --release --target ${{ matrix.target }} --package martin --features=$FEATURES

	mkdir -p target_releases
	mv target/${{ matrix.target }}/release/mbtiles${{ matrix.ext }} target_releases/
	mv target/${{ matrix.target }}/release/martin${{ matrix.ext }} target_releases/
	- name: Save build artifacts to build-${{ matrix.target }}
	uses: actions/upload-artifact@v3
	with:
	name: build-${{ matrix.target }}
	path: target_releases/*

	build-cross:
	name: Cross-platform builds
	runs-on: ubuntu-latest
	env:
	TARGETS: "aarch64-unknown-linux-musl x86_64-unknown-linux-musl"
	# TODO: aarch64-unknown-linux-gnu
	steps:
	- name: Checkout sources
	uses: actions/checkout@v4
	- uses: Swatinem/rust-cache@v2
	- name: Install cross
	run: \|
	cargo install cross
	# Install latest cross version from git (disabled as it is probably less stable)
	# cargo install cross --git https://github.com/cross-rs/cross
	cross --version
	- name: Build targets
	run: \|
	for target in $TARGETS; do
	echo -e "\n----------------------------------------------"
	echo "Building $target"

	export "CARGO_TARGET_$(echo $target \| tr 'a-z-' 'A-Z_')_RUSTFLAGS"='-C strip=debuginfo'
	cross build --release --target $target --package martin-mbtiles
	cross build --release --target $target --package martin --features=vendored-openssl

	mkdir -p target_releases/$target
	mv target/$target/release/mbtiles target_releases/$target
	mv target/$target/release/martin target_releases/$target
	done
	- name: Save build artifacts to build-${{ matrix.target }}
	uses: actions/upload-artifact@v3
	with:
	name: build-cross
	path: target_releases/*

	test:
	name: Test ${{ matrix.target }}
	runs-on: ${{ matrix.os }}
	needs: [ build ]
	strategy:
	fail-fast: true
	matrix:
	include:
	- target: x86_64-apple-darwin
	os: macOS-latest
	- target: x86_64-pc-windows-msvc
	os: windows-latest
	ext: '.exe'
	- target: x86_64-unknown-linux-gnu
	os: ubuntu-latest
	steps:
	- name: Checkout sources
	uses: actions/checkout@v4
	- uses: Swatinem/rust-cache@v2
	- name: Start postgres
	uses: nyurik/action-setup-postgis@v1
	id: pg
	with:
	username: test
	password: test
	database: test
	rights: --superuser
	- name: Init database
	run: \|
	echo "DATABASE_URL=$DATABASE_URL"
	echo "Print the same in base64 to bypass Github's obfuscation (uses hardcoded password):"
	echo "$DATABASE_URL" \| base64
	tests/fixtures/initdb.sh
	env:
	DATABASE_URL: ${{ steps.pg.outputs.connection-uri }}
	- name: Unit Tests (Linux)
	if: matrix.target == 'x86_64-unknown-linux-gnu'
	run: \|
	set -x
	cargo test --package martin-tile-utils
	cargo test --package martin-mbtiles --no-default-features
	cargo test --package martin-mbtiles
	cargo test --package martin
	cargo test --package martin --features vendored-openssl
	cargo test --doc
	RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --workspace
	cargo clean
	env:
	DATABASE_URL: ${{ steps.pg.outputs.connection-uri }}
	- name: Download build artifact build-${{ matrix.target }}
	uses: actions/download-artifact@v3
	with:
	name: build-${{ matrix.target }}
	path: target/
	- name: Integration Tests
	run: \|
	export MARTIN_BUILD=-
	export MARTIN_BIN=target/martin${{ matrix.ext }}
	export MBTILES_BUILD=-
	export MBTILES_BIN=target/mbtiles${{ matrix.ext }}
	if [[ "${{ runner.os }}" != "Windows" ]]; then
	chmod +x "$MARTIN_BIN" "$MBTILES_BIN"
	fi
	tests/test.sh
	env:
	DATABASE_URL: ${{ steps.pg.outputs.connection-uri }}
	- name: Compare test output results (Linux)
	if: matrix.target == 'x86_64-unknown-linux-gnu'
	run: diff --brief --recursive --new-file tests/output tests/expected
	- name: Download Debian package (Linux)
	if: matrix.target == 'x86_64-unknown-linux-gnu'
	uses: actions/download-artifact@v3
	with:
	name: build-debian-x86_64
	path: target/
	- name: Tests Debian package (Linux)
	if: matrix.target == 'x86_64-unknown-linux-gnu'
	run: \|
	sudo dpkg -i target/debian-x86_64.deb
	export MARTIN_BUILD=-
	export MARTIN_BIN=/usr/bin/martin${{ matrix.ext }}
	export MBTILES_BUILD=-
	export MBTILES_BIN=/usr/bin/mbtiles${{ matrix.ext }}
	tests/test.sh
	env:
	DATABASE_URL: ${{ steps.pg.outputs.connection-uri }}
	- name: Save test output on failure (Linux)
	if: failure() && matrix.target == 'x86_64-unknown-linux-gnu'
	uses: actions/upload-artifact@v3
	with:
	name: failed-test-output
	path: tests/output/*
	retention-days: 5

	test-legacy:
	name: Test Legacy DB
	runs-on: ubuntu-latest
	needs: [ build ]
	strategy:
	fail-fast: true
	matrix:
	include:
	# These must match the versions of postgres used in the docker-compose.yml
	- image: postgis/postgis:11-3.0-alpine
	args: postgres
	sslmode: disable
	- image: postgis/postgis:14-3.3-alpine
	args: postgres
	sslmode: disable
	# alpine images don't support SSL, so for this we use the debian images
	- image: postgis/postgis:15-3.3
	args: postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
	sslmode: require
	env:
	# PG_* variables are used by psql
	PGDATABASE: test
	PGHOST: localhost
	PGUSER: postgres
	PGPASSWORD: postgres
	services:
	postgres:
	image: ${{ matrix.image }}
	ports:
	# will assign a random free host port
	- 5432/tcp
	# Sadly there is currently no way to pass arguments to the service image other than this hack
	# See also https://stackoverflow.com/a/62720566/177275
	options: >-
	-e POSTGRES_DB=test
	-e POSTGRES_USER=postgres
	-e POSTGRES_PASSWORD=postgres
	-e PGDATABASE=test
	-e PGUSER=postgres
	-e PGPASSWORD=postgres
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	--entrypoint sh
	${{ matrix.image }}
	-c "exec docker-entrypoint.sh ${{ matrix.args }}"
	steps:
	- name: Checkout sources
	uses: actions/checkout@v4
	- uses: Swatinem/rust-cache@v2
	- name: Setup database
	run: tests/fixtures/initdb.sh
	env:
	PGPORT: ${{ job.services.postgres.ports[5432] }}
	- name: Download build artifact build-x86_64-unknown-linux-gnu
	uses: actions/download-artifact@v3
	with:
	name: build-x86_64-unknown-linux-gnu
	path: target_releases/
	- name: Integration Tests
	run: \|
	export MARTIN_BUILD=-
	export MARTIN_BIN=target_releases/martin
	export MBTILES_BUILD=-
	export MBTILES_BIN=target_releases/mbtiles
	chmod +x "$MARTIN_BIN" "$MBTILES_BIN"
	tests/test.sh
	rm -rf target_releases
	env:
	DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=${{ matrix.sslmode }}
	- name: Download Debian package
	uses: actions/download-artifact@v3
	with:
	name: build-debian-x86_64
	path: target_releases/
	- name: Tests Debian package
	run: \|
	sudo dpkg -i target_releases/debian-x86_64.deb
	export MARTIN_BUILD=-
	export MARTIN_BIN=/usr/bin/martin
	export MBTILES_BUILD=-
	export MBTILES_BIN=/usr/bin/mbtiles
	tests/test.sh
	sudo dpkg -P martin
	rm -rf target_releases
	env:
	DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=${{ matrix.sslmode }}
	- name: Unit Tests
	run: \|
	echo "Running unit tests, connecting to DATABASE_URL=$DATABASE_URL"
	echo "Same but as base64 to prevent GitHub obfuscation (this is not a secret):"
	echo "$DATABASE_URL" \| base64
	set -x
	cargo test --package martin
	cargo clean
	env:
	DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=${{ matrix.sslmode }}
	- name: On error, save test output
	if: failure()
	uses: actions/upload-artifact@v3
	with:
	name: test-output
	path: tests/output/*
	retention-days: 5

	docker:
	name: Build docker images
	runs-on: ubuntu-latest
	needs: [ build, build-cross ]
	env:
	# PG_* variables are used by psql
	PGDATABASE: test
	PGHOST: localhost
	PGUSER: postgres
	PGPASSWORD: postgres
	services:
	postgres:
	image: postgis/postgis:15-3.3
	ports:
	# will assign a random free host port
	- 5432/tcp
	# Sadly there is currently no way to pass arguments to the service image other than this hack
	# See also https://stackoverflow.com/a/62720566/177275
	options: >-
	-e POSTGRES_DB=test
	-e POSTGRES_USER=postgres
	-e POSTGRES_PASSWORD=postgres
	-e PGDATABASE=test
	-e PGUSER=postgres
	-e PGPASSWORD=postgres
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	--entrypoint sh
	postgis/postgis:15-3.3
	-c "exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key"
	steps:
	- name: Checkout sources
	uses: actions/checkout@v4
	- uses: Swatinem/rust-cache@v2
	- name: Setup database
	run: tests/fixtures/initdb.sh
	env:
	PGPORT: ${{ job.services.postgres.ports[5432] }}

	- name: Set up QEMU
	uses: docker/setup-qemu-action@v3
	# https://github.com/docker/setup-qemu-action
	with:
	platforms: linux/amd64,linux/arm64
	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3
	# https://github.com/docker/setup-buildx-action
	with:
	install: true
	platforms: linux/amd64,linux/arm64

	- run: rm -rf target_releases2
	- name: Download build-cross artifacts
	uses: actions/download-artifact@v3
	with:
	name: build-cross
	path: target_releases2
	- name: Reorganize build artifacts
	run: \|
	chmod -R +x target_releases2/
	mkdir -p target_releases/linux/arm64
	mv target_releases2/aarch64-unknown-linux-musl/* target_releases/linux/arm64/
	mkdir -p target_releases/linux/amd64
	mv target_releases2/x86_64-unknown-linux-musl/* target_releases/linux/amd64/
	rm -rf target_releases2

	- name: Build linux/arm64 Docker image
	id: docker_aarch64-unknown-linux-gnu
	uses: docker/build-push-action@v5
	# https://github.com/docker/build-push-action
	with:
	context: .
	file: multi-platform.Dockerfile
	load: true
	tags: ${{ github.repository }}:linux-arm64
	platforms: linux/arm64
	- name: Test linux/arm64 Docker image
	run: \|
	PLATFORM=linux/arm64
	TAG=${{ github.repository }}:linux-arm64
	export MBTILES_BUILD=-
	export MBTILES_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests --entrypoint /usr/local/bin/mbtiles $TAG"
	export MARTIN_BUILD=-
	export MARTIN_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests $TAG"
	tests/test.sh
	env:
	DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=require

	- name: Build linux/amd64 Docker image
	id: docker_x86_64-unknown-linux-gnu
	uses: docker/build-push-action@v5
	# https://github.com/docker/build-push-action
	with:
	context: .
	file: multi-platform.Dockerfile
	load: true
	tags: ${{ github.repository }}:linux-amd64
	platforms: linux/amd64
	- name: Test linux/amd64 Docker image
	run: \|
	PLATFORM=linux/amd64
	TAG=${{ github.repository }}:linux-amd64
	export MBTILES_BUILD=-
	export MBTILES_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests --entrypoint /usr/local/bin/mbtiles $TAG"
	export MARTIN_BUILD=-
	export MARTIN_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests $TAG"
	tests/test.sh
	env:
	DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=require

	- name: Login to GitHub Docker registry
	if: github.event_name != 'pull_request'
	uses: docker/login-action@v3
	# https://github.com/docker/login-action
	with:
	registry: ghcr.io
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}

	- name: Docker meta
	id: docker_meta
	uses: docker/metadata-action@v5
	# https://github.com/docker/metadata-action
	with:
	images: ghcr.io/${{ github.repository }}
	- name: Push the Docker image
	if: github.event_name != 'pull_request'
	uses: docker/build-push-action@v5
	with:
	context: .
	file: multi-platform.Dockerfile
	push: true
	tags: ${{ steps.docker_meta.outputs.tags }}
	labels: ${{ steps.docker_meta.outputs.labels }}
	platforms: linux/amd64,linux/arm64

	package:
	name: Package ${{ matrix.target }}
	runs-on: ${{ matrix.os }}
	needs: [ test, test-legacy ]
	strategy:
	fail-fast: true
	matrix:
	include:
	- target: aarch64-apple-darwin
	os: ubuntu-latest
	name: martin-Darwin-aarch64.tar.gz
	sha: 'true'
	- target: debian-x86_64
	os: ubuntu-latest
	name: martin-Debian-x86_64.deb
	# - target: aarch64-unknown-linux-gnu
	# os: ubuntu-latest
	# name: martin-Linux-aarch64.tar.gz
	- target: x86_64-apple-darwin
	os: macOS-latest
	name: martin-Darwin-x86_64.tar.gz
	sha: 'true'
	- target: x86_64-pc-windows-msvc
	os: windows-latest
	name: martin-Windows-x86_64.zip
	ext: '.exe'
	- target: x86_64-unknown-linux-gnu
	os: ubuntu-latest
	name: martin-Linux-x86_64.tar.gz
	# From the cross build
	- target: aarch64-unknown-linux-musl
	os: ubuntu-latest
	cross: 'true'
	name: martin-Linux-aarch64-musl.tar.gz
	- target: x86_64-unknown-linux-musl
	os: ubuntu-latest
	cross: 'true'
	name: martin-Linux-x86_64-musl.tar.gz
	steps:
	- name: Checkout sources
	uses: actions/checkout@v4
	- name: Download build artifact build-${{ matrix.target }}
	if: matrix.cross != 'true'
	uses: actions/download-artifact@v3
	with:
	name: build-${{ matrix.target }}
	path: target/
	- name: Download cross-build artifact build-${{ matrix.target }}
	if: matrix.cross == 'true'
	uses: actions/download-artifact@v3
	with:
	name: build-cross
	path: target/
	- name: Package
	run: \|
	cd target/
	if [[ "${{ runner.os }}" == "Windows" ]]; then
	7z a ../${{ matrix.name }} martin${{ matrix.ext }} mbtiles${{ matrix.ext }}
	elif [[ "${{ matrix.target }}" == "debian-x86_64" ]]; then
	mv debian-x86_64.deb ../${{ matrix.name }}
	else
	if [[ "${{ matrix.cross }}" == "true" ]]; then
	mv ${{ matrix.target }}/* .
	fi
	tar czvf ../${{ matrix.name }} martin${{ matrix.ext }} mbtiles${{ matrix.ext }}
	fi
	# TODO: why is this needed and where should the result go?
	# - name: Generate SHA-256 (MacOS)
	# if: matrix.sha == 'true'
	# run: shasum -a 256 ${{ matrix.name }}
	- name: Publish
	if: startsWith(github.ref, 'refs/tags/')
	uses: softprops/action-gh-release@v1
	with:
	draft: true
	files: 'martin*'
	body_path: CHANGELOG.md
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

	# This final step is needed to mark the whole workflow as successful
	# Don't change its name - it is used by the merge protection rules
	done:
	name: CI Finished
	runs-on: ubuntu-latest
	needs: [ docker, package ]
	steps:
	- name: Finished
	run: echo "CI finished successfully"

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Use MUSL cross-build for ARM64 & dockers #2006

Workflow file

Use MUSL cross-build for ARM64 & dockers #2006

Jobs

Run details

Workflow file for this run