From bcbf03100527de7781e6c4d53edb39ad086d787d Mon Sep 17 00:00:00 2001 From: "C.Lee Taylor" Date: Thu, 5 Oct 2023 16:27:33 +0200 Subject: [PATCH] GHA - workaround space restraint and fix Debian cross-compile for arm64 --- .github/workflows/build_dockers_workflow.yml | 20 ++++++- docker_rig/README.md | 12 ++++ docker_rig/cross-compile-aarch64.sh | 9 +++ docker_rig/dan-testing.Dockerfile | 63 +++++++++++++------- 4 files changed, 81 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build_dockers_workflow.yml b/.github/workflows/build_dockers_workflow.yml index 9d7b88c..ceaed5c 100644 --- a/.github/workflows/build_dockers_workflow.yml +++ b/.github/workflows/build_dockers_workflow.yml @@ -33,7 +33,7 @@ name: Build docker image - workflow_call/on-demand env: IMAGE_NAME: dan-testing TARI_REPO: tari-project/tari - TARI_BRANCH: v0.51.0-pre.4 + TARI_BRANCH: feature-dan2 TARI-DAN_REPO: tari-project/tari-dan TARI-DAN_BRANCH: development TARI-CONNECTOR_REPO: tari-project/tari-connector @@ -129,6 +129,24 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + # android=14GB - dotnet=2GB - haskell=0B + # large-packages=5GB - docker-images=6GB - swap-storage=4GB + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: false + - name: Docker image build and push id: docker_build uses: docker/build-push-action@v5 diff --git a/docker_rig/README.md b/docker_rig/README.md index 2b3552a..357d533 100644 --- a/docker_rig/README.md +++ b/docker_rig/README.md @@ -12,6 +12,18 @@ docker build -f dan-testing/docker_rig/dan-testing.Dockerfile \ -t local/dan-testing . ``` +# Targeted testing and cross platform builds +```bash +docker build -f dan-testing/docker_rig/dan-testing.Dockerfile \ + -t local/dan-testing-tari-dan --target=builder-tari-dan . +``` +or +```bash +docker buildx build -f dan-testing/docker_rig/dan-testing.Dockerfile \ + -t local/dan-testing-tari-dan-arm64 --target=builder-tari-dan \ + --platform linux/arm64 . +``` + # Docker Testing Notes Launching the docker image with local ports redirected to docker container ports 18000 to 19000 ```bash diff --git a/docker_rig/cross-compile-aarch64.sh b/docker_rig/cross-compile-aarch64.sh index df046c3..48c0c37 100644 --- a/docker_rig/cross-compile-aarch64.sh +++ b/docker_rig/cross-compile-aarch64.sh @@ -18,3 +18,12 @@ export RUSTFLAGS="-C target_cpu=$ARCH" export ROARING_ARCH=$ARCH rustup target add aarch64-unknown-linux-gnu rustup toolchain install stable-aarch64-unknown-linux-gnu --force-non-host + +# Check for Debian +if [ -f "/etc/debian_version" ] ; then + dpkg --add-architecture arm64 + apt-get update + apt-get install -y pkg-config libssl-dev:arm64 + export AARCH64_UNKNOWN_LINUX_GNU_OPENSSL_INCLUDE_DIR=/usr/include/aarch64-linux-gnu/openssl/ + export PKG_CONFIG_ALLOW_CROSS=1 +fi diff --git a/docker_rig/dan-testing.Dockerfile b/docker_rig/dan-testing.Dockerfile index 67b0bba..cf68d25 100644 --- a/docker_rig/dan-testing.Dockerfile +++ b/docker_rig/dan-testing.Dockerfile @@ -1,7 +1,10 @@ # syntax = docker/dockerfile:1.3 # https://hub.docker.com/_/rust -ARG RUST_VERSION=1.70 +ARG RUST_VERSION=1.72 + +# Node Version +ARG NODE_MAJOR=20 # rust source compile with cross platform build support FROM --platform=$BUILDPLATFORM rust:$RUST_VERSION-bullseye as builder-tari @@ -18,16 +21,22 @@ ARG TARGETVARIANT ARG RUST_TOOLCHAIN ARG RUST_TARGET ARG RUST_VERSION + ARG DAN_TESTING_WEBUI_PORT -# Prep nodejs lts - 18.x +# Node Version +ARG NODE_MAJOR +ENV NODE_MAJOR=$NODE_MAJOR + +# Prep nodejs lts - 20.x RUN apt-get update && apt-get install -y \ apt-transport-https \ - bash \ ca-certificates \ curl \ gpg && \ - curl -fsSL https://deb.nodesource.com/setup_lts.x | bash - + mkdir -p /etc/apt/keyrings && \ + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list RUN apt-get update && apt-get install -y \ libreadline-dev \ @@ -74,20 +83,20 @@ RUN if [ "${TARGETARCH}" = "arm64" ] && [ "${BUILDARCH}" != "${TARGETARCH}" ] ; rustup show && \ cargo build ${RUST_TARGET} \ --release --features ${FEATURES} --locked \ - --bin tari_base_node \ - --bin tari_console_wallet \ - --bin tari_miner && \ + --bin minotari_node \ + --bin minotari_console_wallet \ + --bin minotari_miner && \ # Copy executable out of the cache so it is available in the runtime image. - cp -v /tari/target/${BUILD_TARGET}release/tari_base_node /usr/local/bin/ && \ - cp -v /tari/target/${BUILD_TARGET}release/tari_console_wallet /usr/local/bin/ && \ - cp -v /tari/target/${BUILD_TARGET}release/tari_miner /usr/local/bin/ && \ + cp -v /tari/target/${BUILD_TARGET}release/minotari_node /usr/local/bin/ && \ + cp -v /tari/target/${BUILD_TARGET}release/minotari_console_wallet /usr/local/bin/ && \ + cp -v /tari/target/${BUILD_TARGET}release/minotari_miner /usr/local/bin/ && \ echo "Tari Build Done" -RUN mkdir -p "/usr/local/lib/tari/protos/" && \ +RUN mkdir -p "/usr/local/lib/minotari/protos/" && \ python3 -m grpc_tools.protoc \ - --proto_path /tari/applications/tari_app_grpc/proto/ \ - --python_out=/usr/local/lib/tari/protos \ - --grpc_python_out=/usr/local/lib/tari/protos /tari/applications/tari_app_grpc/proto/*.proto + --proto_path /tari/applications/minotari_app_grpc/proto/ \ + --python_out=/usr/local/lib/minotari/protos \ + --grpc_python_out=/usr/local/lib/minotari/protos /tari/applications/minotari_app_grpc/proto/*.proto # rust source compile with cross platform build support FROM --platform=$BUILDPLATFORM rust:$RUST_VERSION-bullseye as builder-tari-dan @@ -105,14 +114,19 @@ ARG RUST_TOOLCHAIN ARG RUST_TARGET ARG RUST_VERSION -# Prep nodejs lts - 18.x +# Node Version +ARG NODE_MAJOR +ENV NODE_MAJOR=$NODE_MAJOR + +# Prep nodejs lts - 20.x RUN apt-get update && apt-get install -y \ apt-transport-https \ - bash \ ca-certificates \ curl \ gpg && \ - curl -fsSL https://deb.nodesource.com/setup_lts.x | bash - + mkdir -p /etc/apt/keyrings && \ + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list RUN apt-get update && apt-get install -y \ libreadline-dev \ @@ -196,14 +210,19 @@ ARG VERSION # Disable Prompt During Packages Installation ARG DEBIAN_FRONTEND=noninteractive -# Prep nodejs 18.x +# Node Version +ARG NODE_MAJOR +ENV NODE_MAJOR=$NODE_MAJOR + +# Prep nodejs 20.x RUN apt-get update && apt-get install -y \ apt-transport-https \ - bash \ ca-certificates \ curl \ gpg && \ - curl -fsSL https://deb.nodesource.com/setup_lts.x | bash - + mkdir -p /etc/apt/keyrings && \ + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list RUN apt-get update && apt-get --no-install-recommends install -y \ libreadline8 \ @@ -273,8 +292,8 @@ RUN npm link WORKDIR /home/tari/sources/dan-testing RUN npm link tari-connector -COPY --chown=tari:tari --from=builder-tari /usr/local/lib/tari/protos /home/tari/sources/dan-testing/protos -COPY --from=builder-tari /usr/local/bin/tari_* /usr/local/bin/ +COPY --chown=tari:tari --from=builder-tari /usr/local/lib/minotari/protos /home/tari/sources/dan-testing/protos +COPY --from=builder-tari /usr/local/bin/minotari_* /usr/local/bin/ COPY --from=builder-tari-dan /usr/local/bin/tari_* /usr/local/bin/ ENV DAN_TESTING_WEBUI_PORT=${DAN_TESTING_WEBUI_PORT:-18000}