diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index bd4cf2349f3f..131bf152d147 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -3,13 +3,28 @@ name: Build and push multi-platform docker images on: push: tags: - - '^v[0-9]{2}\.[0-9]{2}(\.[0-9]{1,2})?$' + - '^v[0-9]{2}\.[0-9]{2}(\.[0-9]{1,2})?([a-zA-Z0-9]*)?$' workflow_dispatch: inputs: version: description: 'Release version' required: true + repository-name: + description: 'Docker repository name' + default: 'elementsproject' + required: false + + platforms-to-build: + description: 'List of platforms to build' + default: 'linux/amd64,linux/arm64,linux/arm/v7' + required: false + + push-latest: + description: 'Push the latest tag also?' + default: 'false' + required: false + jobs: build: runs-on: ubuntu-latest @@ -30,8 +45,8 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Set up version - id: set-version + - name: Set up values + id: set-values run: | if [ "${{ github.event.inputs.version }}" != "" ]; then VERSION=${{ github.event.inputs.version }} @@ -42,13 +57,58 @@ jobs: exit 1 fi echo "VERSION=$VERSION" >> $GITHUB_ENV - + + if [ "${{ github.event.inputs.repository-name }}" != "" ]; then + REPONAME=${{ github.event.inputs.repository-name }} + else + REPONAME="elementsproject" + fi + echo "REPONAME=$REPONAME" >> $GITHUB_ENV + + if [ "${{ github.event.inputs.platforms-to-build }}" != "" ]; then + PLATFORMS=${{ github.event.inputs.platforms-to-build }} + else + PLATFORMS="linux/amd64,linux/arm64,linux/arm/v7" + fi + echo "PLATFORMS=$PLATFORMS" >> $GITHUB_ENV + + if [ + "${{ github.event.inputs.push-latest }}" == "true" || + ( "${{ github.ref_type }}" == "tag" && [[ ! "${{ env.VERSION }}" =~ rc ]] ) + ]; then + PUSHLATEST="true" + else + PUSHLATEST="false" + fi + echo "PUSHLATEST=$PUSHLATEST" >> $GITHUB_ENV + + - name: Set Tags + id: set-tags + run: | + TAGS="${{ env.REPONAME }}/lightningd:${{ env.VERSION }}" + if [ "${{ env.PUSHLATEST }}" == "true" ]; then + TAGS="$TAGS,${{ env.REPONAME }}/lightningd:latest" + fi + echo "TAGS=$TAGS" >> $GITHUB_ENV + + - name: Print GitHub Ref Values + run: | + echo "GITHUB REF TYPE: ${{ github.ref_type }}" + echo "GITHUB REF NAME: ${{ github.ref_name }}" + echo "EVENT INPUT VERSION: ${{ github.event.inputs.version }}" + echo "EVENT INPUT REPO: ${{ github.event.inputs.repository-name }}" + echo "EVENT INPUT PLATFORMS: ${{ github.event.inputs.platforms-to-build }}" + echo "EVENT INPUT PUSH LATEST: ${{ github.event.inputs.push-latest }}" + echo "VERSION ENV: ${{ env.VERSION }}" + echo "REPO NAME: ${{ env.REPONAME }}" + echo "PLATFORMS: ${{ env.PLATFORMS }}" + echo "PUSH LATEST: ${{ env.PUSHLATEST }}" + echo "TAGS: ${{ env.TAGS }}" + - name: Build and push Docker image uses: docker/build-push-action@v5 with: context: . push: true - platforms: linux/amd64,linux/arm64,linux/arm/v7 - tags: | - elementsproject/lightningd:$VERSION - elementsproject/lightningd:latest + platforms: ${{ env.PLATFORMS }} + tags: ${{ env.TAGS }} diff --git a/Dockerfile b/Dockerfile index 2aab878cdc13..e8dd59f8934f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,28 +1,29 @@ # This Dockerfile is used by buildx to build ARM64, AMD64, and ARM32 Docker images from an AMD64 host. # To speed up the build process, we are cross-compiling rather than relying on QEMU. # There are four main stages: -# * downloader: Downloads specific binaries needed for c-lightning for each architecture. +# * downloader: Downloads specific binaries needed for core lightning for each architecture. # * builder: Cross-compiles for each architecture. -# * builder-python: Builds Python dependencies for cln-rest with QEMU. +# * builder-python: Builds Python dependencies for clnrest & wss-proxy with QEMU. # * final: Creates the runtime image. +ARG DEFAULT_TARGETPLATFORM="linux/amd64" ARG BASE_DISTRO="debian:bullseye-slim" -FROM --platform=$BUILDPLATFORM ${BASE_DISTRO} as base-downloader +FROM --platform=$BUILDPLATFORM ${BASE_DISTRO} AS base-downloader RUN set -ex \ && apt-get update \ && apt-get install -qq --no-install-recommends ca-certificates dirmngr wget qemu-user-static binfmt-support -FROM base-downloader as base-downloader-linux-amd64 +FROM base-downloader AS base-downloader-linux-amd64 ENV TARBALL_ARCH_FINAL=x86_64-linux-gnu -FROM base-downloader as base-downloader-linux-arm64 +FROM base-downloader AS base-downloader-linux-arm64 ENV TARBALL_ARCH_FINAL=aarch64-linux-gnu -FROM base-downloader as base-downloader-linux-arm +FROM base-downloader AS base-downloader-linux-arm ENV TARBALL_ARCH_FINAL=arm-linux-gnueabihf -FROM base-downloader-${TARGETOS}-${TARGETARCH} as downloader +FROM base-downloader-${TARGETOS}-${TARGETARCH} AS downloader RUN set -ex \ && apt-get update \ @@ -54,7 +55,7 @@ RUN mkdir /opt/litecoin && cd /opt/litecoin \ && tar -xzvf litecoin.tar.gz litecoin-$LITECOIN_VERSION/bin/litecoin-cli --strip-components=1 --exclude=*-qt \ && rm litecoin.tar.gz -FROM --platform=linux/amd64 ${BASE_DISTRO} as base-builder +FROM --platform=${DEFAULT_TARGETPLATFORM} ${BASE_DISTRO} AS base-builder RUN apt-get update -qq && \ apt-get install -qq -y --no-install-recommends \ autoconf \ @@ -86,6 +87,7 @@ RUN apt-get update -qq && \ unzip \ tclsh +ENV PATH="/root/.local/bin:$PATH" ENV PYTHON_VERSION=3 RUN curl -sSL https://install.python-poetry.org | python3 - RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1 @@ -100,14 +102,13 @@ RUN git clone --recursive /tmp/lightning . && \ git checkout $(git --work-tree=/tmp/lightning --git-dir=/tmp/lightning/.git rev-parse HEAD) # Do not build python plugins (clnrest & wss-proxy) here, python doesn't support cross compilation. -RUN sed -i '/^clnrest\|^wss-proxy/d' pyproject.toml && \ - /root/.local/bin/poetry export -o requirements.txt --without-hashes +RUN sed -i '/^clnrest\|^wss-proxy/d' pyproject.toml && poetry export -o requirements.txt --without-hashes RUN pip3 install -r requirements.txt && pip3 cache purge WORKDIR / -FROM base-builder as base-builder-linux-amd64 +FROM base-builder AS base-builder-linux-amd64 -FROM base-builder as base-builder-linux-arm64 +FROM base-builder AS base-builder-linux-arm64 ENV target_host=aarch64-linux-gnu \ target_host_rust=aarch64-unknown-linux-gnu \ target_host_qemu=qemu-aarch64-static @@ -133,7 +134,7 @@ ENV \ ZLIB_CONFIG="--prefix=${QEMU_LD_PREFIX}" \ SQLITE_CONFIG="--host=${target_host} --prefix=$QEMU_LD_PREFIX" -FROM base-builder as base-builder-linux-arm +FROM base-builder AS base-builder-linux-arm ENV target_host=arm-linux-gnueabihf \ target_host_rust=armv7-unknown-linux-gnueabihf \ @@ -160,7 +161,7 @@ ENV \ ZLIB_CONFIG="--prefix=${QEMU_LD_PREFIX}" \ SQLITE_CONFIG="--host=${target_host} --prefix=$QEMU_LD_PREFIX" -FROM base-builder-${TARGETOS}-${TARGETARCH} as builder +FROM base-builder-${TARGETOS}-${TARGETARCH} AS builder ENV LIGHTNINGD_VERSION=master @@ -179,7 +180,7 @@ RUN unzip sqlite.zip \ && make install && cd .. && rm sqlite.zip && rm -rf sqlite-* ENV RUST_PROFILE=release -ENV PATH=$PATH:/root/.cargo/bin/ +ENV PATH="/root/.cargo/bin:/root/.local/bin:$PATH" RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ${RUSTUP_INSTALL_OPTS} RUN rustup toolchain install stable --component rustfmt --allow-downgrade @@ -196,15 +197,20 @@ RUN ( ! [ "${target_host}" = "arm-linux-gnueabihf" ] ) || \ # Ensure that the desired grpcio-tools & protobuf versions are installed # https://github.com/ElementsProject/lightning/pull/7376#issuecomment-2161102381 -RUN /root/.local/bin/poetry lock --no-update && \ - /root/.local/bin/poetry install +RUN poetry lock --no-update && poetry install -RUN ./configure --prefix=/tmp/lightning_install --enable-static && \ - make && \ - /root/.local/bin/poetry run make install +RUN ./configure --enable-static && make && poetry run make install + +# Export the requirements for the plugins so we can install them in builder-python stage +WORKDIR /opt/lightningd/plugins/clnrest +RUN poetry export -o requirements.txt --without-hashes +WORKDIR /opt/lightningd/plugins/wss-proxy +RUN poetry export -o requirements.txt --without-hashes +WORKDIR /opt/lightningd +RUN echo 'RUSTUP_INSTALL_OPTS="${RUSTUP_INSTALL_OPTS}"' > /tmp/rustup_install_opts.txt # We need to build python plugins on the target's arch because python doesn't support cross build -FROM ${BASE_DISTRO} as builder-python +FROM ${BASE_DISTRO} AS builder-python RUN apt-get update -qq && \ apt-get install -qq -y --no-install-recommends \ git \ @@ -222,27 +228,29 @@ RUN apt-get update -qq && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -RUN curl -sSL https://install.python-poetry.org | python3 - RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1 - ENV PYTHON_VERSION=3 -WORKDIR /opt/lightningd +RUN pip3 install --upgrade pip setuptools wheel + +# Copy rustup_install_opts.txt file from builder +COPY --from=builder /tmp/rustup_install_opts.txt /tmp/rustup_install_opts.txt +# Setup ENV $RUSTUP_INSTALL_OPTS for this stage +RUN export $(cat /tmp/rustup_install_opts.txt) +ENV PATH="/root/.cargo/bin:$PATH" +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ${RUSTUP_INSTALL_OPTS} -COPY plugins/clnrest/pyproject.toml plugins/clnrest/pyproject.toml -COPY plugins/wss-proxy/pyproject.toml plugins/wss-proxy/pyproject.toml +WORKDIR /opt/lightningd/plugins/clnrest +COPY --from=builder /opt/lightningd/plugins/clnrest/requirements.txt . +RUN pip3 install -r requirements.txt -RUN cd plugins/clnrest && \ - /root/.local/bin/poetry export -o requirements.txt --without-hashes && \ - pip3 install -r requirements.txt && \ - cd /opt/lightningd +WORKDIR /opt/lightningd/plugins/wss-proxy +COPY --from=builder /opt/lightningd/plugins/wss-proxy/requirements.txt . +RUN pip3 install -r requirements.txt +RUN pip3 cache purge -RUN cd plugins/wss-proxy && \ - /root/.local/bin/poetry export -o requirements.txt --without-hashes && \ - pip3 install -r requirements.txt && \ - cd /opt/lightningd && \ - pip3 cache purge +WORKDIR /opt/lightningd -FROM ${BASE_DISTRO} as final +FROM ${BASE_DISTRO} AS final RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -265,7 +273,9 @@ RUN mkdir $LIGHTNINGD_DATA && \ touch $LIGHTNINGD_DATA/config VOLUME [ "/root/.lightning" ] -COPY --from=builder /tmp/lightning_install/ /usr/local/ +COPY --from=builder /usr/local/bin/lightning-cli /usr/local/bin/lightning-hsmtool /usr/local/bin/lightningd /usr/local/bin/reckless /usr/local/bin/ +COPY --from=builder /usr/local/libexec/ /usr/local/libexec +COPY --from=builder /usr/local/share/ /usr/local/share COPY --from=builder-python /usr/local/lib/python3.9/dist-packages/ /usr/local/lib/python3.9/dist-packages/ COPY --from=downloader /opt/bitcoin/bin /usr/bin COPY --from=downloader /opt/litecoin/bin /usr/bin