GitHub docker testing

.github/workflows/docker-release.yml

@@ -3,13 +3,28 @@ name: Build and push multi-platform docker images
 on:
   push:
     tags:
-      - '^v[0-9]{2}\.[0-9]{2}(\.[0-9]{1,2})?$'
+      - '^v[0-9]{2}\.[0-9]{2}(\.[0-9]{1,2})?([a-zA-Z0-9]*)?$'
   workflow_dispatch:
     inputs:
       version:
         description: 'Release version'
         required: true
 
+      repository-name:
+        description: 'Docker repository name'
+        default: 'elementsproject'
+        required: false
+
+      platforms-to-build:
+        description: 'List of platforms to build'
+        default: 'linux/amd64,linux/arm64,linux/arm/v7'
+        required: false
+
+      push-latest:
+        description: 'Push the latest tag also?'
+        default: 'false'
+        required: false
+
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -30,8 +45,8 @@ jobs:
         username: ${{ secrets.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_PASSWORD }}
 
-    - name: Set up version
-      id: set-version
+    - name: Set up values
+      id: set-values
       run: |
         if [ "${{ github.event.inputs.version }}" != "" ]; then
           VERSION=${{ github.event.inputs.version }}
@@ -42,13 +57,58 @@ jobs:
           exit 1
         fi
         echo "VERSION=$VERSION" >> $GITHUB_ENV
-
+
+        if [ "${{ github.event.inputs.repository-name }}" != "" ]; then
+          REPONAME=${{ github.event.inputs.repository-name }}
+        else
+          REPONAME="elementsproject"
+        fi
+        echo "REPONAME=$REPONAME" >> $GITHUB_ENV
+
+        if [ "${{ github.event.inputs.platforms-to-build }}" != "" ]; then
+          PLATFORMS=${{ github.event.inputs.platforms-to-build }}
+        else
+          PLATFORMS="linux/amd64,linux/arm64,linux/arm/v7"
+        fi
+        echo "PLATFORMS=$PLATFORMS" >> $GITHUB_ENV
+
+        if [
+          "${{ github.event.inputs.push-latest }}" == "true" ||
+          ( "${{ github.ref_type }}" == "tag" && [[ ! "${{ env.VERSION }}" =~ rc ]] )
+        ]; then
+          PUSHLATEST="true"
+        else
+          PUSHLATEST="false"
+        fi
+        echo "PUSHLATEST=$PUSHLATEST" >> $GITHUB_ENV
+
+    - name: Set Tags
+      id: set-tags
+      run: |
+        TAGS="${{ env.REPONAME }}/lightningd:${{ env.VERSION }}"
+        if [ "${{ env.PUSHLATEST }}" == "true" ]; then
+          TAGS="$TAGS,${{ env.REPONAME }}/lightningd:latest"
+        fi
+        echo "TAGS=$TAGS" >> $GITHUB_ENV
+
+    - name: Print GitHub Ref Values
+      run: |
+        echo "GITHUB REF TYPE: ${{ github.ref_type }}"
+        echo "GITHUB REF NAME: ${{ github.ref_name }}"
+        echo "EVENT INPUT VERSION: ${{ github.event.inputs.version }}"
+        echo "EVENT INPUT REPO: ${{ github.event.inputs.repository-name }}"
+        echo "EVENT INPUT PLATFORMS: ${{ github.event.inputs.platforms-to-build }}"
+        echo "EVENT INPUT PUSH LATEST: ${{ github.event.inputs.push-latest }}"
+        echo "VERSION ENV: ${{ env.VERSION }}"
+        echo "REPO NAME: ${{ env.REPONAME }}"
+        echo "PLATFORMS: ${{ env.PLATFORMS }}"
+        echo "PUSH LATEST: ${{ env.PUSHLATEST }}"
+        echo "TAGS: ${{ env.TAGS }}"
+
     - name: Build and push Docker image
       uses: docker/build-push-action@v5
       with:
         context: .
         push: true
-        platforms: linux/amd64,linux/arm64,linux/arm/v7
-        tags: |
-          elementsproject/lightningd:$VERSION
-          elementsproject/lightningd:latest
+        platforms: ${{ env.PLATFORMS }}
+        tags: ${{ env.TAGS }}

Dockerfile

@@ -1,28 +1,29 @@
 # This Dockerfile is used by buildx to build ARM64, AMD64, and ARM32 Docker images from an AMD64 host.
 # To speed up the build process, we are cross-compiling rather than relying on QEMU.
 # There are four main stages:
-# * downloader: Downloads specific binaries needed for c-lightning for each architecture.
+# * downloader: Downloads specific binaries needed for core lightning for each architecture.
 # * builder: Cross-compiles for each architecture.
-# * builder-python: Builds Python dependencies for cln-rest with QEMU.
+# * builder-python: Builds Python dependencies for clnrest & wss-proxy with QEMU.
 # * final: Creates the runtime image.
 
+ARG DEFAULT_TARGETPLATFORM="linux/amd64"
 ARG BASE_DISTRO="debian:bullseye-slim"
 
-FROM --platform=$BUILDPLATFORM ${BASE_DISTRO} as base-downloader
+FROM --platform=$BUILDPLATFORM ${BASE_DISTRO} AS base-downloader
 RUN set -ex \
 	&& apt-get update \
 	&& apt-get install -qq --no-install-recommends ca-certificates dirmngr wget qemu-user-static binfmt-support
 
-FROM base-downloader as base-downloader-linux-amd64
+FROM base-downloader AS base-downloader-linux-amd64
 ENV TARBALL_ARCH_FINAL=x86_64-linux-gnu
 
-FROM base-downloader as base-downloader-linux-arm64
+FROM base-downloader AS base-downloader-linux-arm64
 ENV TARBALL_ARCH_FINAL=aarch64-linux-gnu
 
-FROM base-downloader as base-downloader-linux-arm
+FROM base-downloader AS base-downloader-linux-arm
 ENV TARBALL_ARCH_FINAL=arm-linux-gnueabihf
 
-FROM base-downloader-${TARGETOS}-${TARGETARCH} as downloader
+FROM base-downloader-${TARGETOS}-${TARGETARCH} AS downloader
 
 RUN set -ex \
 	&& apt-get update \
@@ -54,7 +55,7 @@ RUN mkdir /opt/litecoin && cd /opt/litecoin \
     && tar -xzvf litecoin.tar.gz litecoin-$LITECOIN_VERSION/bin/litecoin-cli --strip-components=1 --exclude=*-qt \
     && rm litecoin.tar.gz
 
-FROM --platform=linux/amd64 ${BASE_DISTRO} as base-builder
+FROM --platform=${DEFAULT_TARGETPLATFORM} ${BASE_DISTRO} AS base-builder
 RUN apt-get update -qq && \
     apt-get install -qq -y --no-install-recommends \
         autoconf \
@@ -86,6 +87,7 @@ RUN apt-get update -qq && \
         unzip \
         tclsh
 
+ENV PATH="/root/.local/bin:$PATH"
 ENV PYTHON_VERSION=3
 RUN curl -sSL https://install.python-poetry.org | python3 -
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1
@@ -100,14 +102,13 @@ RUN git clone --recursive /tmp/lightning . && \
     git checkout $(git --work-tree=/tmp/lightning --git-dir=/tmp/lightning/.git rev-parse HEAD)
 
 # Do not build python plugins (clnrest & wss-proxy) here, python doesn't support cross compilation.
-RUN sed -i '/^clnrest\|^wss-proxy/d' pyproject.toml && \
-    /root/.local/bin/poetry export -o requirements.txt --without-hashes
+RUN sed -i '/^clnrest\|^wss-proxy/d' pyproject.toml && poetry export -o requirements.txt --without-hashes
 RUN pip3 install -r requirements.txt && pip3 cache purge
 WORKDIR /
 
-FROM base-builder as base-builder-linux-amd64
+FROM base-builder AS base-builder-linux-amd64
 
-FROM base-builder as base-builder-linux-arm64
+FROM base-builder AS base-builder-linux-arm64
 ENV target_host=aarch64-linux-gnu \
     target_host_rust=aarch64-unknown-linux-gnu \
     target_host_qemu=qemu-aarch64-static
@@ -133,7 +134,7 @@ ENV \
 ZLIB_CONFIG="--prefix=${QEMU_LD_PREFIX}" \
 SQLITE_CONFIG="--host=${target_host} --prefix=$QEMU_LD_PREFIX"
 
-FROM base-builder as base-builder-linux-arm
+FROM base-builder AS base-builder-linux-arm
 
 ENV target_host=arm-linux-gnueabihf \
     target_host_rust=armv7-unknown-linux-gnueabihf \
@@ -160,7 +161,7 @@ ENV \
 ZLIB_CONFIG="--prefix=${QEMU_LD_PREFIX}" \
 SQLITE_CONFIG="--host=${target_host} --prefix=$QEMU_LD_PREFIX"
 
-FROM base-builder-${TARGETOS}-${TARGETARCH} as builder
+FROM base-builder-${TARGETOS}-${TARGETARCH} AS builder
 
 ENV LIGHTNINGD_VERSION=master
 
@@ -179,7 +180,7 @@ RUN unzip sqlite.zip \
     && make install && cd .. && rm sqlite.zip && rm -rf sqlite-*
 
 ENV RUST_PROFILE=release
-ENV PATH=$PATH:/root/.cargo/bin/
+ENV PATH="/root/.cargo/bin:/root/.local/bin:$PATH"
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ${RUSTUP_INSTALL_OPTS}
 RUN rustup toolchain install stable --component rustfmt --allow-downgrade
 
@@ -196,15 +197,20 @@ RUN ( ! [ "${target_host}" = "arm-linux-gnueabihf" ] ) || \
 
 # Ensure that the desired grpcio-tools & protobuf versions are installed
 # https://github.com/ElementsProject/lightning/pull/7376#issuecomment-2161102381
-RUN /root/.local/bin/poetry lock --no-update && \
-    /root/.local/bin/poetry install
+RUN poetry lock --no-update && poetry install
 
-RUN ./configure --prefix=/tmp/lightning_install --enable-static && \
-    make && \
-    /root/.local/bin/poetry run make install
+RUN ./configure --enable-static && make && poetry run make install
+
+# Export the requirements for the plugins so we can install them in builder-python stage
+WORKDIR /opt/lightningd/plugins/clnrest
+RUN poetry export -o requirements.txt --without-hashes
+WORKDIR /opt/lightningd/plugins/wss-proxy
+RUN poetry export -o requirements.txt --without-hashes
+WORKDIR /opt/lightningd
+RUN echo 'RUSTUP_INSTALL_OPTS="${RUSTUP_INSTALL_OPTS}"' > /tmp/rustup_install_opts.txt
 
 # We need to build python plugins on the target's arch because python doesn't support cross build
-FROM ${BASE_DISTRO} as builder-python
+FROM ${BASE_DISTRO} AS builder-python
 RUN apt-get update -qq && \
     apt-get install -qq -y --no-install-recommends \
         git \
@@ -222,27 +228,29 @@ RUN apt-get update -qq && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-RUN curl -sSL https://install.python-poetry.org | python3 -
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1
-
 ENV PYTHON_VERSION=3
-WORKDIR /opt/lightningd
+RUN pip3 install --upgrade pip setuptools wheel
+
+# Copy rustup_install_opts.txt file from builder
+COPY --from=builder /tmp/rustup_install_opts.txt /tmp/rustup_install_opts.txt
+# Setup ENV $RUSTUP_INSTALL_OPTS for this stage
+RUN export $(cat /tmp/rustup_install_opts.txt)
+ENV PATH="/root/.cargo/bin:$PATH"
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ${RUSTUP_INSTALL_OPTS}
 
-COPY plugins/clnrest/pyproject.toml plugins/clnrest/pyproject.toml
-COPY plugins/wss-proxy/pyproject.toml plugins/wss-proxy/pyproject.toml
+WORKDIR /opt/lightningd/plugins/clnrest
+COPY --from=builder /opt/lightningd/plugins/clnrest/requirements.txt .
+RUN pip3 install -r requirements.txt
 
-RUN cd plugins/clnrest && \
-    /root/.local/bin/poetry export -o requirements.txt --without-hashes && \
-    pip3 install -r requirements.txt && \
-    cd /opt/lightningd
+WORKDIR /opt/lightningd/plugins/wss-proxy
+COPY --from=builder /opt/lightningd/plugins/wss-proxy/requirements.txt .
+RUN pip3 install -r requirements.txt
+RUN pip3 cache purge
 
-RUN cd plugins/wss-proxy && \
-    /root/.local/bin/poetry export -o requirements.txt --without-hashes && \
-    pip3 install -r requirements.txt && \
-    cd /opt/lightningd && \
-    pip3 cache purge
+WORKDIR /opt/lightningd
 
-FROM ${BASE_DISTRO} as final
+FROM ${BASE_DISTRO} AS final
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
@@ -265,7 +273,9 @@ RUN mkdir $LIGHTNINGD_DATA && \
     touch $LIGHTNINGD_DATA/config
 VOLUME [ "/root/.lightning" ]
 
-COPY --from=builder /tmp/lightning_install/ /usr/local/
+COPY --from=builder /usr/local/bin/lightning-cli /usr/local/bin/lightning-hsmtool /usr/local/bin/lightningd /usr/local/bin/reckless /usr/local/bin/
+COPY --from=builder /usr/local/libexec/ /usr/local/libexec
+COPY --from=builder /usr/local/share/ /usr/local/share
 COPY --from=builder-python /usr/local/lib/python3.9/dist-packages/ /usr/local/lib/python3.9/dist-packages/
 COPY --from=downloader /opt/bitcoin/bin /usr/bin
 COPY --from=downloader /opt/litecoin/bin /usr/bin