diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 000000000000..f04f271478c6 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,34 @@ +name: Docker packaging +on: + push: + tags: + - 'basedon-*' + +jobs: + main: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - + name: Environment variables + run: env + - + name: Create images + env: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_PASS: ${{ secrets.DOCKERHUB_PASS }} + DOCKERHUB_REPO: ${{ vars.DOCKERHUB_REPO }} + shell: bash + run: | + LATEST_TAG=${GITHUB_REF#refs/tags/} + LATEST_TAG=${LATEST_TAG:8} #trim "basedon-" from tag + echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USER" --password-stdin + docker buildx create --use + DOCKER_BUILDX_OPTS="--platform linux/amd64,linux/arm64,linux/arm/v7 --push" + docker buildx build $DOCKER_BUILDX_OPTS -t $DOCKERHUB_REPO:$LATEST_TAG . \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index a5ef573f0630..e61d0b04c919 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,34 @@ -# This dockerfile is meant to compile a core-lightning x64 image -# It is using multi stage build: -# * downloader: Download litecoin/bitcoin and qemu binaries needed for core-lightning -# * builder: Compile core-lightning dependencies, then core-lightning itself with static linking -# * final: Copy the binaries required at runtime -# The resulting image uploaded to dockerhub will only contain what is needed for runtime. -# From the root of the repository, run "docker build -t yourimage:yourtag ." -FROM debian:bullseye-slim as downloader +# This Dockerfile is used by buildx to build ARM64, AMD64, and ARM32 Docker images from an AMD64 host. +# To speed up the build process, we are cross-compiling rather than relying on QEMU. +# There are four main stages: +# * downloader: Downloads specific binaries needed for c-lightning for each architecture. +# * builder: Cross-compiles for each architecture. +# * builder-python: Builds Python dependencies for cln-rest with QEMU. +# * final: Creates the runtime image. + +FROM debian:bullseye-slim as base-downloader + +FROM --platform=$BUILDPLATFORM debian:bullseye-slim as base-downloader +RUN set -ex \ + && apt-get update \ + && apt-get install -qq --no-install-recommends ca-certificates dirmngr wget qemu-user-static binfmt-support + +FROM --platform=$BUILDPLATFORM base-downloader as base-downloader-linux-amd64 +ENV TARBALL_ARCH_FINAL=x86_64-linux-gnu +ENV DESCHASHPLUGIN_ARCH=linux-amd64 +ENV DESCHASHPLUGIN_HASH=deadc00c68fac80b2718d92f69bf06acd8fff646228d497bbb76a4f0a12ca217 + +FROM --platform=$BUILDPLATFORM base-downloader as base-downloader-linux-arm64 +ENV TARBALL_ARCH_FINAL=aarch64-linux-gnu +ENV DESCHASHPLUGIN_ARCH=linux-arm64 +ENV DESCHASHPLUGIN_HASH=d48c3e5aede77bd9cb72d78689ce12c0327f624435cb0496b3eacb92df416363 + +FROM --platform=$BUILDPLATFORM base-downloader as base-downloader-linux-arm +ENV TARBALL_ARCH_FINAL=arm-linux-gnueabihf +ENV DESCHASHPLUGIN_ARCH=linux-arm +ENV DESCHASHPLUGIN_HASH=f7df336c72dd1674bd18ff23862a410b6a9691a3e13752264dcffa0950e21c74 + +FROM base-downloader-${TARGETOS}-${TARGETARCH} as downloader RUN set -ex \ && apt-get update \ @@ -14,9 +37,7 @@ RUN set -ex \ WORKDIR /opt -ARG BITCOIN_VERSION=22.0 -ARG TARBALL_ARCH=x86_64-linux-gnu -ENV TARBALL_ARCH_FINAL=$TARBALL_ARCH +ENV BITCOIN_VERSION=22.0 ENV BITCOIN_TARBALL bitcoin-${BITCOIN_VERSION}-${TARBALL_ARCH_FINAL}.tar.gz ENV BITCOIN_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/$BITCOIN_TARBALL ENV BITCOIN_ASC_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/SHA256SUMS @@ -39,9 +60,16 @@ RUN mkdir /opt/litecoin && cd /opt/litecoin \ && tar -xzvf litecoin.tar.gz litecoin-$LITECOIN_VERSION/bin/litecoin-cli --strip-components=1 --exclude=*-qt \ && rm litecoin.tar.gz -FROM debian:bullseye-slim as builder +ENV DESCHASHPLUGIN_URL https://github.com/nbd-wtf/invoicewithdescriptionhash/releases/download/v1.4/invoicewithdescriptionhash-v1.4-${DESCHASHPLUGIN_ARCH}.tar.gz +ENV DESCHASHPLUGIN_SHA256 ${DESCHASHPLUGIN_HASH} +RUN mkdir /opt/deschashplugin && cd /opt/deschashplugin \ + && wget -qO invoicewithdescriptionhash.tar.gz "$DESCHASHPLUGIN_URL" \ + && echo "$DESCHASHPLUGIN_SHA256 invoicewithdescriptionhash.tar.gz" | sha256sum -c - \ + && tar -xzvf invoicewithdescriptionhash.tar.gz && rm invoicewithdescriptionhash.tar.gz \ + && chmod a+x invoicewithdescriptionhash -ENV LIGHTNINGD_VERSION=master + +FROM --platform=$BUILDPLATFORM debian:bullseye-slim as base-builder RUN apt-get update -qq && \ apt-get install -qq -y --no-install-recommends \ autoconf \ @@ -68,31 +96,94 @@ RUN apt-get update -qq && \ libev-dev \ libevent-dev \ qemu-user-static \ - wget + wget \ + unzip \ + tclsh + +RUN wget -q https://zlib.net/fossils/zlib-1.2.13.tar.gz -O zlib.tar.gz && \ + wget -q https://www.sqlite.org/2019/sqlite-src-3290000.zip -O sqlite.zip + + +FROM --platform=linux/amd64 base-builder as base-builder-linux-amd64 + +FROM --platform=linux/amd64 base-builder as base-builder-linux-arm64 +ENV target_host=aarch64-linux-gnu \ + target_host_rust=aarch64-unknown-linux-gnu \ + target_host_qemu=qemu-aarch64-static -RUN wget -q https://zlib.net/fossils/zlib-1.2.13.tar.gz \ - && tar xvf zlib-1.2.13.tar.gz \ - && cd zlib-1.2.13 \ - && ./configure \ +RUN apt-get install -qq -y --no-install-recommends \ + libc6-arm64-cross \ + gcc-${target_host} \ + g++-${target_host} + +ENV AR=${target_host}-ar \ +AS=${target_host}-as \ +CC=${target_host}-gcc \ +CXX=${target_host}-g++ \ +LD=${target_host}-ld \ +STRIP=${target_host}-strip \ +QEMU_LD_PREFIX=/usr/${target_host} \ +HOST=${target_host} \ +TARGET=${target_host_rust} \ +RUSTUP_INSTALL_OPTS="--target ${target_host_rust} --default-host ${target_host_rust}" \ +PKG_CONFIG_PATH="/usr/${target_host}/lib/pkgconfig" + +ENV \ +ZLIB_CONFIG="--prefix=${QEMU_LD_PREFIX}" \ +SQLITE_CONFIG="--host=${target_host} --prefix=$QEMU_LD_PREFIX" + +FROM --platform=linux/amd64 base-builder as base-builder-linux-arm + +ENV target_host=arm-linux-gnueabihf \ + target_host_rust=armv7-unknown-linux-gnueabihf \ + target_host_qemu=qemu-arm-static + +RUN apt-get install -qq -y --no-install-recommends \ + libc6-armhf-cross \ + gcc-${target_host} \ + g++-${target_host} + +ENV AR=${target_host}-ar \ +AS=${target_host}-as \ +CC=${target_host}-gcc \ +CXX=${target_host}-g++ \ +LD=${target_host}-ld \ +STRIP=${target_host}-strip \ +QEMU_LD_PREFIX=/usr/${target_host} \ +HOST=${target_host} \ +TARGET=${target_host_rust} \ +RUSTUP_INSTALL_OPTS="--target ${target_host_rust} --default-host ${target_host_rust}" \ +PKG_CONFIG_PATH="/usr/${target_host}/lib/pkgconfig" + +ENV \ +ZLIB_CONFIG="--prefix=${QEMU_LD_PREFIX}" \ +SQLITE_CONFIG="--host=${target_host} --prefix=$QEMU_LD_PREFIX" + +FROM --platform=$BUILDPLATFORM base-builder-${TARGETOS}-${TARGETARCH} as builder + +ENV LIGHTNINGD_VERSION=master + +RUN mkdir zlib && tar xvf zlib.tar.gz -C zlib --strip-components=1 \ + && cd zlib \ + && ./configure ${ZLIB_CONFIG} \ && make \ && make install && cd .. && \ - rm zlib-1.2.13.tar.gz && \ - rm -rf zlib-1.2.13 - -RUN apt-get install -y --no-install-recommends unzip tclsh \ - && wget -q https://www.sqlite.org/2019/sqlite-src-3290000.zip \ - && unzip sqlite-src-3290000.zip \ - && cd sqlite-src-3290000 \ - && ./configure --enable-static --disable-readline --disable-threadsafe --disable-load-extension \ + rm zlib.tar.gz && \ + rm -rf zlib + +RUN unzip sqlite.zip \ + && cd sqlite-* \ + && ./configure --enable-static --disable-readline --disable-threadsafe --disable-load-extension ${SQLITE_CONFIG} \ && make \ - && make install && cd .. && rm sqlite-src-3290000.zip && rm -rf sqlite-src-3290000 + && make install && cd .. && rm sqlite.zip && rm -rf sqlite-* USER root ENV RUST_PROFILE=release ENV PATH=$PATH:/root/.cargo/bin/ -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ${RUSTUP_INSTALL_OPTS} RUN rustup toolchain install stable --component rustfmt --allow-downgrade +COPY --from=downloader /usr/bin/${target_host_qemu} /usr/bin/${target_host_qemu} WORKDIR /opt/lightningd COPY . /tmp/lightning @@ -111,10 +202,47 @@ RUN pip3 install grpcio-tools RUN /root/.local/bin/poetry export -o requirements.txt --without-hashes --with dev RUN pip3 install -r requirements.txt +# If cross-compiling, need to tell it to cargo. +RUN ( ! [ -n "${target_host}" ] ) || \ + (mkdir -p .cargo && echo "[target.${target_host_rust}]\nlinker = \"${target_host}-gcc\"" > .cargo/config) + +# Weird errors with cargo for cln-grpc on arm7 https://github.com/ElementsProject/lightning/issues/6596 +RUN ( ! [ "${target_host}" = "arm-linux-gnueabihf" ] ) || \ + (sed -i '/documentation = "https:\/\/docs.rs\/cln-grpc"/a include = ["**\/*.*"]' cln-grpc/Cargo.toml) + RUN ./configure --prefix=/tmp/lightning_install --enable-static && \ make && \ /root/.local/bin/poetry run make install +# We need to build cln-rest on the target's arch because python doesn't support cross build +FROM debian:bullseye-slim as builder-python +RUN apt-get update -qq && \ + apt-get install -qq -y --no-install-recommends \ + git \ + curl \ + libtool \ + pkg-config \ + autoconf \ + automake \ + build-essential \ + libffi-dev \ + libssl-dev \ + python3.9 \ + python3-dev \ + python3-pip + +ENV PATH=$PATH:/root/.cargo/bin/ +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +RUN rustup toolchain install stable --component rustfmt --allow-downgrade + +WORKDIR /opt/lightningd +COPY . /tmp/lightning +RUN git clone --recursive /tmp/lightning . && \ + git checkout $(git --work-tree=/tmp/lightning --git-dir=/tmp/lightning/.git rev-parse HEAD) + +ENV PYTHON_VERSION=3 +RUN pip3 install -r plugins/clnrest/requirements.txt + FROM debian:bullseye-slim as final RUN apt-get update && \ @@ -122,9 +250,9 @@ RUN apt-get update && \ tini \ socat \ inotify-tools \ + jq \ python3.9 \ python3-pip \ - qemu-user-static \ libpq5 && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -135,13 +263,17 @@ ENV LIGHTNINGD_PORT=9735 ENV LIGHTNINGD_NETWORK=bitcoin RUN mkdir $LIGHTNINGD_DATA && \ + mkdir /etc/bundledplugins && \ + mkdir $LIGHTNINGD_DATA/plugins && \ touch $LIGHTNINGD_DATA/config VOLUME [ "/root/.lightning" ] COPY --from=builder /tmp/lightning_install/ /usr/local/ -COPY --from=builder /usr/local/lib/python3.9/dist-packages/ /usr/local/lib/python3.9/dist-packages/ +COPY --from=builder-python /usr/local/lib/python3.9/dist-packages/ /usr/local/lib/python3.9/dist-packages/ COPY --from=downloader /opt/bitcoin/bin /usr/bin COPY --from=downloader /opt/litecoin/bin /usr/bin +COPY --from=downloader /opt/deschashplugin $LIGHTNINGD_DATA/plugins +COPY --from=downloader /opt/deschashplugin /etc/bundledplugins COPY tools/docker-entrypoint.sh entrypoint.sh EXPOSE 9735 9835 diff --git a/contrib/msggen/msggen/__main__.py b/contrib/msggen/msggen/__main__.py index ee1a5d1df0d9..71758e010cd2 100644 --- a/contrib/msggen/msggen/__main__.py +++ b/contrib/msggen/msggen/__main__.py @@ -13,7 +13,7 @@ logging.basicConfig( - level=logging.DEBUG, + level=logging.ERROR, format="%(asctime)s [%(levelname)s] %(message)s", handlers=[ logging.StreamHandler() diff --git a/plugins/bcli.c b/plugins/bcli.c index e04159b44377..fee4aed57617 100644 --- a/plugins/bcli.c +++ b/plugins/bcli.c @@ -969,6 +969,7 @@ static void wait_and_check_bitcoind(struct plugin *p) pid_t child; const char **cmd = gather_args(bitcoind, "getnetworkinfo", NULL); bool printed = false; + bool isWarmup = false; char *output = NULL; for (;;) { @@ -1004,17 +1005,20 @@ static void wait_and_check_bitcoind(struct plugin *p) /* bitcoin/src/rpc/protocol.h: * RPC_IN_WARMUP = -28, //!< Client still warming up */ - if (WEXITSTATUS(status) != 28) { - if (WEXITSTATUS(status) == 1) - bitcoind_failure(p, "Could not connect to bitcoind using" - " bitcoin-cli. Is bitcoind running?"); - bitcoind_failure(p, tal_fmt(bitcoind, "%s exited with code %i: %s", - cmd[0], WEXITSTATUS(status), output)); - } + isWarmup = WEXITSTATUS(status) == 28; if (!printed) { - plugin_log(p, LOG_UNUSUAL, - "Waiting for bitcoind to warm up..."); + if (isWarmup) + { + plugin_log(p, LOG_UNUSUAL, + "Waiting for bitcoind to warm up..."); + } + else + { + plugin_log(p, LOG_UNUSUAL, + tal_fmt(bitcoind, "%s exited with code %i: %s... retrying", + cmd[0], WEXITSTATUS(status), output)); + } printed = true; } sleep(1); diff --git a/tools/docker-entrypoint.sh b/tools/docker-entrypoint.sh index 8d7bbfd2d920..4e2fe97930cc 100755 --- a/tools/docker-entrypoint.sh +++ b/tools/docker-entrypoint.sh @@ -2,19 +2,77 @@ : "${EXPOSE_TCP:=false}" -networkdatadir="${LIGHTNINGD_DATA}/${LIGHTNINGD_NETWORK}" +cat <<-EOF > "$LIGHTNINGD_DATA/config" +${LIGHTNINGD_OPT} +bind-addr=0.0.0.0:${LIGHTNINGD_PORT} +EOF + +LIGHTNINGD_NETWORK_NAME="" + +if [ "$LIGHTNINGD_CHAIN" == "btc" ] && [ "$LIGHTNINGD_NETWORK" == "mainnet" ]; then + LIGHTNINGD_NETWORK_NAME="bitcoin" +elif [ "$LIGHTNINGD_CHAIN" == "btc" ] && [ "$LIGHTNINGD_NETWORK" == "testnet" ]; then + LIGHTNINGD_NETWORK_NAME="testnet" +elif [ "$LIGHTNINGD_CHAIN" == "btc" ] && [ "$LIGHTNINGD_NETWORK" == "regtest" ]; then + LIGHTNINGD_NETWORK_NAME="regtest" +elif [ "$LIGHTNINGD_CHAIN" == "ltc" ] && [ "$LIGHTNINGD_NETWORK" == "mainnet" ]; then + LIGHTNINGD_NETWORK_NAME="litecoin" +elif [ "$LIGHTNINGD_CHAIN" == "ltc" ] && [ "$LIGHTNINGD_NETWORK" == "testnet" ]; then + LIGHTNINGD_NETWORK_NAME="litecoin-testnet" +else + echo "Invalid combinaion of LIGHTNINGD_NETWORK and LIGHTNINGD_CHAIN. LIGHTNINGD_CHAIN should be btc or ltc. LIGHTNINGD_NETWORK should be mainnet, testnet or regtest." + echo "ltc regtest is not supported" + exit +fi + +echo "network=$LIGHTNINGD_NETWORK_NAME" >> "$LIGHTNINGD_DATA/config" +echo "network=$LIGHTNINGD_NETWORK_NAME added in $LIGHTNINGD_DATA/config" + +if [[ "${LIGHTNINGD_ANNOUNCEADDR}" ]]; then + echo "announce-addr=$LIGHTNINGD_ANNOUNCEADDR:${LIGHTNINGD_PORT}" >> "$LIGHTNINGD_DATA/config" +fi + +if [[ "${LIGHTNINGD_ALIAS}" ]]; then + # This allow to strip this parameter if LND_ALIGHTNINGD_ALIASLIAS is empty or null, and truncate it + LIGHTNINGD_ALIAS="$(echo "$LIGHTNINGD_ALIAS" | cut -c -32)" + echo "alias=$LIGHTNINGD_ALIAS" >> "$LIGHTNINGD_DATA/config" + echo "alias=$LIGHTNINGD_ALIAS added to $LIGHTNINGD_DATA/config" +fi + +if [[ "${LIGHTNINGD_READY_FILE}" ]]; then + echo "Waiting $LIGHTNINGD_READY_FILE to be created..." + while [ ! -f "$LIGHTNINGD_READY_FILE" ]; do sleep 1; done + echo "The chain is fully synched" +fi + +if [[ "${LIGHTNINGD_HIDDENSERVICE_HOSTNAME_FILE}" ]]; then + echo "Waiting $LIGHTNINGD_HIDDENSERVICE_HOSTNAME_FILE to be created by tor..." + while [ ! -f "$LIGHTNINGD_HIDDENSERVICE_HOSTNAME_FILE" ]; do sleep 1; done + HIDDENSERVICE_ONION="$(head -n 1 "$LIGHTNINGD_HIDDENSERVICE_HOSTNAME_FILE"):${LIGHTNINGD_PORT}" + echo "announce-addr=$HIDDENSERVICE_ONION" >> "$LIGHTNINGD_DATA/config" + echo "announce-addr=$HIDDENSERVICE_ONION added to $LIGHTNINGD_DATA/config" +fi + +if ! grep -q "^rpc-file=" "$LIGHTNINGD_DATA/config"; then + echo "rpc-file=$LIGHTNINGD_DATA/lightning-rpc" >> "$LIGHTNINGD_DATA/config" + echo "rpc-file=$LIGHTNINGD_DATA/lightning-rpc added to $LIGHTNINGD_DATA/config" +fi + +echo "Installing bundled plugins" +mkdir -p "$LIGHTNINGD_DATA/plugins" +cp -u /etc/bundledplugins/* $LIGHTNINGD_DATA/plugins/ set -m -lightningd --network="${LIGHTNINGD_NETWORK}" "$@" & +lightningd "$@" & echo "Core-Lightning starting" while read -r i; do if [ "$i" = "lightning-rpc" ]; then break; fi; done \ - < <(inotifywait -e create,open --format '%f' --quiet "${networkdatadir}" --monitor) + < <(inotifywait -e create,open --format '%f' --quiet "$LIGHTNINGD_DATA" --monitor) if [ "$EXPOSE_TCP" == "true" ]; then echo "Core-Lightning started, RPC available on port $LIGHTNINGD_RPC_PORT" - socat "TCP4-listen:$LIGHTNINGD_RPC_PORT,fork,reuseaddr" "UNIX-CONNECT:${networkdatadir}/lightning-rpc" & + socat "TCP4-listen:$LIGHTNINGD_RPC_PORT,fork,reuseaddr" "UNIX-CONNECT:$LIGHTNINGD_DATA/lightning-rpc" & fi # Now run any scripts which exist in the lightning-poststart.d directory @@ -24,4 +82,36 @@ if [ -d "$LIGHTNINGD_DATA"/lightning-poststart.d ]; then done fi +RUNE_PATH="$LIGHTNINGD_DATA/rune.env" +if [ -f "$RUNE_PATH" ]; then + source "$RUNE_PATH" + matches=$(lightning-cli showrunes | jq --arg rune "$LIGHTNING_RUNE" --arg unique_id "$UNIQUE_ID" \ + '.runes[] | select(.rune == $rune and .unique_id == $unique_id)') + if [[ -n $matches ]]; then + echo "Rune already created" + else + LIGHTNING_RUNE="" + UNIQUE_ID="" + echo "Rune not found, re-creating..." + fi +fi + +if ! [[ "$LIGHTNING_RUNE" ]]; then + echo "Creating rune..." + RUNE_RESPONSE=$(lightning-cli createrune 'restrictions=[["For Applications#"]]') + LIGHTNING_RUNE=$(echo "$RUNE_RESPONSE" | jq -r '.rune') + UNIQUE_ID=$(echo "$RUNE_RESPONSE" | jq -r '.unique_id') + if [[ "$LIGHTNING_RUNE" ]]; then + echo "LIGHTNING_RUNE=${LIGHTNING_RUNE}" > "$RUNE_PATH" + echo "UNIQUE_ID=${UNIQUE_ID}" >> "$RUNE_PATH" + echo "Rune created" + source "$RUNE_PATH" + fi +fi + +if ! [[ "$LIGHTNING_RUNE" ]]; then + echo "Error while creating a rune..." + echo "$RUNE_RESPONSE" +fi + fg %-