Adapt Dockerfile for BTCPay deployment

.github/workflows/docker.yml

@@ -0,0 +1,32 @@
+name: Docker packaging
+on:
+  push:
+    tags:
+      - 'basedon-*'
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - 
+        name: Create images
+        env:
+          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+          DOCKERHUB_PASS: ${{ secrets.DOCKERHUB_PASS }}
+          DOCKERHUB_REPO: ${{ env.DOCKERHUB_REPO }}
+        shell: bash
+        run: |
+          LATEST_TAG=${GITHUB_REF#refs/tags/}
+          echo "First: $LATEST_TAG"
+          LATEST_TAG=${LATEST_TAG:8} #trim "basedon-" from tag
+          echo "Second: $LATEST_TAG"
+          echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USER" --password-stdin
+          docker buildx create --use
+          DOCKER_BUILDX_OPTS="--platform linux/amd64,linux/arm64,linux/arm/v7 --push"
+          docker buildx build $DOCKER_BUILDX_OPTS -t $DOCKERHUB_REPO:$LATEST_TAG .

Dockerfile

@@ -5,7 +5,30 @@
 # * final: Copy the binaries required at runtime
 # The resulting image uploaded to dockerhub will only contain what is needed for runtime.
 # From the root of the repository, run "docker build -t yourimage:yourtag ."
-FROM debian:bullseye-slim as downloader
+
+FROM debian:bullseye-slim as base-downloader
+
+FROM --platform=$BUILDPLATFORM debian:bullseye-slim as base-downloader
+RUN set -ex \
+	&& apt-get update \
+	&& apt-get install -qq --no-install-recommends ca-certificates dirmngr wget
+
+FROM --platform=$BUILDPLATFORM base-downloader as base-downloader-linux-amd64
+ENV TARBALL_ARCH_FINAL=x86_64-linux-gnu
+ENV DESCHASHPLUGIN_ARCH=linux-amd64
+ENV DESCHASHPLUGIN_HASH=deadc00c68fac80b2718d92f69bf06acd8fff646228d497bbb76a4f0a12ca217
+
+FROM --platform=$BUILDPLATFORM base-downloader as base-downloader-linux-arm64
+ENV TARBALL_ARCH_FINAL=aarch64-linux-gnu
+ENV DESCHASHPLUGIN_ARCH=linux-arm64
+ENV DESCHASHPLUGIN_HASH=d48c3e5aede77bd9cb72d78689ce12c0327f624435cb0496b3eacb92df416363
+
+FROM --platform=$BUILDPLATFORM base-downloader as base-downloader-linux-arm
+ENV TARBALL_ARCH_FINAL=arm-linux-gnueabihf
+ENV DESCHASHPLUGIN_ARCH=linux-arm
+ENV DESCHASHPLUGIN_HASH=f7df336c72dd1674bd18ff23862a410b6a9691a3e13752264dcffa0950e21c74
+
+FROM base-downloader-${TARGETOS}-${TARGETARCH} as downloader
 
 RUN set -ex \
 	&& apt-get update \
@@ -14,9 +37,7 @@ RUN set -ex \
 WORKDIR /opt
 
 
-ARG BITCOIN_VERSION=22.0
-ARG TARBALL_ARCH=x86_64-linux-gnu
-ENV TARBALL_ARCH_FINAL=$TARBALL_ARCH
+ENV BITCOIN_VERSION=22.0
 ENV BITCOIN_TARBALL bitcoin-${BITCOIN_VERSION}-${TARBALL_ARCH_FINAL}.tar.gz
 ENV BITCOIN_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/$BITCOIN_TARBALL
 ENV BITCOIN_ASC_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/SHA256SUMS
@@ -39,6 +60,14 @@ RUN mkdir /opt/litecoin && cd /opt/litecoin \
     && tar -xzvf litecoin.tar.gz litecoin-$LITECOIN_VERSION/bin/litecoin-cli --strip-components=1 --exclude=*-qt \
     && rm litecoin.tar.gz
 
+ENV DESCHASHPLUGIN_URL https://github.com/nbd-wtf/invoicewithdescriptionhash/releases/download/v1.4/invoicewithdescriptionhash-v1.4-${DESCHASHPLUGIN_ARCH}.tar.gz
+ENV DESCHASHPLUGIN_SHA256 ${DESCHASHPLUGIN_HASH}
+RUN mkdir /opt/deschashplugin && cd /opt/deschashplugin \
+    && wget -qO invoicewithdescriptionhash.tar.gz "$DESCHASHPLUGIN_URL" \
+    && echo "$DESCHASHPLUGIN_SHA256  invoicewithdescriptionhash.tar.gz" | sha256sum -c - \
+    && tar -xzvf invoicewithdescriptionhash.tar.gz && rm invoicewithdescriptionhash.tar.gz \
+    && chmod a+x invoicewithdescriptionhash
+
 FROM debian:bullseye-slim as builder
 
 ENV LIGHTNINGD_VERSION=master
@@ -99,6 +128,7 @@ COPY . /tmp/lightning
 RUN git clone --recursive /tmp/lightning . && \
     git checkout $(git --work-tree=/tmp/lightning --git-dir=/tmp/lightning/.git rev-parse HEAD)
 
+ARG DEVELOPER=0
 ENV PYTHON_VERSION=3
 RUN curl -sSL https://install.python-poetry.org | python3 -
 
@@ -117,6 +147,11 @@ RUN ./configure --prefix=/tmp/lightning_install --enable-static && \
 
 FROM debian:bullseye-slim as final
 
+ARG TRACE_TOOLS=false
+ENV TRACE_TOOLS=$TRACE_TOOLS
+ENV TRACE_LOCATION=/opt/traces
+VOLUME /opt/traces
+
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
       tini \
@@ -125,23 +160,36 @@ RUN apt-get update && \
       python3.9 \
       python3-pip \
       qemu-user-static \
-      libpq5 && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+      libpq5 \
+    && \
+    ( ! $TRACE_TOOLS || \
+        ( \
+            apt-get install -y --no-install-recommends perl linux-base curl ca-certificates && \
+            mkdir FlameGraph && cd FlameGraph && \
+            curl -Lo FlameGraph.tar.gz "https://github.com/brendangregg/FlameGraph/archive/v1.0.tar.gz" && \
+            tar -zxvf FlameGraph.tar.gz --strip-components=1 && rm FlameGraph.tar.gz && cd .. \
+        ) \
+    ) \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 ENV LIGHTNINGD_DATA=/root/.lightning
 ENV LIGHTNINGD_RPC_PORT=9835
 ENV LIGHTNINGD_PORT=9735
 ENV LIGHTNINGD_NETWORK=bitcoin
 
 RUN mkdir $LIGHTNINGD_DATA && \
+    mkdir /etc/bundledplugins && \
+    mkdir $LIGHTNINGD_DATA/plugins && \
     touch $LIGHTNINGD_DATA/config
 VOLUME [ "/root/.lightning" ]
 
 COPY --from=builder /tmp/lightning_install/ /usr/local/
 COPY --from=builder /usr/local/lib/python3.9/dist-packages/ /usr/local/lib/python3.9/dist-packages/
 COPY --from=downloader /opt/bitcoin/bin /usr/bin
 COPY --from=downloader /opt/litecoin/bin /usr/bin
+COPY --from=downloader /opt/deschashplugin $LIGHTNINGD_DATA/plugins
+COPY --from=downloader /opt/deschashplugin /etc/bundledplugins
 COPY tools/docker-entrypoint.sh entrypoint.sh
 
 EXPOSE 9735 9835

plugins/bcli.c

@@ -969,6 +969,7 @@ static void wait_and_check_bitcoind(struct plugin *p)
 	pid_t child;
 	const char **cmd = gather_args(bitcoind, "getnetworkinfo", NULL);
 	bool printed = false;
+	bool isWarmup = false;
 	char *output = NULL;
 
 	for (;;) {
@@ -1004,17 +1005,20 @@ static void wait_and_check_bitcoind(struct plugin *p)
 		/* bitcoin/src/rpc/protocol.h:
 		 *	RPC_IN_WARMUP = -28, //!< Client still warming up
 		 */
-		if (WEXITSTATUS(status) != 28) {
-			if (WEXITSTATUS(status) == 1)
-				bitcoind_failure(p, "Could not connect to bitcoind using"
-						    " bitcoin-cli. Is bitcoind running?");
-			bitcoind_failure(p, tal_fmt(bitcoind, "%s exited with code %i: %s",
-						    cmd[0], WEXITSTATUS(status), output));
-		}
+		isWarmup = WEXITSTATUS(status) == 28;
 
 		if (!printed) {
-			plugin_log(p, LOG_UNUSUAL,
-				   "Waiting for bitcoind to warm up...");
+			if (isWarmup)
+			{
+				plugin_log(p, LOG_UNUSUAL,
+					"Waiting for bitcoind to warm up...");
+			}
+			else
+			{
+				plugin_log(p, LOG_UNUSUAL,
+					tal_fmt(bitcoind, "%s exited with code %i: %s... retrying",
+						    cmd[0], WEXITSTATUS(status), output));
+			}
 			printed = true;
 		}
 		sleep(1);

tools/docker-entrypoint.sh

@@ -2,19 +2,111 @@
 
 : "${EXPOSE_TCP:=false}"
 
-networkdatadir="${LIGHTNINGD_DATA}/${LIGHTNINGD_NETWORK}"
+cat <<-EOF > "$LIGHTNINGD_DATA/config"
+${LIGHTNINGD_OPT}
+bind-addr=0.0.0.0:${LIGHTNINGD_PORT}
+EOF
+
+LIGHTNINGD_NETWORK_NAME=""
+
+if [ "$LIGHTNINGD_CHAIN" == "btc" ] && [ "$LIGHTNINGD_NETWORK" == "mainnet" ]; then
+    LIGHTNINGD_NETWORK_NAME="bitcoin"
+elif [ "$LIGHTNINGD_CHAIN" == "btc" ] && [ "$LIGHTNINGD_NETWORK" == "testnet" ]; then
+    LIGHTNINGD_NETWORK_NAME="testnet"
+elif [ "$LIGHTNINGD_CHAIN" == "btc" ] && [ "$LIGHTNINGD_NETWORK" == "regtest" ]; then
+    LIGHTNINGD_NETWORK_NAME="regtest"
+elif [ "$LIGHTNINGD_CHAIN" == "ltc" ] && [ "$LIGHTNINGD_NETWORK" == "mainnet" ]; then
+    LIGHTNINGD_NETWORK_NAME="litecoin"
+elif [ "$LIGHTNINGD_CHAIN" == "ltc" ] && [ "$LIGHTNINGD_NETWORK" == "testnet" ]; then
+    LIGHTNINGD_NETWORK_NAME="litecoin-testnet"
+else
+    echo "Invalid combinaion of LIGHTNINGD_NETWORK and LIGHTNINGD_CHAIN. LIGHTNINGD_CHAIN should be btc or ltc. LIGHTNINGD_NETWORK should be mainnet, testnet or regtest."
+    echo "ltc regtest is not supported"
+    exit
+fi
+
+echo "network=$LIGHTNINGD_NETWORK_NAME" >> "$LIGHTNINGD_DATA/config"
+echo "network=$LIGHTNINGD_NETWORK_NAME added in $LIGHTNINGD_DATA/config"
+
+echo "disable-plugin=clnrest.py" >> "$LIGHTNINGD_DATA/config"
+echo "disable-plugin=clnrest.py added in $LIGHTNINGD_DATA/config"
+
+if [[ $TRACE_TOOLS == "true" ]]; then
+echo "Trace tools detected, installing sample.sh..."
+echo 0 > /proc/sys/kernel/kptr_restrict
+echo "
+# This script will take one minute of stacktrace samples and plot it in a flamegraph
+LIGHTNING_PROCESSES=\$(pidof lightningd lightning_chann lightning_closi lightning_gossi lightning_hsmd lightning_oncha lightning_openi lightning_hsmd lightning_gossipd lightning_channeld  | sed -e 's/\s/,/g')
+perf record -F 99 -g -a --pid \$LIGHTNING_PROCESSES -o \"$TRACE_LOCATION/perf.data\" -- sleep 60
+perf script -i \"$TRACE_LOCATION/perf.data\" > \"$TRACE_LOCATION/output.trace\"
+cd /FlameGraph
+./stackcollapse-perf.pl \"$TRACE_LOCATION/output.trace\" > \"$TRACE_LOCATION/output.trace.folded\"
+svg=\"$TRACE_LOCATION/\$((\$SECONDS / 60))min.svg\"
+./flamegraph.pl \"$TRACE_LOCATION/output.trace.folded\" > \"\$svg\"
+rm \"$TRACE_LOCATION/perf.data\"
+rm \"$TRACE_LOCATION/output.trace\"
+rm \"$TRACE_LOCATION/output.trace.folded\"
+echo \"flamegraph taken: \$svg\"
+" > /usr/bin/sample.sh
+chmod +x /usr/bin/sample.sh
+
+echo "
+# This script will run sample.sh after 2 min then every 10 minutes
+sleep 120
+sample.sh
+while true; do
+    sleep 300
+    . sample.sh
+done
+" > /usr/bin/sample-loop.sh
+chmod +x /usr/bin/sample-loop.sh
+fi
+
+if [[ "${LIGHTNINGD_ANNOUNCEADDR}" ]]; then
+    echo "announce-addr=$LIGHTNINGD_ANNOUNCEADDR:${LIGHTNINGD_PORT}" >> "$LIGHTNINGD_DATA/config"
+fi
+
+if [[ "${LIGHTNINGD_ALIAS}" ]]; then
+    # This allow to strip this parameter if LND_ALIGHTNINGD_ALIASLIAS is empty or null, and truncate it
+    LIGHTNINGD_ALIAS="$(echo "$LIGHTNINGD_ALIAS" | cut -c -32)"
+    echo "alias=$LIGHTNINGD_ALIAS" >> "$LIGHTNINGD_DATA/config"
+    echo "alias=$LIGHTNINGD_ALIAS added to $LIGHTNINGD_DATA/config"
+fi
+
+if [[ "${LIGHTNINGD_READY_FILE}" ]]; then
+    echo "Waiting $LIGHTNINGD_READY_FILE to be created..."
+    while [ ! -f "$LIGHTNINGD_READY_FILE" ]; do sleep 1; done
+    echo "The chain is fully synched"
+fi
+
+if [[ "${LIGHTNINGD_HIDDENSERVICE_HOSTNAME_FILE}" ]]; then
+    echo "Waiting $LIGHTNINGD_HIDDENSERVICE_HOSTNAME_FILE to be created by tor..."
+    while [ ! -f "$LIGHTNINGD_HIDDENSERVICE_HOSTNAME_FILE" ]; do sleep 1; done
+    HIDDENSERVICE_ONION="$(head -n 1 "$LIGHTNINGD_HIDDENSERVICE_HOSTNAME_FILE"):${LIGHTNINGD_PORT}"
+    echo "announce-addr=$HIDDENSERVICE_ONION" >> "$LIGHTNINGD_DATA/config"
+    echo "announce-addr=$HIDDENSERVICE_ONION added to $LIGHTNINGD_DATA/config"
+fi
+
+if ! grep -q "^rpc-file=" "$LIGHTNINGD_DATA/config"; then
+    echo "rpc-file=$LIGHTNINGD_DATA/lightning-rpc" >> "$LIGHTNINGD_DATA/config"
+    echo "rpc-file=$LIGHTNINGD_DATA/lightning-rpc added to $LIGHTNINGD_DATA/config"
+fi
+
+echo "Installing bundled plugins"
+mkdir -p "$LIGHTNINGD_DATA/plugins"
+cp -u /etc/bundledplugins/* $LIGHTNINGD_DATA/plugins/
 
 set -m
-lightningd --network="${LIGHTNINGD_NETWORK}" "$@" &
+lightningd "$@" &
 
 echo "Core-Lightning starting"
 while read -r i; do if [ "$i" = "lightning-rpc" ]; then break; fi; done \
-    < <(inotifywait -e create,open --format '%f' --quiet "${networkdatadir}" --monitor)
+    < <(inotifywait -e create,open --format '%f' --quiet "$LIGHTNINGD_DATA" --monitor)
 
 if [ "$EXPOSE_TCP" == "true" ]; then
     echo "Core-Lightning started, RPC available on port $LIGHTNINGD_RPC_PORT"
 
-    socat "TCP4-listen:$LIGHTNINGD_RPC_PORT,fork,reuseaddr" "UNIX-CONNECT:${networkdatadir}/lightning-rpc" &
+    socat "TCP4-listen:$LIGHTNINGD_RPC_PORT,fork,reuseaddr" "UNIX-CONNECT:$LIGHTNINGD_DATA/lightning-rpc" &
 fi
 
 # Now run any scripts which exist in the lightning-poststart.d directory