From a638489db5f5f66614b36b475fb1e10c085f086d Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Wed, 7 Mar 2018 13:50:44 -0500 Subject: [PATCH] Adapt Dockerfile for BTCPay deployment --- .github/workflows/docker.yml | 32 +++++++++++ Dockerfile | 62 +++++++++++++++++++--- plugins/bcli.c | 22 ++++---- tools/docker-entrypoint.sh | 100 +++++++++++++++++++++++++++++++++-- 4 files changed, 196 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 000000000000..41f1af990dec --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,32 @@ +name: Docker packaging +on: + push: + tags: + - 'basedon-*' + +jobs: + main: + runs-on: ubuntu-latest + steps: + - + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - + name: Create images + env: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_PASS: ${{ secrets.DOCKERHUB_PASS }} + DOCKERHUB_REPO: ${{ env.DOCKERHUB_REPO }} + shell: bash + run: | + LATEST_TAG=${GITHUB_REF#refs/tags/} + echo "First: $LATEST_TAG" + LATEST_TAG=${LATEST_TAG:8} #trim "basedon-" from tag + echo "Second: $LATEST_TAG" + echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USER" --password-stdin + docker buildx create --use + DOCKER_BUILDX_OPTS="--platform linux/amd64,linux/arm64,linux/arm/v7 --push" + docker buildx build $DOCKER_BUILDX_OPTS -t $DOCKERHUB_REPO:$LATEST_TAG . \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index a5ef573f0630..10b4c1ba1a3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,30 @@ # * final: Copy the binaries required at runtime # The resulting image uploaded to dockerhub will only contain what is needed for runtime. # From the root of the repository, run "docker build -t yourimage:yourtag ." -FROM debian:bullseye-slim as downloader + +FROM debian:bullseye-slim as base-downloader + +FROM --platform=$BUILDPLATFORM debian:bullseye-slim as base-downloader +RUN set -ex \ + && apt-get update \ + && apt-get install -qq --no-install-recommends ca-certificates dirmngr wget + +FROM --platform=$BUILDPLATFORM base-downloader as base-downloader-linux-amd64 +ENV TARBALL_ARCH_FINAL=x86_64-linux-gnu +ENV DESCHASHPLUGIN_ARCH=linux-amd64 +ENV DESCHASHPLUGIN_HASH=deadc00c68fac80b2718d92f69bf06acd8fff646228d497bbb76a4f0a12ca217 + +FROM --platform=$BUILDPLATFORM base-downloader as base-downloader-linux-arm64 +ENV TARBALL_ARCH_FINAL=aarch64-linux-gnu +ENV DESCHASHPLUGIN_ARCH=linux-arm64 +ENV DESCHASHPLUGIN_HASH=d48c3e5aede77bd9cb72d78689ce12c0327f624435cb0496b3eacb92df416363 + +FROM --platform=$BUILDPLATFORM base-downloader as base-downloader-linux-arm +ENV TARBALL_ARCH_FINAL=arm-linux-gnueabihf +ENV DESCHASHPLUGIN_ARCH=linux-arm +ENV DESCHASHPLUGIN_HASH=f7df336c72dd1674bd18ff23862a410b6a9691a3e13752264dcffa0950e21c74 + +FROM base-downloader-${TARGETOS}-${TARGETARCH} as downloader RUN set -ex \ && apt-get update \ @@ -14,9 +37,7 @@ RUN set -ex \ WORKDIR /opt -ARG BITCOIN_VERSION=22.0 -ARG TARBALL_ARCH=x86_64-linux-gnu -ENV TARBALL_ARCH_FINAL=$TARBALL_ARCH +ENV BITCOIN_VERSION=22.0 ENV BITCOIN_TARBALL bitcoin-${BITCOIN_VERSION}-${TARBALL_ARCH_FINAL}.tar.gz ENV BITCOIN_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/$BITCOIN_TARBALL ENV BITCOIN_ASC_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/SHA256SUMS @@ -39,6 +60,14 @@ RUN mkdir /opt/litecoin && cd /opt/litecoin \ && tar -xzvf litecoin.tar.gz litecoin-$LITECOIN_VERSION/bin/litecoin-cli --strip-components=1 --exclude=*-qt \ && rm litecoin.tar.gz +ENV DESCHASHPLUGIN_URL https://github.com/nbd-wtf/invoicewithdescriptionhash/releases/download/v1.4/invoicewithdescriptionhash-v1.4-${DESCHASHPLUGIN_ARCH}.tar.gz +ENV DESCHASHPLUGIN_SHA256 ${DESCHASHPLUGIN_HASH} +RUN mkdir /opt/deschashplugin && cd /opt/deschashplugin \ + && wget -qO invoicewithdescriptionhash.tar.gz "$DESCHASHPLUGIN_URL" \ + && echo "$DESCHASHPLUGIN_SHA256 invoicewithdescriptionhash.tar.gz" | sha256sum -c - \ + && tar -xzvf invoicewithdescriptionhash.tar.gz && rm invoicewithdescriptionhash.tar.gz \ + && chmod a+x invoicewithdescriptionhash + FROM debian:bullseye-slim as builder ENV LIGHTNINGD_VERSION=master @@ -99,6 +128,7 @@ COPY . /tmp/lightning RUN git clone --recursive /tmp/lightning . && \ git checkout $(git --work-tree=/tmp/lightning --git-dir=/tmp/lightning/.git rev-parse HEAD) +ARG DEVELOPER=0 ENV PYTHON_VERSION=3 RUN curl -sSL https://install.python-poetry.org | python3 - @@ -117,6 +147,11 @@ RUN ./configure --prefix=/tmp/lightning_install --enable-static && \ FROM debian:bullseye-slim as final +ARG TRACE_TOOLS=false +ENV TRACE_TOOLS=$TRACE_TOOLS +ENV TRACE_LOCATION=/opt/traces +VOLUME /opt/traces + RUN apt-get update && \ apt-get install -y --no-install-recommends \ tini \ @@ -125,9 +160,18 @@ RUN apt-get update && \ python3.9 \ python3-pip \ qemu-user-static \ - libpq5 && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* + libpq5 \ + && \ + ( ! $TRACE_TOOLS || \ + ( \ + apt-get install -y --no-install-recommends perl linux-base curl ca-certificates && \ + mkdir FlameGraph && cd FlameGraph && \ + curl -Lo FlameGraph.tar.gz "https://github.com/brendangregg/FlameGraph/archive/v1.0.tar.gz" && \ + tar -zxvf FlameGraph.tar.gz --strip-components=1 && rm FlameGraph.tar.gz && cd .. \ + ) \ + ) \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* ENV LIGHTNINGD_DATA=/root/.lightning ENV LIGHTNINGD_RPC_PORT=9835 @@ -135,6 +179,8 @@ ENV LIGHTNINGD_PORT=9735 ENV LIGHTNINGD_NETWORK=bitcoin RUN mkdir $LIGHTNINGD_DATA && \ + mkdir /etc/bundledplugins && \ + mkdir $LIGHTNINGD_DATA/plugins && \ touch $LIGHTNINGD_DATA/config VOLUME [ "/root/.lightning" ] @@ -142,6 +188,8 @@ COPY --from=builder /tmp/lightning_install/ /usr/local/ COPY --from=builder /usr/local/lib/python3.9/dist-packages/ /usr/local/lib/python3.9/dist-packages/ COPY --from=downloader /opt/bitcoin/bin /usr/bin COPY --from=downloader /opt/litecoin/bin /usr/bin +COPY --from=downloader /opt/deschashplugin $LIGHTNINGD_DATA/plugins +COPY --from=downloader /opt/deschashplugin /etc/bundledplugins COPY tools/docker-entrypoint.sh entrypoint.sh EXPOSE 9735 9835 diff --git a/plugins/bcli.c b/plugins/bcli.c index e04159b44377..fee4aed57617 100644 --- a/plugins/bcli.c +++ b/plugins/bcli.c @@ -969,6 +969,7 @@ static void wait_and_check_bitcoind(struct plugin *p) pid_t child; const char **cmd = gather_args(bitcoind, "getnetworkinfo", NULL); bool printed = false; + bool isWarmup = false; char *output = NULL; for (;;) { @@ -1004,17 +1005,20 @@ static void wait_and_check_bitcoind(struct plugin *p) /* bitcoin/src/rpc/protocol.h: * RPC_IN_WARMUP = -28, //!< Client still warming up */ - if (WEXITSTATUS(status) != 28) { - if (WEXITSTATUS(status) == 1) - bitcoind_failure(p, "Could not connect to bitcoind using" - " bitcoin-cli. Is bitcoind running?"); - bitcoind_failure(p, tal_fmt(bitcoind, "%s exited with code %i: %s", - cmd[0], WEXITSTATUS(status), output)); - } + isWarmup = WEXITSTATUS(status) == 28; if (!printed) { - plugin_log(p, LOG_UNUSUAL, - "Waiting for bitcoind to warm up..."); + if (isWarmup) + { + plugin_log(p, LOG_UNUSUAL, + "Waiting for bitcoind to warm up..."); + } + else + { + plugin_log(p, LOG_UNUSUAL, + tal_fmt(bitcoind, "%s exited with code %i: %s... retrying", + cmd[0], WEXITSTATUS(status), output)); + } printed = true; } sleep(1); diff --git a/tools/docker-entrypoint.sh b/tools/docker-entrypoint.sh index 8d7bbfd2d920..2c9346efb118 100755 --- a/tools/docker-entrypoint.sh +++ b/tools/docker-entrypoint.sh @@ -2,19 +2,111 @@ : "${EXPOSE_TCP:=false}" -networkdatadir="${LIGHTNINGD_DATA}/${LIGHTNINGD_NETWORK}" +cat <<-EOF > "$LIGHTNINGD_DATA/config" +${LIGHTNINGD_OPT} +bind-addr=0.0.0.0:${LIGHTNINGD_PORT} +EOF + +LIGHTNINGD_NETWORK_NAME="" + +if [ "$LIGHTNINGD_CHAIN" == "btc" ] && [ "$LIGHTNINGD_NETWORK" == "mainnet" ]; then + LIGHTNINGD_NETWORK_NAME="bitcoin" +elif [ "$LIGHTNINGD_CHAIN" == "btc" ] && [ "$LIGHTNINGD_NETWORK" == "testnet" ]; then + LIGHTNINGD_NETWORK_NAME="testnet" +elif [ "$LIGHTNINGD_CHAIN" == "btc" ] && [ "$LIGHTNINGD_NETWORK" == "regtest" ]; then + LIGHTNINGD_NETWORK_NAME="regtest" +elif [ "$LIGHTNINGD_CHAIN" == "ltc" ] && [ "$LIGHTNINGD_NETWORK" == "mainnet" ]; then + LIGHTNINGD_NETWORK_NAME="litecoin" +elif [ "$LIGHTNINGD_CHAIN" == "ltc" ] && [ "$LIGHTNINGD_NETWORK" == "testnet" ]; then + LIGHTNINGD_NETWORK_NAME="litecoin-testnet" +else + echo "Invalid combinaion of LIGHTNINGD_NETWORK and LIGHTNINGD_CHAIN. LIGHTNINGD_CHAIN should be btc or ltc. LIGHTNINGD_NETWORK should be mainnet, testnet or regtest." + echo "ltc regtest is not supported" + exit +fi + +echo "network=$LIGHTNINGD_NETWORK_NAME" >> "$LIGHTNINGD_DATA/config" +echo "network=$LIGHTNINGD_NETWORK_NAME added in $LIGHTNINGD_DATA/config" + +echo "disable-plugin=clnrest.py" >> "$LIGHTNINGD_DATA/config" +echo "disable-plugin=clnrest.py added in $LIGHTNINGD_DATA/config" + +if [[ $TRACE_TOOLS == "true" ]]; then +echo "Trace tools detected, installing sample.sh..." +echo 0 > /proc/sys/kernel/kptr_restrict +echo " +# This script will take one minute of stacktrace samples and plot it in a flamegraph +LIGHTNING_PROCESSES=\$(pidof lightningd lightning_chann lightning_closi lightning_gossi lightning_hsmd lightning_oncha lightning_openi lightning_hsmd lightning_gossipd lightning_channeld | sed -e 's/\s/,/g') +perf record -F 99 -g -a --pid \$LIGHTNING_PROCESSES -o \"$TRACE_LOCATION/perf.data\" -- sleep 60 +perf script -i \"$TRACE_LOCATION/perf.data\" > \"$TRACE_LOCATION/output.trace\" +cd /FlameGraph +./stackcollapse-perf.pl \"$TRACE_LOCATION/output.trace\" > \"$TRACE_LOCATION/output.trace.folded\" +svg=\"$TRACE_LOCATION/\$((\$SECONDS / 60))min.svg\" +./flamegraph.pl \"$TRACE_LOCATION/output.trace.folded\" > \"\$svg\" +rm \"$TRACE_LOCATION/perf.data\" +rm \"$TRACE_LOCATION/output.trace\" +rm \"$TRACE_LOCATION/output.trace.folded\" +echo \"flamegraph taken: \$svg\" +" > /usr/bin/sample.sh +chmod +x /usr/bin/sample.sh + +echo " +# This script will run sample.sh after 2 min then every 10 minutes +sleep 120 +sample.sh +while true; do + sleep 300 + . sample.sh +done +" > /usr/bin/sample-loop.sh +chmod +x /usr/bin/sample-loop.sh +fi + +if [[ "${LIGHTNINGD_ANNOUNCEADDR}" ]]; then + echo "announce-addr=$LIGHTNINGD_ANNOUNCEADDR:${LIGHTNINGD_PORT}" >> "$LIGHTNINGD_DATA/config" +fi + +if [[ "${LIGHTNINGD_ALIAS}" ]]; then + # This allow to strip this parameter if LND_ALIGHTNINGD_ALIASLIAS is empty or null, and truncate it + LIGHTNINGD_ALIAS="$(echo "$LIGHTNINGD_ALIAS" | cut -c -32)" + echo "alias=$LIGHTNINGD_ALIAS" >> "$LIGHTNINGD_DATA/config" + echo "alias=$LIGHTNINGD_ALIAS added to $LIGHTNINGD_DATA/config" +fi + +if [[ "${LIGHTNINGD_READY_FILE}" ]]; then + echo "Waiting $LIGHTNINGD_READY_FILE to be created..." + while [ ! -f "$LIGHTNINGD_READY_FILE" ]; do sleep 1; done + echo "The chain is fully synched" +fi + +if [[ "${LIGHTNINGD_HIDDENSERVICE_HOSTNAME_FILE}" ]]; then + echo "Waiting $LIGHTNINGD_HIDDENSERVICE_HOSTNAME_FILE to be created by tor..." + while [ ! -f "$LIGHTNINGD_HIDDENSERVICE_HOSTNAME_FILE" ]; do sleep 1; done + HIDDENSERVICE_ONION="$(head -n 1 "$LIGHTNINGD_HIDDENSERVICE_HOSTNAME_FILE"):${LIGHTNINGD_PORT}" + echo "announce-addr=$HIDDENSERVICE_ONION" >> "$LIGHTNINGD_DATA/config" + echo "announce-addr=$HIDDENSERVICE_ONION added to $LIGHTNINGD_DATA/config" +fi + +if ! grep -q "^rpc-file=" "$LIGHTNINGD_DATA/config"; then + echo "rpc-file=$LIGHTNINGD_DATA/lightning-rpc" >> "$LIGHTNINGD_DATA/config" + echo "rpc-file=$LIGHTNINGD_DATA/lightning-rpc added to $LIGHTNINGD_DATA/config" +fi + +echo "Installing bundled plugins" +mkdir -p "$LIGHTNINGD_DATA/plugins" +cp -u /etc/bundledplugins/* $LIGHTNINGD_DATA/plugins/ set -m -lightningd --network="${LIGHTNINGD_NETWORK}" "$@" & +lightningd "$@" & echo "Core-Lightning starting" while read -r i; do if [ "$i" = "lightning-rpc" ]; then break; fi; done \ - < <(inotifywait -e create,open --format '%f' --quiet "${networkdatadir}" --monitor) + < <(inotifywait -e create,open --format '%f' --quiet "$LIGHTNINGD_DATA" --monitor) if [ "$EXPOSE_TCP" == "true" ]; then echo "Core-Lightning started, RPC available on port $LIGHTNINGD_RPC_PORT" - socat "TCP4-listen:$LIGHTNINGD_RPC_PORT,fork,reuseaddr" "UNIX-CONNECT:${networkdatadir}/lightning-rpc" & + socat "TCP4-listen:$LIGHTNINGD_RPC_PORT,fork,reuseaddr" "UNIX-CONNECT:$LIGHTNINGD_DATA/lightning-rpc" & fi # Now run any scripts which exist in the lightning-poststart.d directory