ci.yaml

---
name: Continuous Integration
on:
  push:
    branches:
      - "master"
  pull_request:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  # Makes the upload-artifact work more reliably at the cost
  # of a bit of compile time.
  RUST_PROFILE: release
  SLOW_MACHINE: 1
  CI_SERVER_URL: "http://35.239.136.52:3170"

jobs:
  prebuild:
    name: Pre-build checks
    runs-on: ubuntu-20.04
    timeout-minutes: 30
    env:
      BOLTDIR: bolts
    strategy:
      fail-fast: true
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0

      - name: Rebase
        # We can't rebase if we're on master already.
        if: github.ref != 'refs/heads/master'
        run: |
          git config user.name github-actions
          git config user.email github-actions@github.com
          git fetch origin ${{ github.base_ref }}
          git rebase origin/${{ github.base_ref }}

      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: 3.8

      - name: Install dependencies
        run: |
          bash -x .github/scripts/setup.sh
          pip install -U pip wheel poetry
          # Export and then use pip to install into the current env
          poetry export -o /tmp/requirements.txt --without-hashes --with dev
          pip install -r /tmp/requirements.txt
          # We're going to check BOLT quotes, so get the latest version
          git clone https://github.com/lightning/bolts.git ../${BOLTDIR}
      - name: Configure
        run: ./configure --enable-debugbuild --enable-rust
      - name: Check source
        run: make check-source BASE_REF="origin/${{ github.base_ref }}"
      - name: Check Generated Files have been updated
        run: make check-gen-updated
      - name: Check docs
        run: make check-doc

  compile:
    name: Compile CLN ${{ matrix.cfg }}
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    needs:
      - prebuild
    strategy:
      fail-fast: true
      matrix:
        include:
          - CFG: compile-gcc
            VALGRIND: 1
            COMPILER: gcc
          - CFG: compile-gcc-O3
            VALGRIND: 1
            COMPILER: gcc
            COPTFLAGS_VAR: COPTFLAGS="-O3 -Werror"
          # While we're at it let's try to compile with clang
          - CFG: compile-clang
            VALGRIND: 1
            COMPILER: clang
          - CFG: compile-clang-sanitizers
            COMPILER: clang
            ASAN: 1
            UBSAN: 1
            VALGRIND: 0
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: 3.8

      - name: Install dependencies
        run: |
          bash -x .github/scripts/setup.sh

      - name: Build
        env:
          COMPILER: ${{ matrix.COMPILER }}
          ASAN: ${{ matrix.ASAN }}
          UBSAN: ${{ matrix.UBSAN }}
          VALGRIND: ${{ matrix.VALGRIND }}
          COMPAT: 1
          CFG: ${{ matrix.CFG }}
        run: |
          set -e
          pip3 install --user pip wheel poetry
          poetry export -o requirements.txt --with dev --without-hashes
          python3 -m pip install -r requirements.txt
          ./configure --enable-debugbuild CC="$COMPILER" ${{ matrix.COPTFLAGS_VAR }}

          make -j $(nproc) testpack.tar.bz2

          # Rename now so we don't clash
          mv testpack.tar.bz2 cln-${CFG}.tar.bz2
      - name: Check rust packages
        run: cargo test --all
      - uses: actions/upload-artifact@v3
        with:
          name: cln-${{ matrix.CFG }}.tar.bz2
          path: cln-${{ matrix.CFG }}.tar.bz2

  check-units:
    # The unit test checks are not in the critical path (not dependent
    # on the integration tests), so run them with `valgrind`
    name: Run unit tests
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    env:
      BOLTDIR: bolts
    needs:
      - compile
    strategy:
      fail-fast: true
      matrix:
        include:
          - CFG: compile-gcc
            VALGRIND: 1
          - CFG: compile-clang-sanitizers
            VALGRIND: 0
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: 3.8

      - name: Install dependencies
        run: |
          bash -x .github/scripts/setup.sh
          sudo apt-get update -qq
          sudo apt-get install -y -qq lowdown
          pip install -U pip wheel poetry
          # Export and then use pip to install into the current env
          poetry export -o /tmp/requirements.txt --without-hashes --with dev
          pip install -r /tmp/requirements.txt
          # We're going to check BOLT quotes, so get the latest version
          git clone https://github.com/lightning/bolts.git ../${BOLTDIR}

      - name: Download build
        uses: actions/download-artifact@v3
        with:
          name: cln-${{ matrix.CFG }}.tar.bz2

      - name: Check
        run: |
          tar -xaf cln-${{ matrix.CFG }}.tar.bz2
          make -j $(nproc) check-units installcheck VALGRIND=${{ matrix.VALGRIND }}

  check-fuzz:
    name: Run fuzz regression tests
    runs-on: ubuntu-22.04
    needs:
      - prebuild
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: 3.8

      - name: Install dependencies
        run: |
          bash -x .github/scripts/setup.sh
          pip install -U pip wheel poetry
          # Export and then use pip to install into the current env
          poetry export -o /tmp/requirements.txt --without-hashes --with dev
          pip install -r /tmp/requirements.txt

      - name: Build
        run: |
          ./configure --enable-debugbuild --enable-fuzzing --enable-address-sanitizer --enable-ub-sanitizer --disable-valgrind CC=clang
          make -j $(nproc) check-fuzz

  integration:
    name: Test CLN ${{ matrix.name }}
    runs-on: ubuntu-22.04
    timeout-minutes: 120
    env:
      BITCOIN_VERSION: "26.1"
      ELEMENTS_VERSION: 22.0.2
      RUST_PROFILE: release  # Has to match the one in the compile step
      PYTEST_OPTS: --timeout=1200 --force-flaky
    needs:
      - compile
    strategy:
      fail-fast: true
      matrix:
        include:
          - NAME: gcc
            CFG: compile-gcc
            TEST_DB_PROVIDER: sqlite3
            COMPILER: gcc
            TEST_NETWORK: regtest
          # While we're at it let's try to compile with clang
          - NAME: clang
            CFG: compile-clang
            TEST_DB_PROVIDER: sqlite3
            COMPILER: clang
            TEST_NETWORK: regtest
          # And of course we want to test postgres too
          - NAME: postgres
            CFG: compile-gcc
            COMPILER: gcc
            TEST_DB_PROVIDER: postgres
            TEST_NETWORK: regtest
          # And don't forget about elements (like cdecker did when
          # reworking the CI...)
          - NAME: liquid
            CFG: compile-gcc
            COMPILER: gcc
            TEST_NETWORK: liquid-regtest
            TEST_DB_PROVIDER: sqlite3
          # And dual funding!
          - NAME: dual-fund
            CFG: compile-gcc
            TEST_DB_PROVIDER: sqlite3
            COMPILER: gcc
            TEST_NETWORK: regtest
            EXPERIMENTAL_DUAL_FUND: 1
          # And splicing!
          - NAME: splicing
            CFG: compile-gcc
            TEST_DB_PROVIDER: sqlite3
            COMPILER: gcc
            TEST_NETWORK: regtest
            EXPERIMENTAL_SPLICING: 1
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: 3.8

      - name: Install dependencies
        run: |
          pip3 install --user pip wheel poetry
          poetry install

      - name: Install bitcoind
        env:
          TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
        run: .github/scripts/install-bitcoind.sh

      - name: Download build
        uses: actions/download-artifact@v3
        with:
          name: cln-${{ matrix.CFG }}.tar.bz2

      - name: Unpack pre-built CLN
        env:
          CFG: ${{ matrix.CFG }}
        run: |
          tar -xaf cln-${CFG}.tar.bz2

      - name: Switch network
        if: ${{ matrix.TEST_NETWORK == 'liquid-regtest' }}
        run: |
          # Loading the network from config.vars rather than the envvar is a terrible idea...
          sed -i 's/TEST_NETWORK=regtest/TEST_NETWORK=liquid-regtest/g' config.vars
          cat config.vars

      - name: Test
        env:
          COMPILER: ${{ matrix.COMPILER }}
          EXPERIMENTAL_DUAL_FUND: ${{ matrix.EXPERIMENTAL_DUAL_FUND }}
          EXPERIMENTAL_SPLICING: ${{ matrix.EXPERIMENTAL_SPLICING }}
          COMPAT: 1
          CFG: ${{ matrix.CFG }}
          SLOW_MACHINE: 1
          PYTEST_PAR: 10
          TEST_DEBUG: 1
          TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }}
          TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
          LIGHTNINGD_POSTGRES_NO_VACUUM: 1
        run: |
          env
          cat config.vars
          VALGRIND=0 poetry run pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}

  integration-valgrind:
    name: Valgrind Test CLN ${{ matrix.name }}
    runs-on: ubuntu-22.04
    timeout-minutes: 120
    env:
      BITCOIN_VERSION: "26.1"
      ELEMENTS_VERSION: 22.0.2
      RUST_PROFILE: release  # Has to match the one in the compile step
      CFG: compile-gcc
      PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800 --force-flaky
    needs:
      - compile
    strategy:
      fail-fast: true
      matrix:
        include:
          - NAME: Valgrind (01/10)
            PYTEST_OPTS: --test-group=1  --test-group-count=10
          - NAME: Valgrind (02/10)
            PYTEST_OPTS: --test-group=2  --test-group-count=10
          - NAME: Valgrind (03/10)
            PYTEST_OPTS: --test-group=3  --test-group-count=10
          - NAME: Valgrind (04/10)
            PYTEST_OPTS: --test-group=4  --test-group-count=10
          - NAME: Valgrind (05/10)
            PYTEST_OPTS: --test-group=5  --test-group-count=10
          - NAME: Valgrind (06/10)
            PYTEST_OPTS: --test-group=6  --test-group-count=10
          - NAME: Valgrind (07/10)
            PYTEST_OPTS: --test-group=7  --test-group-count=10
          - NAME: Valgrind (08/10)
            PYTEST_OPTS: --test-group=8  --test-group-count=10
          - NAME: Valgrind (09/10)
            PYTEST_OPTS: --test-group=9  --test-group-count=10
          - NAME: Valgrind (10/10)
            PYTEST_OPTS: --test-group=10 --test-group-count=10
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: 3.8

      - name: Install dependencies
        run: |
          sudo apt-get update -qq
          sudo apt-get install -yyq valgrind
          pip3 install --user pip wheel poetry
          poetry install

      - name: Install bitcoind
        run: .github/scripts/install-bitcoind.sh

      - name: Download build
        uses: actions/download-artifact@v3
        with:
          name: cln-compile-gcc.tar.bz2

      - name: Unpack build
        run: tar -xvjf cln-compile-gcc.tar.bz2

      - name: Test
        env:
          SLOW_MACHINE: 1
          TEST_DEBUG: 1
        run: |
          VALGRIND=1 poetry run pytest tests/ -vvv -n 3 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}

  integration-sanitizers:
    name: Sanitizers Test CLN
    runs-on: ubuntu-22.04
    timeout-minutes: 120
    env:
      BITCOIN_VERSION: "26.1"
      ELEMENTS_VERSION: 22.0.2
      RUST_PROFILE: release
      SLOW_MACHINE: 1
      TEST_DEBUG: 1
      PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800 --force-flaky
    needs:
      - compile
    strategy:
      fail-fast: true
      matrix:
        include:
          - NAME: ASan/UBSan (01/10)
            PYTEST_OPTS: --test-group=1  --test-group-count=10
          - NAME: ASan/UBSan (02/10)
            PYTEST_OPTS: --test-group=2  --test-group-count=10 -n 1
          - NAME: ASan/UBSan (03/10)
            PYTEST_OPTS: --test-group=3  --test-group-count=10
          - NAME: ASan/UBSan (04/10)
            PYTEST_OPTS: --test-group=4  --test-group-count=10
          - NAME: ASan/UBSan (05/10)
            PYTEST_OPTS: --test-group=5  --test-group-count=10
          - NAME: ASan/UBSan (06/10)
            PYTEST_OPTS: --test-group=6  --test-group-count=10
          - NAME: ASan/UBSan (07/10)
            PYTEST_OPTS: --test-group=7  --test-group-count=10
          - NAME: ASan/UBSan (08/10)
            PYTEST_OPTS: --test-group=8  --test-group-count=10
          - NAME: ASan/UBSan (09/10)
            PYTEST_OPTS: --test-group=9  --test-group-count=10
          - NAME: ASan/UBSan (10/10)
            PYTEST_OPTS: --test-group=10 --test-group-count=10
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: 3.8

      - name: Install dependencies
        run: |
          bash -x .github/scripts/setup.sh
          set -e
          pip3 install --user pip wheel poetry
          poetry export -o requirements.txt --with dev --without-hashes
          python3 -m pip install -r requirements.txt
          poetry install

      - name: Install bitcoind
        run: .github/scripts/install-bitcoind.sh

      - name: Download build
        uses: actions/download-artifact@v3
        with:
          name: cln-compile-clang-sanitizers.tar.bz2

      - name: Unpack build
        run: tar -xvjf cln-compile-clang-sanitizers.tar.bz2

      - name: Test
        run: |
          poetry run pytest tests/ -vvv -n 2 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}

  gather:
    # A dummy task that depends on the full matrix of tests, and
    # signals successful completion. Used for the PR status to pass
    # before merging.
    name: CI completion
    runs-on: ubuntu-20.04
    needs:
      - integration
      - check-units
      - integration-valgrind
      - integration-sanitizers
    steps:
      - name: Complete
        run: |
          echo CI completed successfully