build-test.yml

name: build - check - upload

on:
  push:
    branches:
    - main
  release:
    types: [published]
  pull_request:
    branches:
    - '**'

concurrency:
  # SHA is added to the end if on `main` to let all main workflows run
  group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}-${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/') || startsWith(github.ref, 'refs/heads/long_lived/')) && github.sha || '' }}
  cancel-in-progress: true

permissions:
  id-token: write
  contents: read

jobs:
  build-wheels:
    name: Wheel - ${{ matrix.os.name }} ${{ matrix.python.major-dot-minor }} ${{ matrix.arch.name }}
    runs-on: ${{ matrix.os.runs-on[matrix.arch.matrix] }}
    strategy:
      fail-fast: false
      matrix:
        os:
          - name: macOS
            matrix: macos
            runs-on:
              arm: [macOS, ARM64]
              intel: [macos-latest]
          - name: Ubuntu
            matrix: ubuntu
            runs-on:
              arm: [Linux, ARM64]
              intel: [ubuntu-latest]
          - name: Windows
            matrix: windows
            runs-on:
              intel: [windows-latest]
        python:
          - major-dot-minor: '3.8'
            cibw-build: 'cp38-*'
            by-arch:
              arm:
                manylinux-version: 2014
                docker-url: ghcr.io/chia-network/build-images/centos-pypa-rust-aarch64
                rustup-target: aarch64-unknown-linux-musl
              intel:
                manylinux-version: 2014
                docker-url: ghcr.io/chia-network/build-images/centos-pypa-rust-x86_64
                rustup-target: x86_64-unknown-linux-musl
            matrix: '3.8'
          - major-dot-minor: '3.9'
            cibw-build: 'cp39-*'
            by-arch:
              arm:
                manylinux-version: 2014
                docker-url: ghcr.io/chia-network/build-images/centos-pypa-rust-aarch64
                rustup-target: aarch64-unknown-linux-musl
              intel:
                manylinux-version: 2014
                docker-url: ghcr.io/chia-network/build-images/centos-pypa-rust-x86_64
                rustup-target: x86_64-unknown-linux-musl
            matrix: '3.9'
          - major-dot-minor: '3.10'
            cibw-build: 'cp310-*'
            by-arch:
              arm:
                manylinux-version: 2014
                docker-url: ghcr.io/chia-network/build-images/centos-pypa-rust-aarch64
                rustup-target: aarch64-unknown-linux-musl
              intel:
                manylinux-version: 2014
                docker-url: ghcr.io/chia-network/build-images/centos-pypa-rust-x86_64
                rustup-target: x86_64-unknown-linux-musl
            matrix: '3.10'
          - major-dot-minor: '3.11'
            cibw-build: 'cp311-*'
            by-arch:
              arm:
                manylinux-version: 2014
                docker-url: ghcr.io/chia-network/build-images/centos-pypa-rust-aarch64
                rustup-target: aarch64-unknown-linux-musl
              intel:
                manylinux-version: 2014
                docker-url: ghcr.io/chia-network/build-images/centos-pypa-rust-x86_64
                rustup-target: x86_64-unknown-linux-musl
            matrix: '3.11'
        arch:
          - name: ARM
            matrix: arm
          - name: Intel
            matrix: intel
        exclude:
          # Only partial entries are required here by GitHub Actions so generally I
          # only specify the `matrix:` entry.  The super linter complains so for now
          # all entries are included to avoid that.  Reported at
          # https://github.com/github/super-linter/issues/3016
          - os:
              name: Windows
              matrix: windows
              runs-on:
                intel: [windows-latest]
            arch:
              name: ARM
              matrix: arm
          - os:
              name: macOS
              matrix: macos
              runs-on:
                arm: [macOS, ARM64]
                intel: [macos-latest]
            python:
              major-dot-minor: '3.7'
              cibw-build: 'cp37-*'
              matrix: '3.7'
            arch:
              name: ARM
              matrix: arm

    steps:
    - name: Clean workspace
      uses: Chia-Network/actions/clean-workspace@main

    - name: Checkout code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - uses: Chia-Network/actions/setup-python@main
      with:
        python-version: ${{ matrix.python.major-dot-minor }}

    - name: Update pip
      run: |
          python -m pip install --upgrade pip

    - name: Set up rust
      uses: actions-rs/toolchain@v1
      with:
        toolchain: stable

    - name: Install dependencies
      run: |
          python -m pip install maturin

    - name: Build MacOs with maturin on Python ${{ matrix.python }}
      if: matrix.os.matrix == 'macos'
      env:
        MACOSX_DEPLOYMENT_TARGET: '11.0'
      run: |
        python${{ matrix.python.major-dot-minor }} -m venv venv
        . venv/bin/activate
        maturin build -i python --release -m wheel/Cargo.toml

    - name: Build Linux with maturin on Python ${{ matrix.python }}
      if: matrix.os.matrix == 'ubuntu'
      run: |
        docker run --rm \
          -v ${{ github.workspace }}:/ws --workdir=/ws \
          ${{ matrix.python.by-arch[matrix.arch.matrix].docker-url }} \
          bash -exc '\
            yum -y install openssl-devel && \
            source $HOME/.cargo/env && \
            rustup target add ${{ matrix.python.by-arch[matrix.arch.matrix].rustup-target }} && \
            python${{ matrix.python.major-dot-minor }} -m venv /venv && \
            . /venv/bin/activate && \
            pip install --upgrade pip && \
            pip install maturin && \
            CC=gcc maturin build --release --manylinux ${{ matrix.python.by-arch[matrix.arch.matrix].manylinux-version }} -m wheel/Cargo.toml \
          '

    - name: Build Windows with maturin on Python ${{ matrix.python }}
      if: matrix.os.matrix == 'windows'
      run: |
        py -${{ matrix.python.major-dot-minor }} -m venv venv
        . .\venv\Scripts\Activate.ps1
        maturin build -i python --release -m wheel/Cargo.toml

    - uses: Chia-Network/actions/create-venv@main
      id: create-venv

    - uses: Chia-Network/actions/activate-venv@main
      with:
        directories: ${{ steps.create-venv.outputs.activate-venv-directories }}

    - name: Install chia_rs wheel
      run: |
        pip install --no-index --find-links wheel/target/wheels/ chia_rs

    - name: Upload artifacts
      uses: actions/upload-artifact@v3
      with:
        name: packages
        path: ./wheel/target/wheels/

  check-typestubs:
    name: Check chia_rs.pyi
    runs-on: ubuntu-latest

    steps:
    - name: Checkout code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - uses: Chia-Network/actions/setup-python@main

    - name: check generated chia_rs.pyi
      run: |
        python wheel/generate_type_stubs.py
        git diff --exit-code

  build-tools:
    name: build chia-tools
    runs-on: ubuntu-latest
    steps:
    - name: Clean workspace
      uses: Chia-Network/actions/clean-workspace@main

    - name: Checkout code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - name: cargo build
      run: |
        cd chia-tools
        cargo build

  build-sdist:
    name: sdist - ${{ matrix.os.name }} ${{ matrix.python.major-dot-minor }} ${{ matrix.arch.name }}
    runs-on: ${{ matrix.os.runs-on[matrix.arch.matrix] }}
    strategy:
      fail-fast: false
      matrix:
        os:
          - name: Ubuntu
            matrix: ubuntu
            runs-on:
              arm: [Linux, ARM64]
              intel: [ubuntu-latest]
        python:
          - major-dot-minor: '3.8'
            matrix: '3.8'
        arch:
          - name: Intel
            matrix: intel

    steps:
    - name: Clean workspace
      uses: Chia-Network/actions/clean-workspace@main

    - name: Checkout code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - uses: Chia-Network/actions/setup-python@main
      with:
        python-version: ${{ matrix.python.major-dot-minor }}

    - name: Build source distribution
      run: |
        pip install maturin
        maturin sdist -m wheel/Cargo.toml

    - name: Upload artifacts
      uses: actions/upload-artifact@v3
      with:
        name: packages
        path: ./wheel/target/wheels/

  fmt:
    runs-on: ubuntu-latest
    name: cargo fmt
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 1
      - name: Install rust
        uses: actions-rs/toolchain@v1
        with:
            toolchain: nightly
            components: rustfmt, clippy
            override: true
      - name: fmt
        run: |
            cargo fmt --all -- --files-with-diff --check
            cd wheel
            cargo fmt -- --files-with-diff --check

  clippy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly
          components: clippy
          override: true
      - name: workspace
        run: |
            cargo clippy --workspace --all-features -- -Dwarnings
      - name: wheel
        run: |
            cd wheel
            cargo clippy --all-features -- -Dwarnings

  fuzz_targets:
    runs-on: ubuntu-latest
    env:
        CARGO_PROFILE_RELEASE_LTO: false
    steps:
      - uses: actions/checkout@v3
      - uses: actions-rs/toolchain@v1
        with:
            toolchain: nightly
      - name: cargo-fuzz
        run: cargo +nightly install cargo-fuzz
      - name: cargo fuzz (chia_rs)
        run: cargo fuzz list | xargs -I "%" sh -c "cargo +nightly fuzz run % -- -max_total_time=20 || exit 255"
      - name: cargo fuzz (chia-bls)
        env:
          # we disable leak reports here because blspy appears to be allocating
          # memory that's not freed. It might be a false positive since python is
          # not unloaded before exiting
          LSAN_OPTIONS: detect_leaks=0
        run: |
          cd chia-bls
          python -m pip install blspy
          cargo fuzz list | xargs -I "%" sh -c "cargo +nightly fuzz run % -- -max_total_time=10 || exit 255"
      - name: cargo fuzz (clvm-utils)
        run: |
          cd clvm-utils
          cargo fuzz list | xargs -I "%" sh -c "cargo +nightly fuzz run % -- -max_total_time=20 || exit 255"
      - name: cargo fuzz (chia-protocol)
        run: |
          cd chia-protocol
          cargo +nightly fuzz build
          cargo fuzz list | xargs -I "%" sh -c "cargo +nightly fuzz run % -- -max_total_time=20 || exit 255"
      - name: cargo fuzz (chia-wallet)
        run: |
          cd chia-wallet
          cargo +nightly fuzz build
          cargo fuzz list | xargs -I "%" sh -c "cargo +nightly fuzz run % -- -max_total_time=20 || exit 255"

  unit_tests:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [macos-latest, ubuntu-latest, windows-latest]
    name: Unit tests
    steps:
      # the test files are read verbatim, making it problematic if git is
      # allowed to insert \r when checking out files
      - name: disable git autocrlf
        run: |
          git config --global core.autocrlf false

      - uses: actions/checkout@v3
        with:
          fetch-depth: 1
      - uses: actions-rs/toolchain@v1
        with:
            toolchain: stable
      - name: Prepare for coverage
        if: matrix.os == 'ubuntu-latest'
        run: |
          cargo install grcov
          echo "RUSTFLAGS=-Cinstrument-coverage" >> "$GITHUB_ENV"
          echo "LLVM_PROFILE_FILE=$(pwd)/target/chia_rs-%p-%m.profraw" >> "$GITHUB_ENV"
          echo "CARGO_TARGET_DIR=$(pwd)/target" >> "$GITHUB_ENV"
      - name: cargo test
        run: cargo test --workspace --all-features
      - name: cargo test (release)
        run: cargo test --workspace --all-features --release
      - name: Continue with coverage
        if: matrix.os == 'ubuntu-latest'
        run: |
          sudo apt-get update
          sudo apt-get install lcov -y
          rustup component add llvm-tools-preview
          python -m venv venv
          source venv/bin/activate
          git clone https://github.com/Chia-Network/clvm_tools.git --branch=main --single-branch
          pip install ./clvm_tools
          pip install colorama maturin pytest
          maturin develop --release -m wheel/Cargo.toml
          pytest tests
          grcov . --binary-path target -s . --branch --ignore-not-existing --ignore='*/.cargo/*' --ignore='tests/*' --ignore='venv/*' -o rust_cov.info
          python -c 'with open("rust_cov.info") as f: lines = [l for l in f if not (l.startswith("DA:") and int(l.split(",")[1].strip()) >= 2**63)]; open("lcov.info", "w").writelines(lines)'
      - name: Upload to Coveralls
        uses: coverallsapp/github-action@v2
        if: matrix.os == 'ubuntu-latest'
        env:
          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
        with:
          path-to-lcov: './lcov.info'

  upload:
    name: Upload to PyPI - ${{ matrix.os.name }} ${{ matrix.python.major-dot-minor }} ${{ matrix.arch.name }}
    runs-on: ${{ matrix.os.runs-on[matrix.arch.matrix] }}
    needs:
      - build-wheels
      - build-sdist
      - fmt
      - clippy
      - unit_tests
    strategy:
      fail-fast: false
      matrix:
        os:
          - name: Ubuntu
            matrix: ubuntu
            runs-on:
              arm: [Linux, ARM64]
              intel: [ubuntu-latest]
        python:
          - major-dot-minor: '3.9'
            matrix: '3.9'
        arch:
          - name: Intel
            matrix: intel

    steps:
    - name: Clean workspace
      uses: Chia-Network/actions/clean-workspace@main

    - name: Checkout code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - uses: Chia-Network/actions/setup-python@main
      with:
        python-version: ${{ matrix.python.major-dot-minor }}

    - name: Download artifacts
      uses: actions/download-artifact@v3
      with:
        name: packages
        path: ./dist

    - name: Set Env
      uses: Chia-Network/actions/setjobenv@main
      env:
        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

    - name: publish (PyPi)
      if: env.RELEASE == 'true'
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        packages-dir: dist/
        skip-existing: true

    - name: publish (Test PyPi)
      if: env.PRE_RELEASE == 'true'
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        repository-url: https://test.pypi.org/legacy/
        packages-dir: dist/
        skip-existing: true