linux.yml

name: Nightly builds on linux
on:
  push:
    paths:
      - '.github/workflows/linux.yml'
      - 'scripts/**'
  pull_request:
    paths:
      - '.github/workflows/linux.yml'
      - 'scripts/**'
  schedule:
    - cron: "0 2 * * *" # Every night at 2 AM UTC (9 PM EST; 10 PM EDT)
  workflow_dispatch:
    inputs:
      LIBTILEDB_REF:
        description: >-
          The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB
        default: dev
      TILEDB_PY_REF:
        description: >-
          The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB-Py
        default: dev
      TILEDB_R_REF:
        description: >-
          The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB-R
        default: master
      LIBTILEDBVCF_REF:
        description: >-
          The Git reference (tag, branch, commit) to checkout
          TileDB-Inc/TileDB-VCF (for building libtiledbvcf)
        default: main
      TILEDBVCF_PY_REF:
        description: >-
          The Git reference (tag, branch, commit) to checkout
          TileDB-Inc/TileDB-VCF (for building tiledbvcf-py)
        default: main
      LIBTILEDBSOMA_REF:
        description: >-
          The Git reference (tag, branch, commit) to checkout
          single-cell-data/TileDB-SOMA (for building libtiledbsoma)
        default: main
      TILEDBSOMA_PY_REF:
        description: >-
          The Git reference (tag, branch, commit) to checkout
          single-cell-data/TileDB-SOMA (for builing tiledbsoma-py)
        default: main
      TILEDBSOMA_R_REF:
        description: >-
          The Git reference (tag, branch, commit) to checkout
          single-cell-data/TileDB-SOMA (for building tiledbsoma-r)
        default: main
      TILEDB_MARIADB_REF:
        description: >-
          The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB-MariaDB
        default: master
      TILEDB_GO_REF:
        description: >-
          The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB-Go
        default: master
jobs:
  libtiledb:
    runs-on: ubuntu-latest
    steps:
      - name: Clone TileDB
        uses: actions/checkout@v3
        with:
          repository: TileDB-Inc/TileDB
          ref: ${{ github.event.inputs.LIBTILEDB_REF || 'dev' }}
      - name: Download release tarball
        run: wget https://github.com/TileDB-Inc/TileDB/releases/download/2.18.0/tiledb-linux-x86_64-2.18.0-71ca8b4.tar.gz
      - name: Unpack release tarball
        run: |
          mkdir install-libtiledb
          tar xzf tiledb-linux-x86_64-*.tar.gz -C install-libtiledb
          ls -R install-libtiledb/
      - name: Patch tiledb.pc from release tarball
        run: |
          new_prefix=prefix=\"$(pwd)/install-libtiledb\"
          echo $new_prefix
          sed -i "1 s+.*+$new_prefix+" install-libtiledb/lib/pkgconfig/tiledb.pc
          cat install-libtiledb/lib/pkgconfig/tiledb.pc
      # - name: Configure TileDB
      #   run: |
      #     cmake -B build-libtiledb \
      #       -D CMAKE_INSTALL_PREFIX:PATH=$(pwd)/install-libtiledb/ \
      #       -D TILEDB_VCPKG=OFF \
      #       -D TILEDB_WERROR=ON \
      #       -D TILEDB_SERIALIZATION=ON \
      #       -D CMAKE_BUILD_TYPE=Release \
      #       -D VCPKG_TARGET_TRIPLET=x64-linux-release
      # - name: Build TileDB
      #   run: cmake --build build-libtiledb -j $(nproc) --config Release
      # - name: Install TileDB
      #   run: cmake --build build-libtiledb --config Release --target install-tiledb
      - name: Upload libtiledb as artifact
        uses: actions/upload-artifact@v3
        with:
          name: nightly-libtiledb-linux
          path: |
            install-libtiledb/*
          retention-days: 14
          if-no-files-found: error
      # - name: Test TileDB
      #   run: cmake --build build-libtiledb --target check --config Release
  tiledb-py:
    runs-on: ubuntu-latest
    needs: libtiledb
    env:
      LD_LIBRARY_PATH: ${{ github.workspace }}/install-libtiledb/lib
      PKG_CONFIG_PATH: ${{ github.workspace }}/install-libtiledb/lib/pkgconfig
    steps:
      - name: Clone TileDB-Py
        uses: actions/checkout@v3
        with:
          repository: TileDB-Inc/TileDB-Py
          ref: ${{ github.event.inputs.TILEDB_PY_REF || 'dev' }}
          fetch-depth: 0 # fetch everything for python setuptools_scm
      - name: Download nightly-libtiledb
        uses: actions/download-artifact@v3
        with:
          name: nightly-libtiledb-linux
          path: install-libtiledb
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"
      - name: Install dependencies
        run: python -m pip install --upgrade -r misc/requirements_ci.txt
      - name: Build TileDB-Py
        run: |
          echo $LD_LIBRARY_PATH
          python setup.py develop --tiledb=$(pwd)/install-libtiledb/
      - name: Import TileDB-Py
        run: |
          python -c "import tiledb; print('successful import')"
          python -c "import tiledb; print(tiledb.libtiledb.version())"
          python -c "import tiledb; print(tiledb.version())"
      - name: Test TileDB-Py
        run: pytest -vv
      - name: Create a wheel
        run: python setup.py bdist_wheel --tiledb=$(pwd)/install-libtiledb/
      - name: Upload Python wheel as artifact
        uses: actions/upload-artifact@v3
        with:
          name: nightly-tiledb-py-linux
          path: dist/tiledb-*-linux_x86_64.whl
          retention-days: 14
          if-no-files-found: error
  tiledb-r:
    runs-on: ubuntu-latest
    needs: libtiledb
    env:
      _R_CHECK_FORCE_SUGGESTS_: "FALSE"
    steps:
      - name: Clone TileDB-R
        uses: actions/checkout@v3
        with:
          repository: TileDB-Inc/TileDB-R
          ref: ${{ github.event.inputs.TILEDB_R_REF || 'master' }}
      - name: Download nightly-libtiledb
        uses: actions/download-artifact@v3
        with:
          name: nightly-libtiledb-linux
          path: install-libtiledb
      - name: Bootstrap
        run: ./.github/r-ci.sh bootstrap
      - name: Dependencies
        run: ./.github/r-ci.sh install_all
      - name: Build
        run: R CMD build --no-manual --no-build-vignettes .
      - name: Install
        run: |
          R CMD INSTALL --configure-args=--with-tiledb=$(pwd)/install-libtiledb --build tiledb_*.tar.gz
      - name: Upload R binary tarball as artifact
        uses: actions/upload-artifact@v3
        with:
          name: nightly-tiledb-r-linux
          path: tiledb_*_R_x86_64-pc-linux-gnu.tar.gz
          retention-days: 14
          if-no-files-found: error
      - name: Test
        run: R -e 'tinytest::test_package("tiledb")'
  libtiledbvcf:
    runs-on: ubuntu-latest
    needs: libtiledb
    env:
      LD_LIBRARY_PATH: ${{ github.workspace }}/install-libtiledb/lib
    steps:
      - name: Clone TileDB-VCF
        uses: actions/checkout@v3
        with:
          repository: TileDB-Inc/TileDB-VCF
          ref: ${{ github.event.inputs.LIBTILEDBVCF_REF || 'main' }}
      - name: Download nightly-libtiledb
        uses: actions/download-artifact@v3
        with:
          name: nightly-libtiledb-linux
          path: install-libtiledb
      - name: Setup
        run: |
          sudo apt-get update
          sudo apt-get install --yes cmake
      - name: Configure libtiledbvcf
        run: |
          cmake -S libtiledbvcf -B build-libtiledbvcf \
            -D CMAKE_BUILD_TYPE=Release \
            -D CMAKE_PREFIX_PATH=$(pwd)/install-libtiledb/ \
            -D CMAKE_INSTALL_PREFIX:PATH=$(pwd)/install-libtiledbvcf/ \
            -D OVERRIDE_INSTALL_PREFIX=OFF \
            -D TILEDB_WERROR=OFF
      - name: Build libtiledbvcf
        run: cmake --build build-libtiledbvcf -j $(nproc) --config Release
      - name: Install libtiledbvcf
        run: |
          cmake --build build-libtiledbvcf --config Release --target install-libtiledbvcf
          ls -R install-libtiledbvcf/
      - name: Upload libtiledbvcf as artifact
        uses: actions/upload-artifact@v3
        with:
          name: nightly-libtiledbvcf-linux
          path: |
            install-libtiledbvcf/*
          retention-days: 14
          if-no-files-found: error
      - name: libtiledbvcf version
        run: ./install-libtiledbvcf/bin/tiledbvcf version
      - name: Confirm linking
        run: ldd install-libtiledbvcf/lib/libtiledbvcf.so
      - name: Install bcftools (for tests)
        run: sudo apt-get install --yes bcftools
      - name: Test libtiledbvcf
        run: |
          make -j $(nproc) -C build-libtiledbvcf/libtiledbvcf tiledb_vcf_unit
          ./build-libtiledbvcf/libtiledbvcf/test/tiledb_vcf_unit

          # cli tests (require bcftools)
          # USAGE: run-cli-tests.sh <build-dir> <inputs-dir>
          libtiledbvcf/test/run-cli-tests.sh build-libtiledbvcf libtiledbvcf/test/inputs
  tiledbvcf-py:
    runs-on: ubuntu-latest
    needs: [libtiledb, libtiledbvcf, tiledb-py]
    env:
      LD_LIBRARY_PATH: "${{ github.workspace }}/install-libtiledb/lib:${{ github.workspace }}/install-libtiledbvcf/lib"
    steps:
      - name: Clone TileDB-VCF
        uses: actions/checkout@v3
        with:
          repository: TileDB-Inc/TileDB-VCF
          ref: ${{ github.event.inputs.TILEDBVCF_PY_REF || 'main' }}
          fetch-depth: 0 # fetch everything for python setuptools_scm
      - name: Download nightly-libtiledb
        uses: actions/download-artifact@v3
        with:
          name: nightly-libtiledb-linux
          path: install-libtiledb
      - name: Download nightly-libtiledbvcf
        uses: actions/download-artifact@v3
        with:
          name: nightly-libtiledbvcf-linux
          path: install-libtiledbvcf
      - name: Download nightly-tiledb-py
        uses: actions/download-artifact@v3
        with:
          name: nightly-tiledb-py-linux
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"
      - name: Install dependencies
        run: python -m pip install \
          dask \
          distributed \
          'fsspec<2023.3.0' \
          pyarrow==10.0.1 \
          pandas \
          pybind11 \
          pytest \
          setuptools \
          setuptools_scm \
          setuptools_scm_git_archive \
          wheel
      - name: Install nightly tiledb-py
        run: |
          pip install tiledb-*-linux_x86_64.whl
          pip list
      - name: Build tiledbvcf-py
        run: |
          echo $LD_LIBRARY_PATH
          cd apis/python
          python setup.py install --libtiledbvcf=$(pwd)/install-libtiledbvcf/
      - name: Confirm linking
        run: ldd $pythonLocation/lib/python*/site-packages/tiledbvcf-*-py*-linux-x86_64.egg/tiledbvcf/libtiledbvcf.cpython-*-x86_64-linux-gnu.so
      - name: Install bcftools (for tests)
        run: |
          sudo apt-get update
          sudo apt-get install --yes bcftools
      - name: Test tiledbvcf-py
        run: |
          python -c "import tiledbvcf; print(tiledbvcf.version)"
          cd apis/python
          pytest
      - name: Create a wheel
        run: |
          cd apis/python
          python setup.py bdist_wheel --libtiledbvcf=$(pwd)/install-libtiledbvcf/
      - name: Upload Python wheel as artifact
        uses: actions/upload-artifact@v3
        with:
          name: nightly-tiledbvcf-py-linux
          path: dist/tiledbvcf-*-linux_x86_64.whl
          retention-days: 14
          if-no-files-found: error
  libtiledbsoma:
    runs-on: ubuntu-latest
    needs: [libtiledb]
    env:
      TileDB_DIR: ${{ github.workspace }}/install-libtiledb
      LD_LIBRARY_PATH: ${{ github.workspace }}/install-libtiledb/lib
    steps:
      - name: Clone TileDB-SOMA
        uses: actions/checkout@v3
        with:
          repository: single-cell-data/TileDB-SOMA
          ref: ${{ github.event.inputs.LIBTILEDBSOMA_REF || 'main' }}
      - name: Download nightly-libtiledb
        uses: actions/download-artifact@v3
        with:
          name: nightly-libtiledb-linux
          path: install-libtiledb
      - name: Setup
        run: |
          sudo apt-get update
          sudo apt-get install --yes cmake
      - name: Configure libtiledbsoma
        run: |
          cmake -S libtiledbsoma -B build-libtiledbsoma \
            -D CMAKE_BUILD_TYPE=Release \
            -D CMAKE_PREFIX_PATH=$(pwd)/install-libtiledb/ \
            -D CMAKE_INSTALL_PREFIX:PATH=$(pwd)/install-libtiledbsoma/ \
            -D OVERRIDE_INSTALL_PREFIX=OFF \
            -D DOWNLOAD_TILEDB_PREBUILT=OFF \
            -D FORCE_BUILD_TILEDB=OFF
      - name: Build libtiledbsoma
        run: cmake --build build-libtiledbsoma -j $(nproc)
      - name: Install libtiledbsoma
        run: |
          cmake --build build-libtiledbsoma --target install-libtiledbsoma
          ls -R $(pwd)/install-libtiledb/
      - name: Upload libtiledbsoma as artifact
        uses: actions/upload-artifact@v3
        with:
          name: nightly-libtiledbsoma-linux
          path: |
            install-libtiledbsoma/*
      - name: Confirm linking
        run: ldd install-libtiledbsoma/lib/libtiledbsoma.so
      - name: Test libtiledbsoma
        run: |
          cmake --build build-libtiledbsoma/libtiledbsoma --target build_tests -j $(nproc)
          ctest --test-dir build-libtiledbsoma/libtiledbsoma -C Release --verbose --rerun-failed --output-on-failure
  tiledbsoma-py:
    runs-on: ubuntu-latest
    needs: [libtiledb, libtiledbsoma, tiledb-py]
    env:
      LD_LIBRARY_PATH: "${{ github.workspace }}/install-libtiledb/lib:${{ github.workspace }}/install-libtiledbsoma/lib"
      TILEDB_PATH: ${{ github.workspace }}/install-libtiledb
      TILEDBSOMA_PATH: ${{ github.workspace }}/install-libtiledbsoma
      PKG_CONFIG_PATH: "${{ github.workspace }}/install-libtiledb/lib/pkgconfig:${{ github.workspace }}/install-libtiledbsoma/lib/pkgconfig"
    steps:
      - name: Clone TileDB-SOMA
        uses: actions/checkout@v3
        with:
          repository: single-cell-data/TileDB-SOMA
          ref: ${{ github.event.inputs.TILEDBSOMA_PY_REF || 'main' }}
          fetch-depth: 0 # fetch everything for python setuptools_scm
      - name: Download nightly-libtiledb
        uses: actions/download-artifact@v3
        with:
          name: nightly-libtiledb-linux
          path: install-libtiledb
      - name: Download nightly-libtiledbsoma
        uses: actions/download-artifact@v3
        with:
          name: nightly-libtiledbsoma-linux
          path: install-libtiledbsoma
      - name: Download nightly-tiledb-py
        uses: actions/download-artifact@v3
        with:
          name: nightly-tiledb-py-linux
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"
      - name: Install dependencies
        run: python -m pip install -r apis/python/requirements_dev.txt wheel
      - name: Install nightly tiledb-py
        run: |
          pip install tiledb-*-linux_x86_64.whl
          pip list
      - name: Install tiledbsoma-py
        run: |
          python -m pip -v install -e apis/python
          pip list
      - name: Search for pybind11 headers
        run: |
          find /opt/hostedtoolcache/Python/3.11.6/x64 -iname pybind11.h
          find /opt/hostedtoolcache/Python/3.11.6/x64 -iname numpy.h
      - name: Confirm INCLUDEPY
        run: python -c "import sysconfig; print(sysconfig.get_config_var('INCLUDEPY'))"
      - name: Confirm linking
        run: |
          find apis/python -name '*.so'
          find apis/python -name '*.so' | xargs ldd
      - name: Remove extra libtiledbsoma.so
        # Deleting this bundled libtiledbsoma.so to ensure the external
        # shared object is being used in the tests
        run: rm -f apis/python/src/tiledbsoma/libtiledbsoma.so
      - name: Test tiledbsoma-py
        run: |
          python -c "import tiledbsoma; print('successful import')"
          python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())"
          python -c "import tiledbsoma; print(f'tiledbsoma-py={tiledbsoma.__version__}')"
          # Obtain test data
          cd test
          tar zxf soco.tgz
          cd ..
          # Run test suite
          PYTHONPATH=$(pwd)/apis/python/src python -m pytest libtiledbsoma/test
          PYTHONPATH=$(pwd)/apis/python/src python -m pytest apis/python/tests
      - name: Create a wheel
        run: |
          cd apis/python
          python setup.py bdist_wheel
      - name: Upload Python wheel as artifact
        uses: actions/upload-artifact@v3
        with:
          name: nightly-tiledbsoma-py-linux
          path: dist/tiledbsoma-*-linux_x86_64.whl
          retention-days: 14
          if-no-files-found: error
  tiledbsoma-r:
    runs-on: ubuntu-latest
    needs: [libtiledb, libtiledbsoma, tiledb-r]
    env:
      LD_LIBRARY_PATH: "${{ github.workspace }}/install-libtiledb/lib:${{ github.workspace }}/install-libtiledbsoma/lib"
      PKG_CONFIG_PATH: "${{ github.workspace }}/install-libtiledb/lib/pkgconfig:${{ github.workspace }}/install-libtiledbsoma/lib/pkgconfig"
    steps:
      - name: Clone TileDB-SOMA
        uses: actions/checkout@v3
        with:
          repository: single-cell-data/TileDB-SOMA
          ref: ${{ github.event.inputs.TILEDBSOMA_R_REF || 'main' }}
      - name: Download nightly-libtiledb
        uses: actions/download-artifact@v3
        with:
          name: nightly-libtiledb-linux
          path: install-libtiledb
      - name: Download nightly-libtiledbsoma
        uses: actions/download-artifact@v3
        with:
          name: nightly-libtiledbsoma-linux
          path: install-libtiledbsoma
      - name: Download nightly-tiledb-r
        uses: actions/download-artifact@v3
        with:
          name: nightly-tiledb-r-linux
      - name: Bootstrap
        run: cd apis/r && tools/r-ci.sh bootstrap
      - name: Install nightly tiledb-r
        run: R CMD INSTALL tiledb_*_R_x86_64-pc-linux-gnu.tar.gz
      - name: Dependencies
        run: cd apis/r && tools/r-ci.sh install_all
      - name: Build
        run: |
          export CXX17FLAGS="-Wno-deprecated-declarations -Wno-deprecated"
          cd apis/r && R CMD build --no-manual --no-build-vignettes .
      - name: Install
        run:
          cd apis/r && R CMD INSTALL --install-tests --build tiledbsoma_*.tar.gz
      - name: Upload R binary tarball as artifact
        uses: actions/upload-artifact@v3
        with:
          name: nightly-tiledbsoma-r-linux
          path: apis/r/tiledbsoma_*_R_x86_64-pc-linux-gnu.tar.gz
          retention-days: 14
          if-no-files-found: error
      - name: Test
        run:
          Rscript -e 'testthat::test_package("tiledbsoma")'
  tiledb-mariadb:
    runs-on: ubuntu-latest
    needs: [libtiledb]
    steps:
      - name: Clone TileDB-MariaDB
        uses: actions/checkout@v3
        with:
          repository: TileDB-Inc/TileDB-MariaDB
          ref: ${{ github.event.inputs.TILEDB_MARIADB_REF || 'master' }}
  tiledb-go:
    runs-on: ubuntu-latest
    needs: [libtiledb]
    steps:
      - name: Clone TileDB-Go
        uses: actions/checkout@v3
        with:
          repository: TileDB-Inc/TileDB-Go
          ref: ${{ github.event.inputs.TILEDB_GO_REF || 'master' }}
      - name: Download nightly-libtiledb
        uses: actions/download-artifact@v3
        with:
          name: nightly-libtiledb-linux
          path: install-libtiledb
      - name: Install dependencies
        run: go get -t .
      - name: Test TileDB-Go
        run: |
          export CPATH=$(pwd)/install-libtiledb/include
          export LIBRARY_PATH=$(pwd)/install-libtiledb/lib
          export LD_LIBRARY_PATH=$(pwd)/install-libtiledb/lib
          go test -v ./...
  build-all:
    # this is simply a target to build everything locally with `act` without
    # creating a new release
    name: Build everything
    runs-on: ubuntu-latest
    needs: [libtiledb, tiledb-py, tiledb-r, libtiledbvcf, tiledbvcf-py, libtiledbsoma,
            tiledbsoma-py, tiledbsoma-r, tiledb-mariadb, tiledb-go]
    steps:
      - run: echo "Success! Everything was built"
  create-release:
    name: Create a new release
    runs-on: ubuntu-latest
    permissions:
      contents: write
    env:
      GH_TOKEN: ${{ github.token }}
      TZ: "America/New_York"
    steps:
      - uses: actions/checkout@v3
      - name: Create release
        run: |
          the_date=$(date +%Y-%m-%d)
          if ! gh release list | cut -f3 | grep -q $the_date
          then
            echo "Creating new release: $the_date"
            gh release create $the_date
          else
            echo "Release already exists: $the_date"
          fi
  upload-libtiledb:
    runs-on: ubuntu-latest
    needs: [create-release, libtiledb]
    permissions:
      contents: write
    env:
      GH_TOKEN: ${{ github.token }}
      TZ: "America/New_York"
    steps:
      - uses: actions/checkout@v3
      - name: Download nightly-libtiledb
        uses: actions/download-artifact@v3
        with:
          name: nightly-libtiledb-linux
          path: install-libtiledb
      - name: Upload libtiledb to release
        run: |
          the_date=$(date +%Y-%m-%d)
          tar -C install-libtiledb -cvzf libtiledb-${the_date}.tar.gz ./
          gh release upload ${the_date} libtiledb-${the_date}.tar.gz --clobber
  upload-libtiledbvcf:
    runs-on: ubuntu-latest
    needs: [create-release, libtiledbvcf]
    permissions:
      contents: write
    env:
      GH_TOKEN: ${{ github.token }}
      TZ: "America/New_York"
    steps:
      - uses: actions/checkout@v3
      - name: Download nightly-libtiledbvcf
        uses: actions/download-artifact@v3
        with:
          name: nightly-libtiledbvcf-linux
          path: install-libtiledbvcf
      - name: Upload libtiledbvcf to release
        run: |
          the_date=$(date +%Y-%m-%d)
          tar -C install-libtiledbvcf -cvzf libtiledbvcf-${the_date}.tar.gz ./
          gh release upload ${the_date} libtiledbvcf-${the_date}.tar.gz --clobber
  upload-libtiledbsoma:
    runs-on: ubuntu-latest
    needs: [create-release, libtiledbsoma]
    permissions:
      contents: write
    env:
      GH_TOKEN: ${{ github.token }}
      TZ: "America/New_York"
    steps:
      - uses: actions/checkout@v3
      - name: Download nightly-libtiledbsoma
        uses: actions/download-artifact@v3
        with:
          name: nightly-libtiledbsoma-linux
          path: install-libtiledbsoma
      - name: Upload libtiledbsoma to release
        run: |
          the_date=$(date +%Y-%m-%d)
          tar -C install-libtiledbsoma -cvzf libtiledbsoma-${the_date}.tar.gz ./
          gh release upload ${the_date} libtiledbsoma-${the_date}.tar.gz --clobber