Nightly builds on linux #63
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly builds on linux | |
| on: | |
| push: | |
| paths: | |
| - '.github/workflows/linux.yml' | |
| - 'scripts/**' | |
| pull_request: | |
| paths: | |
| - '.github/workflows/linux.yml' | |
| - 'scripts/**' | |
| schedule: | |
| - cron: "0 2 * * *" # Every night at 2 AM UTC (9 PM EST; 10 PM EDT) | |
| workflow_dispatch: | |
| inputs: | |
| LIBTILEDB_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB | |
| default: dev | |
| TILEDB_PY_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB-Py | |
| default: dev | |
| TILEDB_R_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB-R | |
| default: master | |
| LIBTILEDBVCF_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout | |
| TileDB-Inc/TileDB-VCF (for building libtiledbvcf) | |
| default: main | |
| TILEDBVCF_PY_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout | |
| TileDB-Inc/TileDB-VCF (for building tiledbvcf-py) | |
| default: main | |
| LIBTILEDBSOMA_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout | |
| single-cell-data/TileDB-SOMA (for building libtiledbsoma) | |
| default: main | |
| TILEDBSOMA_PY_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout | |
| single-cell-data/TileDB-SOMA (for builing tiledbsoma-py) | |
| default: main | |
| TILEDBSOMA_R_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout | |
| single-cell-data/TileDB-SOMA (for building tiledbsoma-r) | |
| default: main | |
| TILEDB_MARIADB_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB-MariaDB | |
| default: master | |
| TILEDB_GO_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB-Go | |
| default: master | |
| jobs: | |
| libtiledb: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Clone TileDB | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: TileDB-Inc/TileDB | |
| ref: ${{ github.event.inputs.LIBTILEDB_REF || 'dev' }} | |
| - name: Download release tarball | |
| run: wget https://github.com/TileDB-Inc/TileDB/releases/download/2.17.1/tiledb-linux-x86_64-2.17.1-7c8c73b.tar.gz | |
| - name: Unpack release tarball | |
| run: | | |
| mkdir install-libtiledb | |
| tar xzf tiledb-linux-x86_64-*.tar.gz -C install-libtiledb | |
| ls -R install-libtiledb/ | |
| # - name: Configure TileDB | |
| # run: | | |
| # cmake -B build-libtiledb \ | |
| # -D CMAKE_INSTALL_PREFIX:PATH=$GITHUB_WORKSPACE/install-libtiledb/ \ | |
| # -D TILEDB_VCPKG=OFF \ | |
| # -D TILEDB_WERROR=ON \ | |
| # -D TILEDB_SERIALIZATION=ON \ | |
| # -D CMAKE_BUILD_TYPE=Release | |
| # - name: Build TileDB | |
| # run: cmake --build build-libtiledb -j2 --config Release | |
| # - name: Install TileDB | |
| # run: cmake --build build-libtiledb --config Release --target install-tiledb | |
| - name: Upload libtiledb as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: | | |
| install-libtiledb/* | |
| retention-days: 14 | |
| # - name: Test TileDB | |
| # run: cmake --build build-libtiledb --target check --config Release | |
| tiledb-py: | |
| runs-on: ubuntu-latest | |
| needs: libtiledb | |
| env: | |
| LD_LIBRARY_PATH: ${{ github.workspace }}/install-libtiledb/lib | |
| steps: | |
| - name: Clone TileDB-Py | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: TileDB-Inc/TileDB-Py | |
| ref: ${{ github.event.inputs.TILEDB_PY_REF || 'dev' }} | |
| fetch-depth: 0 # fetch everything for python setuptools_scm | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: python -m pip install --upgrade -r misc/requirements_ci.txt | |
| - name: Build TileDB-Py | |
| run: | | |
| echo $LD_LIBRARY_PATH | |
| python setup.py install --tiledb=$GITHUB_WORKSPACE/install-libtiledb/ | |
| # - name: Test TileDB-Py | |
| # run: | | |
| # python -c "import tiledb; print(tiledb.libtiledb.version())" | |
| # pytest -vv | |
| - name: Create a wheel | |
| run: python setup.py bdist_wheel --tiledb=$GITHUB_WORKSPACE/install-libtiledb/ | |
| - name: Upload Python wheel as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: nightly-tiledb-py-linux | |
| path: dist/tiledb-*-linux_x86_64.whl | |
| retention-days: 14 | |
| tiledb-r: | |
| runs-on: ubuntu-latest | |
| needs: libtiledb | |
| env: | |
| _R_CHECK_FORCE_SUGGESTS_: "FALSE" | |
| steps: | |
| - name: Clone TileDB-R | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: TileDB-Inc/TileDB-R | |
| ref: ${{ github.event.inputs.TILEDB_R_REF || 'master' }} | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Bootstrap | |
| run: ./.github/r-ci.sh bootstrap | |
| - name: Dependencies | |
| run: ./.github/r-ci.sh install_all | |
| - name: Build | |
| run: R CMD build --no-manual --no-build-vignettes . | |
| - name: Install | |
| run: | | |
| R CMD INSTALL --configure-args=--with-tiledb=$GITHUB_WORKSPACE/install-libtiledb --build tiledb_*.tar.gz | |
| - name: Upload R binary tarball as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: nightly-tiledb-r-linux | |
| path: tiledb_*_R_x86_64-pc-linux-gnu.tar.gz | |
| retention-days: 14 | |
| - name: Test | |
| run: R -e 'tinytest::test_package("tiledb")' | |
| libtiledbvcf: | |
| runs-on: ubuntu-latest | |
| needs: libtiledb | |
| env: | |
| LD_LIBRARY_PATH: ${{ github.workspace }}/install-libtiledb/lib | |
| steps: | |
| - name: Clone TileDB-VCF | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: TileDB-Inc/TileDB-VCF | |
| ref: ${{ github.event.inputs.LIBTILEDBVCF_REF || 'main' }} | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Configure libtiledbvcf | |
| run: | | |
| cmake -S libtiledbvcf -B build-libtiledbvcf \ | |
| -D CMAKE_BUILD_TYPE=Release \ | |
| -D CMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/install-libtiledb/ \ | |
| -D CMAKE_INSTALL_PREFIX:PATH=$GITHUB_WORKSPACE/install-libtiledbvcf/ \ | |
| -D OVERRIDE_INSTALL_PREFIX=OFF \ | |
| -D TILEDB_WERROR=OFF | |
| - name: Build libtiledbvcf | |
| run: cmake --build build-libtiledbvcf -j2 --config Release | |
| - name: Install libtiledbvcf | |
| run: | | |
| cmake --build build-libtiledbvcf --config Release --target install-libtiledbvcf | |
| ls -R install-libtiledbvcf/ | |
| - name: Upload libtiledbvcf as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: nightly-libtiledbvcf-linux | |
| path: | | |
| install-libtiledbvcf/* | |
| retention-days: 14 | |
| - name: libtiledbvcf version | |
| run: ./install-libtiledbvcf/bin/tiledbvcf version | |
| - name: Confirm linking | |
| run: ldd install-libtiledbvcf/lib/libtiledbvcf.so | |
| - name: Install bcftools (for tests) | |
| run: sudo apt install bcftools | |
| - name: Test libtiledbvcf | |
| run: | | |
| make -j2 -C build-libtiledbvcf/libtiledbvcf tiledb_vcf_unit | |
| ./build-libtiledbvcf/libtiledbvcf/test/tiledb_vcf_unit | |
| # cli tests (require bcftools) | |
| # USAGE: run-cli-tests.sh <build-dir> <inputs-dir> | |
| libtiledbvcf/test/run-cli-tests.sh build-libtiledbvcf libtiledbvcf/test/inputs | |
| tiledbvcf-py: | |
| runs-on: ubuntu-latest | |
| needs: [libtiledb, libtiledbvcf, tiledb-py] | |
| env: | |
| LD_LIBRARY_PATH: "${{ github.workspace }}/install-libtiledb/lib:${{ github.workspace }}/install-libtiledbvcf/lib" | |
| steps: | |
| - name: Clone TileDB-VCF | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: TileDB-Inc/TileDB-VCF | |
| ref: ${{ github.event.inputs.TILEDBVCF_PY_REF || 'main' }} | |
| fetch-depth: 0 # fetch everything for python setuptools_scm | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Download nightly-libtiledbvcf | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-libtiledbvcf-linux | |
| path: install-libtiledbvcf | |
| - name: Download nightly-tiledb-py | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-tiledb-py-linux | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: python -m pip install \ | |
| dask \ | |
| distributed \ | |
| 'fsspec<2023.3.0' \ | |
| pyarrow==10.0.1 \ | |
| pandas \ | |
| pybind11 \ | |
| pytest \ | |
| setuptools \ | |
| setuptools_scm \ | |
| setuptools_scm_git_archive \ | |
| wheel | |
| - name: Install nightly tiledb-py | |
| run: | | |
| pip install tiledb-*-linux_x86_64.whl | |
| pip list | |
| - name: Build tiledbvcf-py | |
| run: | | |
| echo $LD_LIBRARY_PATH | |
| cd apis/python | |
| python setup.py install --libtiledbvcf=$GITHUB_WORKSPACE/install-libtiledbvcf/ | |
| - name: Confirm linking | |
| run: ldd $pythonLocation/lib/python*/site-packages/tiledbvcf-*-py*-linux-x86_64.egg/tiledbvcf/libtiledbvcf.cpython-*-x86_64-linux-gnu.so | |
| - name: Install bcftools (for tests) | |
| run: sudo apt install bcftools | |
| - name: Test tiledbvcf-py | |
| run: | | |
| python -c "import tiledbvcf; print(tiledbvcf.version)" | |
| cd apis/python | |
| pytest | |
| - name: Create a wheel | |
| run: | | |
| cd apis/python | |
| python setup.py bdist_wheel --libtiledbvcf=$GITHUB_WORKSPACE/install-libtiledbvcf/ | |
| - name: Upload Python wheel as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: nightly-tiledbvcf-py-linux | |
| path: dist/tiledbvcf-*-linux_x86_64.whl | |
| retention-days: 14 | |
| libtiledbsoma: | |
| runs-on: ubuntu-latest | |
| needs: [libtiledb] | |
| env: | |
| TileDB_DIR: ${{ github.workspace }}/install-libtiledb | |
| LD_LIBRARY_PATH: ${{ github.workspace }}/install-libtiledb/lib | |
| steps: | |
| - name: Clone TileDB-SOMA | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: single-cell-data/TileDB-SOMA | |
| ref: ${{ github.event.inputs.LIBTILEDBSOMA_REF || 'main' }} | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Configure libtiledbsoma | |
| run: | | |
| cmake -S libtiledbsoma -B build-libtiledbsoma \ | |
| -D CMAKE_BUILD_TYPE=Release \ | |
| -D CMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/install-libtiledb/ \ | |
| -D CMAKE_INSTALL_PREFIX:PATH=$GITHUB_WORKSPACE/install-libtiledbsoma/ \ | |
| -D OVERRIDE_INSTALL_PREFIX=OFF \ | |
| -D DOWNLOAD_TILEDB_PREBUILT=OFF \ | |
| -D FORCE_BUILD_TILEDB=OFF | |
| - name: Build libtiledbsoma | |
| run: cmake --build build-libtiledbsoma -j2 | |
| - name: Install libtiledbsoma | |
| run: | | |
| cmake --build build-libtiledbsoma --target install-libtiledbsoma | |
| ls -R $GITHUB_WORKSPACE/install-libtiledb/ | |
| - name: Upload libtiledbsoma as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: nightly-libtiledbsoma-linux | |
| path: | | |
| install-libtiledbsoma/* | |
| - name: Confirm linking | |
| run: ldd install-libtiledbsoma/lib/libtiledbsoma.so | |
| - name: Test libtiledbsoma | |
| run: | | |
| cmake --build build-libtiledbsoma/libtiledbsoma --target build_tests -j2 | |
| ctest --test-dir build-libtiledbsoma/libtiledbsoma -C Release --verbose --rerun-failed --output-on-failure | |
| tiledbsoma-py: | |
| runs-on: ubuntu-latest | |
| needs: [libtiledb, libtiledbsoma, tiledb-py] | |
| env: | |
| LD_LIBRARY_PATH: "${{ github.workspace }}/install-libtiledb/lib:${{ github.workspace }}/install-libtiledbsoma/lib" | |
| TILEDB_PATH: ${{ github.workspace }}/install-libtiledb | |
| TILEDBSOMA_PATH: ${{ github.workspace }}/install-libtiledbsoma | |
| steps: | |
| - name: Clone TileDB-SOMA | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: single-cell-data/TileDB-SOMA | |
| ref: ${{ github.event.inputs.TILEDBSOMA_PY_REF || 'main' }} | |
| fetch-depth: 0 # fetch everything for python setuptools_scm | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Download nightly-libtiledbsoma | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-libtiledbsoma-linux | |
| path: install-libtiledbsoma | |
| - name: Download nightly-tiledb-py | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-tiledb-py-linux | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: python -m pip install \ | |
| pybind11-global \ | |
| pytest \ | |
| 'typeguard<3.0' \ | |
| types-setuptools | |
| - name: Install nightly tiledb-py | |
| run: | | |
| pip install tiledb-*-linux_x86_64.whl | |
| pip list | |
| - name: Install spdlog | |
| run: sudo apt install libspdlog-dev | |
| - name: Install tiledbsoma-py | |
| run: | | |
| python -m pip -v install apis/python | |
| pip list | |
| - name: Confirm linking | |
| run: ldd $pythonLocation/lib/python*/site-packages/tiledbsoma/pytiledbsoma.cpython-*-x86_64-linux-gnu.so | |
| - name: Test tiledbsoma-py | |
| run: | | |
| python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())" | |
| cd apis/python | |
| pytest | |
| - name: Create a wheel | |
| run: | | |
| cd apis/python | |
| python setup.py bdist_wheel | |
| - name: Upload Python wheel as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: nightly-tiledbsoma-py-linux | |
| path: dist/tiledbsoma-*-linux_x86_64.whl | |
| retention-days: 14 | |
| tiledbsoma-r: | |
| runs-on: ubuntu-latest | |
| needs: [libtiledb, libtiledbsoma, tiledb-r] | |
| steps: | |
| - name: Clone TileDB-SOMA | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: single-cell-data/TileDB-SOMA | |
| ref: ${{ github.event.inputs.TILEDBSOMA_R_REF || 'main' }} | |
| tiledb-mariadb: | |
| runs-on: ubuntu-latest | |
| needs: [libtiledb] | |
| steps: | |
| - name: Clone TileDB-MariaDB | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: TileDB-Inc/TileDB-MariaDB | |
| ref: ${{ github.event.inputs.TILEDB_MARIADB_REF || 'master' }} | |
| tiledb-go: | |
| runs-on: ubuntu-latest | |
| needs: [libtiledb] | |
| steps: | |
| - name: Clone TileDB-Go | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: TileDB-Inc/TileDB-Go | |
| ref: ${{ github.event.inputs.TILEDB_GO_REF || 'master' }} | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Install dependencies | |
| run: go get -t . | |
| - name: Test TileDB-Go | |
| run: | | |
| export CPATH=$GITHUB_WORKSPACE/install-libtiledb/include | |
| export LIBRARY_PATH=$GITHUB_WORKSPACE/install-libtiledb/lib | |
| export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install-libtiledb/lib | |
| go test -v ./... | |
| create-release: | |
| name: Create a new release | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| TZ: "America/New_York" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Create release | |
| run: | | |
| the_date=$(date +%Y-%m-%d) | |
| if ! gh release list | grep -q $the_date | |
| then | |
| echo "Creating new release: $the_date" | |
| gh release create $the_date | |
| else | |
| echo "Release already exists: $the_date" | |
| fi | |
| upload-libtiledb: | |
| runs-on: ubuntu-latest | |
| needs: [create-release, libtiledb] | |
| permissions: | |
| contents: write | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| TZ: "America/New_York" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Upload libtiledb to release | |
| run: | | |
| the_date=$(date +%Y-%m-%d) | |
| tar -C install-libtiledb -cvzf libtiledb-${the_date}.tar.gz ./ | |
| gh release upload ${the_date} libtiledb-${the_date}.tar.gz --clobber | |
| upload-libtiledbvcf: | |
| runs-on: ubuntu-latest | |
| needs: [create-release, libtiledbvcf] | |
| permissions: | |
| contents: write | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| TZ: "America/New_York" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Download nightly-libtiledbvcf | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-libtiledbvcf-linux | |
| path: install-libtiledbvcf | |
| - name: Upload libtiledbvcf to release | |
| run: | | |
| the_date=$(date +%Y-%m-%d) | |
| tar -C install-libtiledbvcf -cvzf libtiledbvcf-${the_date}.tar.gz ./ | |
| gh release upload ${the_date} libtiledbvcf-${the_date}.tar.gz --clobber | |
| upload-libtiledbsoma: | |
| runs-on: ubuntu-latest | |
| needs: [create-release, libtiledbsoma] | |
| permissions: | |
| contents: write | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| TZ: "America/New_York" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Download nightly-libtiledbsoma | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-libtiledbsoma-linux | |
| path: install-libtiledbsoma | |
| - name: Upload libtiledbsoma to release | |
| run: | | |
| the_date=$(date +%Y-%m-%d) | |
| tar -C install-libtiledbsoma -cvzf libtiledbsoma-${the_date}.tar.gz ./ | |
| gh release upload ${the_date} libtiledbsoma-${the_date}.tar.gz --clobber |