Nightly builds on linux #144

Workflow file for this run

	name: Nightly builds on linux
	on:
	push:
	paths:
	- '.github/workflows/linux.yml'
	- 'scripts/**'
	pull_request:
	paths:
	- '.github/workflows/linux.yml'
	- 'scripts/**'
	schedule:
	- cron: "0 2 * * *" # Every night at 2 AM UTC (9 PM EST; 10 PM EDT)
	workflow_dispatch:
	inputs:
	LIBTILEDB_REF:
	description: >-
	The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB
	default: dev
	TILEDB_PY_REF:
	description: >-
	The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB-Py
	default: dev
	TILEDB_R_REF:
	description: >-
	The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB-R
	default: master
	LIBTILEDBVCF_REF:
	description: >-
	The Git reference (tag, branch, commit) to checkout
	TileDB-Inc/TileDB-VCF (for building libtiledbvcf)
	default: main
	TILEDBVCF_PY_REF:
	description: >-
	The Git reference (tag, branch, commit) to checkout
	TileDB-Inc/TileDB-VCF (for building tiledbvcf-py)
	default: main
	LIBTILEDBSOMA_REF:
	description: >-
	The Git reference (tag, branch, commit) to checkout
	single-cell-data/TileDB-SOMA (for building libtiledbsoma)
	default: main
	TILEDBSOMA_PY_REF:
	description: >-
	The Git reference (tag, branch, commit) to checkout
	single-cell-data/TileDB-SOMA (for builing tiledbsoma-py)
	default: main
	TILEDBSOMA_R_REF:
	description: >-
	The Git reference (tag, branch, commit) to checkout
	single-cell-data/TileDB-SOMA (for building tiledbsoma-r)
	default: main
	TILEDB_MARIADB_REF:
	description: >-
	The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB-MariaDB
	default: master
	TILEDB_GO_REF:
	description: >-
	The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB-Go
	default: master
	jobs:
	libtiledb:
	runs-on: ubuntu-latest
	steps:
	- name: Clone TileDB
	uses: actions/checkout@v3
	with:
	repository: TileDB-Inc/TileDB
	ref: ${{ github.event.inputs.LIBTILEDB_REF \|\| 'dev' }}
	- name: Download release tarball
	run: wget https://github.com/TileDB-Inc/TileDB/releases/download/2.18.0/tiledb-linux-x86_64-2.18.0-71ca8b4.tar.gz
	- name: Unpack release tarball
	run: \|
	mkdir install-libtiledb
	tar xzf tiledb-linux-x86_64-*.tar.gz -C install-libtiledb
	ls -R install-libtiledb/
	- name: Patch tiledb.pc from release tarball
	run: \|
	new_prefix=prefix=\"$(pwd)/install-libtiledb\"
	echo $new_prefix
	sed -i "1 s+.*+$new_prefix+" install-libtiledb/lib/pkgconfig/tiledb.pc
	cat install-libtiledb/lib/pkgconfig/tiledb.pc
	# - name: Configure TileDB
	# run: \|
	# cmake -B build-libtiledb \
	# -D CMAKE_INSTALL_PREFIX:PATH=$(pwd)/install-libtiledb/ \
	# -D TILEDB_VCPKG=OFF \
	# -D TILEDB_WERROR=ON \
	# -D TILEDB_SERIALIZATION=ON \
	# -D CMAKE_BUILD_TYPE=Release \
	# -D VCPKG_TARGET_TRIPLET=x64-linux-release
	# - name: Build TileDB
	# run: cmake --build build-libtiledb -j $(nproc) --config Release
	# - name: Install TileDB
	# run: cmake --build build-libtiledb --config Release --target install-tiledb
	- name: Upload libtiledb as artifact
	uses: actions/upload-artifact@v3
	with:
	name: nightly-libtiledb-linux
	path: \|
	install-libtiledb/*
	retention-days: 14
	if-no-files-found: error
	# - name: Test TileDB
	# run: cmake --build build-libtiledb --target check --config Release
	tiledb-py:
	runs-on: ubuntu-latest
	needs: libtiledb
	env:
	LD_LIBRARY_PATH: ${{ github.workspace }}/install-libtiledb/lib
	PKG_CONFIG_PATH: ${{ github.workspace }}/install-libtiledb/lib/pkgconfig
	steps:
	- name: Clone TileDB-Py
	uses: actions/checkout@v3
	with:
	repository: TileDB-Inc/TileDB-Py
	ref: ${{ github.event.inputs.TILEDB_PY_REF \|\| 'dev' }}
	fetch-depth: 0 # fetch everything for python setuptools_scm
	- name: Download nightly-libtiledb
	uses: actions/download-artifact@v3
	with:
	name: nightly-libtiledb-linux
	path: install-libtiledb
	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: "3.11"
	- name: Install dependencies
	run: python -m pip install --upgrade -r misc/requirements_ci.txt
	- name: Build TileDB-Py
	run: \|
	echo $LD_LIBRARY_PATH
	python setup.py develop --tiledb=$(pwd)/install-libtiledb/
	- name: Import TileDB-Py
	run: \|
	python -c "import tiledb; print('successful import')"
	python -c "import tiledb; print(tiledb.libtiledb.version())"
	python -c "import tiledb; print(tiledb.version())"
	- name: Test TileDB-Py
	run: pytest -vv
	- name: Create a wheel
	run: python setup.py bdist_wheel --tiledb=$(pwd)/install-libtiledb/
	- name: Upload Python wheel as artifact
	uses: actions/upload-artifact@v3
	with:
	name: nightly-tiledb-py-linux
	path: dist/tiledb-*-linux_x86_64.whl
	retention-days: 14
	if-no-files-found: error
	tiledb-r:
	runs-on: ubuntu-latest
	needs: libtiledb
	env:
	_R_CHECK_FORCE_SUGGESTS_: "FALSE"
	steps:
	- name: Clone TileDB-R
	uses: actions/checkout@v3
	with:
	repository: TileDB-Inc/TileDB-R
	ref: ${{ github.event.inputs.TILEDB_R_REF \|\| 'master' }}
	- name: Download nightly-libtiledb
	uses: actions/download-artifact@v3
	with:
	name: nightly-libtiledb-linux
	path: install-libtiledb
	- name: Bootstrap
	run: ./.github/r-ci.sh bootstrap
	- name: Dependencies
	run: ./.github/r-ci.sh install_all
	- name: Build
	run: R CMD build --no-manual --no-build-vignettes .
	- name: Install
	run: \|
	R CMD INSTALL --configure-args=--with-tiledb=$(pwd)/install-libtiledb --build tiledb_*.tar.gz
	- name: Upload R binary tarball as artifact
	uses: actions/upload-artifact@v3
	with:
	name: nightly-tiledb-r-linux
	path: tiledb_*_R_x86_64-pc-linux-gnu.tar.gz
	retention-days: 14
	if-no-files-found: error
	- name: Test
	run: R -e 'tinytest::test_package("tiledb")'
	libtiledbvcf:
	runs-on: ubuntu-latest
	needs: libtiledb
	env:
	LD_LIBRARY_PATH: ${{ github.workspace }}/install-libtiledb/lib
	steps:
	- name: Clone TileDB-VCF
	uses: actions/checkout@v3
	with:
	repository: TileDB-Inc/TileDB-VCF
	ref: ${{ github.event.inputs.LIBTILEDBVCF_REF \|\| 'main' }}
	- name: Download nightly-libtiledb
	uses: actions/download-artifact@v3
	with:
	name: nightly-libtiledb-linux
	path: install-libtiledb
	- name: Setup
	run: \|
	sudo apt-get update
	sudo apt-get install --yes cmake
	- name: Configure libtiledbvcf
	run: \|
	cmake -S libtiledbvcf -B build-libtiledbvcf \
	-D CMAKE_BUILD_TYPE=Release \
	-D CMAKE_PREFIX_PATH=$(pwd)/install-libtiledb/ \
	-D CMAKE_INSTALL_PREFIX:PATH=$(pwd)/install-libtiledbvcf/ \
	-D OVERRIDE_INSTALL_PREFIX=OFF \
	-D TILEDB_WERROR=OFF
	- name: Build libtiledbvcf
	run: cmake --build build-libtiledbvcf -j $(nproc) --config Release
	- name: Install libtiledbvcf
	run: \|
	cmake --build build-libtiledbvcf --config Release --target install-libtiledbvcf
	ls -R install-libtiledbvcf/
	- name: Upload libtiledbvcf as artifact
	uses: actions/upload-artifact@v3
	with:
	name: nightly-libtiledbvcf-linux
	path: \|
	install-libtiledbvcf/*
	retention-days: 14
	if-no-files-found: error
	- name: libtiledbvcf version
	run: ./install-libtiledbvcf/bin/tiledbvcf version
	- name: Confirm linking
	run: ldd install-libtiledbvcf/lib/libtiledbvcf.so
	- name: Install bcftools (for tests)
	run: sudo apt-get install --yes bcftools
	- name: Test libtiledbvcf
	run: \|
	make -j $(nproc) -C build-libtiledbvcf/libtiledbvcf tiledb_vcf_unit
	./build-libtiledbvcf/libtiledbvcf/test/tiledb_vcf_unit

	# cli tests (require bcftools)
	# USAGE: run-cli-tests.sh <build-dir> <inputs-dir>
	libtiledbvcf/test/run-cli-tests.sh build-libtiledbvcf libtiledbvcf/test/inputs
	tiledbvcf-py:
	runs-on: ubuntu-latest
	needs: [libtiledb, libtiledbvcf, tiledb-py]
	env:
	LD_LIBRARY_PATH: "${{ github.workspace }}/install-libtiledb/lib:${{ github.workspace }}/install-libtiledbvcf/lib"
	steps:
	- name: Clone TileDB-VCF
	uses: actions/checkout@v3
	with:
	repository: TileDB-Inc/TileDB-VCF
	ref: ${{ github.event.inputs.TILEDBVCF_PY_REF \|\| 'main' }}
	fetch-depth: 0 # fetch everything for python setuptools_scm
	- name: Download nightly-libtiledb
	uses: actions/download-artifact@v3
	with:
	name: nightly-libtiledb-linux
	path: install-libtiledb
	- name: Download nightly-libtiledbvcf
	uses: actions/download-artifact@v3
	with:
	name: nightly-libtiledbvcf-linux
	path: install-libtiledbvcf
	- name: Download nightly-tiledb-py
	uses: actions/download-artifact@v3
	with:
	name: nightly-tiledb-py-linux
	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: "3.11"
	- name: Install dependencies
	run: python -m pip install \
	dask \
	distributed \
	'fsspec<2023.3.0' \
	pyarrow==10.0.1 \
	pandas \
	pybind11 \
	pytest \
	setuptools \
	setuptools_scm \
	setuptools_scm_git_archive \
	wheel
	- name: Install nightly tiledb-py
	run: \|
	pip install tiledb-*-linux_x86_64.whl
	pip list
	- name: Build tiledbvcf-py
	run: \|
	echo $LD_LIBRARY_PATH
	cd apis/python
	python setup.py install --libtiledbvcf=$(pwd)/install-libtiledbvcf/
	- name: Confirm linking
	run: ldd $pythonLocation/lib/python/site-packages/tiledbvcf--py-linux-x86_64.egg/tiledbvcf/libtiledbvcf.cpython--x86_64-linux-gnu.so
	- name: Install bcftools (for tests)
	run: \|
	sudo apt-get update
	sudo apt-get install --yes bcftools
	- name: Test tiledbvcf-py
	run: \|
	python -c "import tiledbvcf; print(tiledbvcf.version)"
	cd apis/python
	pytest
	- name: Create a wheel
	run: \|
	cd apis/python
	python setup.py bdist_wheel --libtiledbvcf=$(pwd)/install-libtiledbvcf/
	- name: Upload Python wheel as artifact
	uses: actions/upload-artifact@v3
	with:
	name: nightly-tiledbvcf-py-linux
	path: dist/tiledbvcf-*-linux_x86_64.whl
	retention-days: 14
	if-no-files-found: error
	libtiledbsoma:
	runs-on: ubuntu-latest
	needs: [libtiledb]
	env:
	TileDB_DIR: ${{ github.workspace }}/install-libtiledb
	LD_LIBRARY_PATH: ${{ github.workspace }}/install-libtiledb/lib
	steps:
	- name: Clone TileDB-SOMA
	uses: actions/checkout@v3
	with:
	repository: single-cell-data/TileDB-SOMA
	ref: ${{ github.event.inputs.LIBTILEDBSOMA_REF \|\| 'main' }}
	- name: Download nightly-libtiledb
	uses: actions/download-artifact@v3
	with:
	name: nightly-libtiledb-linux
	path: install-libtiledb
	- name: Setup
	run: \|
	sudo apt-get update
	sudo apt-get install --yes cmake
	- name: Configure libtiledbsoma
	run: \|
	cmake -S libtiledbsoma -B build-libtiledbsoma \
	-D CMAKE_BUILD_TYPE=Release \
	-D CMAKE_PREFIX_PATH=$(pwd)/install-libtiledb/ \
	-D CMAKE_INSTALL_PREFIX:PATH=$(pwd)/install-libtiledbsoma/ \
	-D OVERRIDE_INSTALL_PREFIX=OFF \
	-D DOWNLOAD_TILEDB_PREBUILT=OFF \
	-D FORCE_BUILD_TILEDB=OFF
	- name: Build libtiledbsoma
	run: cmake --build build-libtiledbsoma -j $(nproc)
	- name: Install libtiledbsoma
	run: \|
	cmake --build build-libtiledbsoma --target install-libtiledbsoma
	ls -R $(pwd)/install-libtiledb/
	- name: Upload libtiledbsoma as artifact
	uses: actions/upload-artifact@v3
	with:
	name: nightly-libtiledbsoma-linux
	path: \|
	install-libtiledbsoma/*
	- name: Confirm linking
	run: ldd install-libtiledbsoma/lib/libtiledbsoma.so
	- name: Test libtiledbsoma
	run: \|
	cmake --build build-libtiledbsoma/libtiledbsoma --target build_tests -j $(nproc)
	ctest --test-dir build-libtiledbsoma/libtiledbsoma -C Release --verbose --rerun-failed --output-on-failure
	tiledbsoma-py:
	runs-on: ubuntu-latest
	needs: [libtiledb, libtiledbsoma, tiledb-py]
	env:
	LD_LIBRARY_PATH: "${{ github.workspace }}/install-libtiledb/lib:${{ github.workspace }}/install-libtiledbsoma/lib"
	TILEDB_PATH: ${{ github.workspace }}/install-libtiledb
	TILEDBSOMA_PATH: ${{ github.workspace }}/install-libtiledbsoma
	PKG_CONFIG_PATH: "${{ github.workspace }}/install-libtiledb/lib/pkgconfig:${{ github.workspace }}/install-libtiledbsoma/lib/pkgconfig"
	steps:
	- name: Clone TileDB-SOMA
	uses: actions/checkout@v3
	with:
	repository: single-cell-data/TileDB-SOMA
	ref: ${{ github.event.inputs.TILEDBSOMA_PY_REF \|\| 'main' }}
	fetch-depth: 0 # fetch everything for python setuptools_scm
	- name: Download nightly-libtiledb
	uses: actions/download-artifact@v3
	with:
	name: nightly-libtiledb-linux
	path: install-libtiledb
	- name: Download nightly-libtiledbsoma
	uses: actions/download-artifact@v3
	with:
	name: nightly-libtiledbsoma-linux
	path: install-libtiledbsoma
	- name: Download nightly-tiledb-py
	uses: actions/download-artifact@v3
	with:
	name: nightly-tiledb-py-linux
	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: "3.11"
	- name: Install dependencies
	run: python -m pip install -r apis/python/requirements_dev.txt wheel
	- name: Install nightly tiledb-py
	run: \|
	pip install tiledb-*-linux_x86_64.whl
	pip list
	- name: Install tiledbsoma-py
	run: \|
	python -m pip -v install -e apis/python
	pip list
	- name: Search for pybind11 headers
	run: \|
	find /opt/hostedtoolcache/Python/3.11.6/x64 -iname pybind11.h
	find /opt/hostedtoolcache/Python/3.11.6/x64 -iname numpy.h
	- name: Confirm INCLUDEPY
	run: python -c "import sysconfig; print(sysconfig.get_config_var('INCLUDEPY'))"
	- name: Confirm linking
	run: \|
	find apis/python -name '*.so'
	find apis/python -name '*.so' \| xargs ldd
	- name: Remove extra libtiledbsoma.so
	# Deleting this bundled libtiledbsoma.so to ensure the external
	# shared object is being used in the tests
	run: rm -f apis/python/src/tiledbsoma/libtiledbsoma.so
	- name: Test tiledbsoma-py
	run: \|
	python -c "import tiledbsoma; print('successful import')"
	python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())"
	python -c "import tiledbsoma; print(f'tiledbsoma-py={tiledbsoma.__version__}')"
	# Obtain test data
	cd test
	tar zxf soco.tgz
	cd ..
	# Run test suite
	PYTHONPATH=$(pwd)/apis/python/src python -m pytest libtiledbsoma/test
	PYTHONPATH=$(pwd)/apis/python/src python -m pytest apis/python/tests
	- name: Create a wheel
	run: \|
	cd apis/python
	python setup.py bdist_wheel
	- name: Upload Python wheel as artifact
	uses: actions/upload-artifact@v3
	with:
	name: nightly-tiledbsoma-py-linux
	path: dist/tiledbsoma-*-linux_x86_64.whl
	retention-days: 14
	if-no-files-found: error
	tiledbsoma-r:
	runs-on: ubuntu-latest
	needs: [libtiledb, libtiledbsoma, tiledb-r]
	env:
	LD_LIBRARY_PATH: "${{ github.workspace }}/install-libtiledb/lib:${{ github.workspace }}/install-libtiledbsoma/lib"
	PKG_CONFIG_PATH: "${{ github.workspace }}/install-libtiledb/lib/pkgconfig:${{ github.workspace }}/install-libtiledbsoma/lib/pkgconfig"
	steps:
	- name: Clone TileDB-SOMA
	uses: actions/checkout@v3
	with:
	repository: single-cell-data/TileDB-SOMA
	ref: ${{ github.event.inputs.TILEDBSOMA_R_REF \|\| 'main' }}
	- name: Download nightly-libtiledb
	uses: actions/download-artifact@v3
	with:
	name: nightly-libtiledb-linux
	path: install-libtiledb
	- name: Download nightly-libtiledbsoma
	uses: actions/download-artifact@v3
	with:
	name: nightly-libtiledbsoma-linux
	path: install-libtiledbsoma
	- name: Download nightly-tiledb-r
	uses: actions/download-artifact@v3
	with:
	name: nightly-tiledb-r-linux
	- name: Bootstrap
	run: cd apis/r && tools/r-ci.sh bootstrap
	- name: Install nightly tiledb-r
	run: R CMD INSTALL tiledb_*_R_x86_64-pc-linux-gnu.tar.gz
	- name: Dependencies
	run: cd apis/r && tools/r-ci.sh install_all
	- name: Build
	run: \|
	export CXX17FLAGS="-Wno-deprecated-declarations -Wno-deprecated"
	cd apis/r && R CMD build --no-manual --no-build-vignettes .
	- name: Install
	run:
	cd apis/r && R CMD INSTALL --install-tests --build tiledbsoma_*.tar.gz
	- name: Upload R binary tarball as artifact
	uses: actions/upload-artifact@v3
	with:
	name: nightly-tiledbsoma-r-linux
	path: apis/r/tiledbsoma_*_R_x86_64-pc-linux-gnu.tar.gz
	retention-days: 14
	if-no-files-found: error
	- name: Test
	run:
	Rscript -e 'testthat::test_package("tiledbsoma")'
	tiledb-mariadb:
	runs-on: ubuntu-latest
	needs: [libtiledb]
	steps:
	- name: Clone TileDB-MariaDB
	uses: actions/checkout@v3
	with:
	repository: TileDB-Inc/TileDB-MariaDB
	ref: ${{ github.event.inputs.TILEDB_MARIADB_REF \|\| 'master' }}
	tiledb-go:
	runs-on: ubuntu-latest
	needs: [libtiledb]
	steps:
	- name: Clone TileDB-Go
	uses: actions/checkout@v3
	with:
	repository: TileDB-Inc/TileDB-Go
	ref: ${{ github.event.inputs.TILEDB_GO_REF \|\| 'master' }}
	- name: Download nightly-libtiledb
	uses: actions/download-artifact@v3
	with:
	name: nightly-libtiledb-linux
	path: install-libtiledb
	- name: Install dependencies
	run: go get -t .
	- name: Test TileDB-Go
	run: \|
	export CPATH=$(pwd)/install-libtiledb/include
	export LIBRARY_PATH=$(pwd)/install-libtiledb/lib
	export LD_LIBRARY_PATH=$(pwd)/install-libtiledb/lib
	go test -v ./...
	build-all:
	# this is simply a target to build everything locally with `act` without
	# creating a new release
	name: Build everything
	runs-on: ubuntu-latest
	needs: [libtiledb, tiledb-py, tiledb-r, libtiledbvcf, tiledbvcf-py, libtiledbsoma,
	tiledbsoma-py, tiledbsoma-r, tiledb-mariadb, tiledb-go]
	steps:
	- run: echo "Success! Everything was built"
	create-release:
	name: Create a new release
	runs-on: ubuntu-latest
	permissions:
	contents: write
	env:
	GH_TOKEN: ${{ github.token }}
	TZ: "America/New_York"
	steps:
	- uses: actions/checkout@v3
	- name: Create release
	run: \|
	the_date=$(date +%Y-%m-%d)
	if ! gh release list \| cut -f3 \| grep -q $the_date
	then
	echo "Creating new release: $the_date"
	gh release create $the_date
	else
	echo "Release already exists: $the_date"
	fi
	upload-libtiledb:
	runs-on: ubuntu-latest
	needs: [create-release, libtiledb]
	permissions:
	contents: write
	env:
	GH_TOKEN: ${{ github.token }}
	TZ: "America/New_York"
	steps:
	- uses: actions/checkout@v3
	- name: Download nightly-libtiledb
	uses: actions/download-artifact@v3
	with:
	name: nightly-libtiledb-linux
	path: install-libtiledb
	- name: Upload libtiledb to release
	run: \|
	the_date=$(date +%Y-%m-%d)
	tar -C install-libtiledb -cvzf libtiledb-${the_date}.tar.gz ./
	gh release upload ${the_date} libtiledb-${the_date}.tar.gz --clobber
	upload-libtiledbvcf:
	runs-on: ubuntu-latest
	needs: [create-release, libtiledbvcf]
	permissions:
	contents: write
	env:
	GH_TOKEN: ${{ github.token }}
	TZ: "America/New_York"
	steps:
	- uses: actions/checkout@v3
	- name: Download nightly-libtiledbvcf
	uses: actions/download-artifact@v3
	with:
	name: nightly-libtiledbvcf-linux
	path: install-libtiledbvcf
	- name: Upload libtiledbvcf to release
	run: \|
	the_date=$(date +%Y-%m-%d)
	tar -C install-libtiledbvcf -cvzf libtiledbvcf-${the_date}.tar.gz ./
	gh release upload ${the_date} libtiledbvcf-${the_date}.tar.gz --clobber
	upload-libtiledbsoma:
	runs-on: ubuntu-latest
	needs: [create-release, libtiledbsoma]
	permissions:
	contents: write
	env:
	GH_TOKEN: ${{ github.token }}
	TZ: "America/New_York"
	steps:
	- uses: actions/checkout@v3
	- name: Download nightly-libtiledbsoma
	uses: actions/download-artifact@v3
	with:
	name: nightly-libtiledbsoma-linux
	path: install-libtiledbsoma
	- name: Upload libtiledbsoma to release
	run: \|
	the_date=$(date +%Y-%m-%d)
	tar -C install-libtiledbsoma -cvzf libtiledbsoma-${the_date}.tar.gz ./
	gh release upload ${the_date} libtiledbsoma-${the_date}.tar.gz --clobber

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Nightly builds on linux #144

Workflow file

Nightly builds on linux #144

Jobs

Run details

Workflow file for this run