Ivirshup/census builder spatial

api/python/cellxgene_census/pyproject.toml

@@ -30,7 +30,7 @@ dependencies= [
     # NOTE: the tiledbsoma version must be >= to the version used in the Census builder, to
     # ensure that the assets are readable (tiledbsoma supports backward compatible reading).
     # Make sure this version does not fall behind the builder's tiledbsoma version.
-    "tiledbsoma>=1.12.3,!=1.14.1,<1.15",
+    "tiledbsoma>=1.15.3",
     "anndata",
     "numpy>=1.23,<2.0",
     "requests",

api/python/cellxgene_census/tests/test_open.py

@@ -9,7 +9,6 @@
 import numpy as np
 import pytest
 import requests_mock as rm
-import tiledb
 import tiledbsoma as soma
 
 import cellxgene_census
@@ -442,7 +441,7 @@ def test_opening_census_without_anon_access_fails_with_bogus_creds() -> None:
     os.environ["AWS_SECRET_ACCESS_KEY"] = "fake_key"
     # Passing an empty context
     with pytest.raises(
-        (tiledb.TileDBError, soma.DoesNotExistError),
+        soma.DoesNotExistError,
         match=r"does not exist",
     ):
         cellxgene_census.open_soma(census_version="latest", context=soma.SOMATileDBContext())

tools/cellxgene_census_builder/Dockerfile

@@ -6,12 +6,14 @@ ARG COMMIT_SHA
 ENV COMMIT_SHA=${COMMIT_SHA}
 
 # Ubuntu 22 contains only the python3.11 RC as of 2023-12-21, so use deadsnakes
+# TODO (spatial): `git` is added to this dockerfile to be able to install python packages from github. Remove when it is not needed.
 RUN apt update && \
   apt install -y software-properties-common && \
   add-apt-repository -y ppa:deadsnakes/ppa && \
   apt update && \
   apt -y full-upgrade && \
   apt -y install python3.11 python3.11-venv python3-pip awscli && \
+  apt -y install git && \
   apt-get clean
 
 # set python3.11 as default

tools/cellxgene_census_builder/SPATIAL-README.md

@@ -0,0 +1,9 @@
+## Development Environment Setup and Run
+
+- `pip install -e tools/cellxgene_census_builder`
+
+  **NOTE:** When running the builder on MacOS, unpin `pyarrow` in [census builder pyproject.toml](./pyproject.toml)
+
+- `pip install -e api/python/cellxgene_census`
+
+- [Dev tools for spatial](./spatial_dev_tools/) contains scripts and notebooks to aid development and testing

tools/cellxgene_census_builder/pyproject.toml

@@ -1,9 +1,15 @@
 [build-system]
-requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"]
-build-backend = "setuptools.build_meta"
+# TODO: I've updated the build system so I can install the local version of cellxgene_census using hatch's context formatting._format_
+# This is a little half-assed, and should be either reverted or improved as we approach a release.
+# Right now the wheels/ sdists being produces may include a different set of files since there are no include/ exclude configuration values set
+# https://hatch.pypa.io/dev/config/context/ 
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+# requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"]
+# build-backend = "setuptools.build_meta"
 
 [project]
-name = "cellxgene_census_builder"
+name = "cellxgene-census-builder"
 dynamic = ["version"]
 description = "Build Cell Census"
 authors = [
@@ -24,40 +30,54 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
 ]
 dependencies= [
-    "typing_extensions==4.10.0",
-    "pyarrow==15.0.2",
-    "pandas[performance]==2.2.1",
-    "anndata==0.10.6",
-    "numpy==1.26.4",
+    "typing_extensions>=4.10.0",
+    "pyarrow>=15.0.2",
+    "pandas[performance]>=2.2.1",
+    "anndata",
+    "numpy>=1.26.4",
     # IMPORTANT: consider TileDB format compat before advancing this version. It is important that
     # the tiledbsoma _format_ version lag that used in cellxgene-census package, ensuring that
     # recent cellxgene-census _readers_ are able to read the results of a Census build (writer).
     # The compatibility matrix is defined here:
     #    https://github.com/TileDB-Inc/TileDB/blob/dev/format_spec/FORMAT_SPEC.md
-    "tiledbsoma==1.11.4",
-    "cellxgene-census==1.15.0",
-    "cellxgene-ontology-guide==1.2.0",
-    "scipy==1.12.0",
-    "fsspec[http]==2024.3.1",
-    "s3fs==2024.3.1",
-    "requests==2.32.0",
-    "aiohttp==3.10.2",
-    "Cython", # required by owlready2
-    "wheel",  # required by owlready2
-    "owlready2==0.44",
-    "gitpython==3.1.42",
-    "attrs==23.2.0",
-    "psutil==5.9.8",
-    "pyyaml==6.0.1",
-    "numba==0.59.1",
-    "dask==2024.3.1",
-    "distributed==2024.3.1",
+    # TODO (spatial): tiledbsoma pin to a PyPI release is temporarily commented out in favor git commit pin
+    "tiledbsoma==1.15.3",
+    # TODO (spatial): Pin tiledbsoma dependency to an actual released version after tiledbsoma spatial code has been released
+    # "tiledbsoma @ git+https://github.com/single-cell-data/TileDB-SOMA.git@16467fa7405d59ab1763f103081318b839f87610#egg=tiledbsoma&subdirectory=apis/python/",
+    # TODO (spatial): Deal with the following line before release
+    "cellxgene-census @ {root:parent:parent:uri}/api/python/cellxgene_census",
+    "cellxgene-ontology-guide>=1.2",
+    "scipy>=1.12.0",
+    "fsspec[http]",
+    "s3fs",
+    "requests>=2.32.0",
+    "aiohttp",
+    "gitpython>=3.1.42",
+    "attrs>=23.2.0",
+    "psutil>=5.9.8",
+    "pyyaml>=6.0.1",
+    "numba>=0.59.1",
+    "dask",
+    "distributed",
+    "tiledb",
+    # Spatial testing
+    "filelock",
+    "spatialdata",
+    "pooch",
+    "pytest-xdist",
 ]
 
 [project.urls]
 homepage = "https://github.com/chanzuckerberg/cellxgene-census"
 repository = "https://github.com/chanzuckerberg/cellxgene-census"
 
+[tool.hatch.metadata]
+allow-direct-references = true  # This allows us to pin dependencies to specific commits during development, e.g. tiledbsoma @ git+...
+
+[tool.hatch.version]
+source = "vcs"
+raw-options = { root = "../.." }
+
 [tool.setuptools.packages.find]
 where = ["src"]
 include = ["cellxgene_census_builder*"]  # package names should match these glob patterns (["*"] by default)