Merge pull request #4 from fractal-analytics-platform/roi_loader

[WIP] Roi loader, OMEZarrImage class & automated testing

.github/workflows/test_and_deploy.yml

@@ -63,6 +63,13 @@ jobs:
       - name: Coverage
         uses: codecov/codecov-action@v3
 
+      - name: Cache Pooch folder
+        id: cache-pooch-folder
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pooch
+          key: pooch-cache
+
   deploy:
     # this will run when you have tagged a commit, starting with "v*"
     # and requires that you have put your twine API key in your

pyproject.toml

@@ -30,6 +30,7 @@ ignore = [
     "E501", # line too long. let black handle this
     "UP006", "UP007", # type annotation. As using magicgui require runtime type annotation then we disable this.
     "SIM117", # flake8-simplify - some of merged with statements are not looking great with black, reanble after drop python 3.9
+    "G004", # Logging statement uses f-string
 ]
 
 exclude = [

setup.cfg

@@ -35,7 +35,8 @@ install_requires =
     magicgui
     qtpy
     scikit-image
-    fractal-tasks-core==0.14.0
+    fractal-tasks-core==0.14.3
+    ome-zarr
     wget
 
 python_requires = >=3.8

src/napari_ome_zarr_navigator/_sample_data.py

@@ -1,21 +1,111 @@
-"""
-This module is an example of a barebones sample data provider for napari.
+from __future__ import annotations
 
-It implements the "sample data" specification.
-see: https://napari.org/stable/plugins/guides.html?#sample-data
+from pathlib import Path
+from typing import Union
+import re
+import requests
+import urllib
+import hashlib
+import wget
+import shutil
 
-Replace code below according to your needs.
-"""
-from __future__ import annotations
+from napari.types import LayerDataTuple
+
+from ome_zarr.io import parse_url
+from ome_zarr.reader import Reader, Node
+
+from napari_ome_zarr_navigator import _TEST_DATA_DIR
+
+
+def load_ome_zarr_from_zenodo(doi: str, zarr_url: Union[str, Path]):
+    doi_path = Path(_TEST_DATA_DIR).joinpath(doi.replace("/", "_"))
+    zarr_path = doi_path.joinpath(zarr_url)
+    if not doi_path.is_dir():
+        download_from_zenodo(doi, directory=doi_path)
+        shutil.unpack_archive(zarr_path.with_suffix(".zarr.zip"), doi_path)
+    reader = Reader(parse_url(zarr_path))
+    zarr_group = list(reader())[0]
+    return zarr_group
+
+
+def download_from_zenodo(
+    doi: str,
+    overwrite: bool = False,
+    directory: Union[str, Path] = Path(),
+    access_token: str = None,
+):
+    record_id = re.match(r".*/zenodo.(\w+)", doi).group(1)
+    url = "https://zenodo.org/api/records/" + record_id
+    js = requests.get(url).json()
+    doi = js["metadata"]["doi"]
+    print("Title: " + js["metadata"]["title"])
+    print("Publication date: " + js["metadata"]["publication_date"])
+    print("DOI: " + js["metadata"]["doi"])
+    print(
+        "Total file size: {:.1f} MB".format(
+            sum(f["size"] / 10**6 for f in js["files"])
+        )
+    )
+    doi_path = Path(directory)
+    try:
+        doi_path.mkdir(exist_ok=overwrite, parents=True)
+    except FileExistsError:
+        print(f"{doi_path} exists. Don't overwrite.")
+        return
+    for file in js["files"]:
+        file_path = Path(doi_path).joinpath(file["key"])
+        algorithm, checksum = file["checksum"].split(":")
+        try:
+            link = urllib.parse.unquote(file["links"]["self"])
+            wget.download(
+                f"{link}?access_token={access_token}", str(directory)
+            )
+            check_passed, returned_checksum = verify_checksum(
+                file_path, algorithm, checksum
+            )
+            if check_passed:
+                print(f"\nChecksum is correct. ({checksum})")
+            else:
+                print(
+                    f"\nChecksum is incorrect! ({checksum} got: {returned_checksum})"
+                )
+        except urllib.error.HTTPError:
+            pass
+
+
+def verify_checksum(filename: Union[str, Path], algorithm, original_checksum):
+    h = hashlib.new(algorithm)
+    with open(filename, "rb") as f:
+        h.update(f.read())
+        returned_checksum = h.hexdigest()
+    if returned_checksum == original_checksum:
+        return True, returned_checksum
+    else:
+        return False, returned_checksum
+
+
+def hiPSC_zarr() -> list[LayerDataTuple]:
+    doi = "10.5281_zenodo.10424292"
+    zarr_url = "20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr"
+    return load_zarr(doi, zarr_url)
 
-import numpy
 
+def load_zarr(doi: str, zarr_url: Union[str, Path]) -> list[LayerDataTuple]:
+    ome_zarr = load_ome_zarr_from_zenodo(doi, zarr_url)
+    if ome_zarr:
 
-def make_sample_data():
-    """Generates an image"""
-    # Return list of tuples
-    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
-    # Check the documentation for more information about the
-    # add_image_kwargs
-    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
-    return [(numpy.random.rand(512, 512), {})]
+        return [
+            (
+                ome_zarr.data,
+                {
+                    "name": ome_zarr.metadata["name"],
+                    "channel_axis": 0,
+                    "contrast_limits": ome_zarr.metadata["contrast_limits"],
+                    "colormap": ome_zarr.metadata["colormap"],
+                    "metadata": {"sample_path": ome_zarr.zarr.path},
+                },
+                "image",
+            )
+        ]
+    else:
+        return [(None,)]

src/napari_ome_zarr_navigator/_tests/conftest.py

@@ -0,0 +1,63 @@
+import os
+import shutil
+from pathlib import Path
+
+import pooch
+import pytest
+
+
+@pytest.fixture(scope="session")
+def testdata_path() -> Path:
+    TEST_DIR = Path(__file__).parent.parent.parent.parent
+    return TEST_DIR / "test_data/"
+
+
+@pytest.fixture(scope="session")
+def zenodo_zarr(testdata_path: Path) -> list[str]:
+    """
+    This takes care of multiple steps:
+
+    1. Download/unzip two Zarr containers (3D and MIP) from Zenodo, via pooch
+    2. Copy the two Zarr containers into tests/data
+    3. Modify the Zarrs in tests/data, to add whatever is not in Zenodo
+    """
+
+    # 1 Download Zarrs from Zenodo
+    DOI = "10.5281/zenodo.10424292"
+    DOI_slug = DOI.replace("/", "_").replace(".", "_")
+    rootfolder = testdata_path / DOI_slug
+
+    registry = {
+        "20200812-CardiomyocyteDifferentiation14-Cycle1.zarr.zip": None,
+        "20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr.zip": None,
+    }
+    folders = [rootfolder / plate[:-4] for plate in registry]
+
+    base_url = f"doi:{DOI}"
+    POOCH = pooch.create(
+        pooch.os_cache("pooch") / DOI_slug,
+        base_url,
+        registry=registry,
+        retry_if_failed=10,
+        allow_updates=False,
+    )
+
+    for ind, file_name in enumerate(
+        [
+            "20200812-CardiomyocyteDifferentiation14-Cycle1.zarr",
+            "20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr",
+        ]
+    ):
+        # 1) Download/unzip a single Zarr from Zenodo
+        file_paths = POOCH.fetch(
+            f"{file_name}.zip", processor=pooch.Unzip(extract_dir=file_name)
+        )
+        zarr_full_path = file_paths[0].split(file_name)[0] + file_name
+        # print(zarr_full_path)
+        folder = folders[ind]
+
+        # 2) Copy the downloaded Zarr into tests/data
+        if os.path.isdir(str(folder)):
+            shutil.rmtree(str(folder))
+        shutil.copytree(Path(zarr_full_path) / file_name, folder)
+    return [str(f) for f in folders]

src/napari_ome_zarr_navigator/_tests/test_sample_data.py → src/napari_ome_zarr_navigator/_tests/no_run_test_sample_data.py

@@ -1,16 +1,14 @@
 from pathlib import Path
 
-from napari_ome_zarr_navigator.generate_test_data import (
+from napari_ome_zarr_navigator import _TEST_DATA_DIR
+from napari_ome_zarr_navigator._sample_data import (
     load_ome_zarr_from_zenodo,
 )
 
 
-from napari_ome_zarr_navigator import _TEST_DATA_DIR
-
-
 def test_load_zenodo_data():
     doi = "10.5281/zenodo.10424292"
     zarr_url = "20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr"
     doi_path = doi_path = Path(_TEST_DATA_DIR).joinpath(doi.replace("/", "_"))
     load_ome_zarr_from_zenodo(doi, zarr_url)
-    assert doi_path.is_dir() == True
+    assert doi_path.is_dir()