Standardising atlas file downloading

brainglobe_atlasapi/atlas_generation/atlas_scripts/admba_3d_dev_mouse.py

@@ -1,4 +1,5 @@
 __version__ = "1"
+__atlas__ = "admba_3d_dev_mouse"
 
 import dataclasses
 import json
@@ -20,12 +21,15 @@
     create_region_mesh,
 )
 from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data
+from brainglobe_atlasapi.config import DEFAULT_WORKDIR
 from brainglobe_atlasapi.structure_tree_util import get_structures_tree
 
 PARALLEL = False
 
 
-def download_atlas_files(download_dir_path, atlas_file_url, ATLAS_NAME):
+def download_atlas_files(
+    download_dir_path, atlas_file_url, ATLAS_NAME, known_hash
+):
     utils.check_internet_connection()
 
     try:
@@ -36,7 +40,7 @@
 
     pooch.retrieve(
         url=atlas_file_url,
-        known_hash=None,
+        known_hash=known_hash,
         path=destination_path,
         progressbar=True,
         processor=pooch.Unzip(extract_dir="."),
@@ -213,6 +217,8 @@
     citation: str
     root_id: int
     atlas_packager: str
+    # pooch hash for remote atlas file:
+    known_hash: str
 
 
 def create_atlas(
@@ -230,7 +236,7 @@
     if isinstance(working_dir, str):
         working_dir = Path(working_dir)
     # Generated atlas path:
-    working_dir = working_dir / "admba_3d_dev_mouse" / atlas_config.atlas_name
+    working_dir = working_dir / atlas_config.atlas_name
     working_dir.mkdir(exist_ok=True, parents=True)
 
     download_dir_path = working_dir / "downloads"
@@ -245,6 +251,7 @@
             download_dir_path,
             atlas_config.atlas_file_url,
             atlas_config.atlas_name,
+            atlas_config.known_hash,
         )
         ## Load files
 
@@ -310,7 +317,7 @@
 
 if __name__ == "__main__":
     # Generated atlas path:
-    bg_root_dir = Path.home() / ".brainglobe"
+    bg_root_dir = DEFAULT_WORKDIR / __atlas__
     bg_root_dir.mkdir(exist_ok=True, parents=True)
 
     # set up E11.5 atlas settings and use as template for rest of brains
@@ -331,6 +338,7 @@
         atlas_packager="Pradeep Rajasekhar, WEHI, Australia, "
         "rajasekhardotp@wehidotedudotau; David Young, UCSF, "
         "United States, davedotyoung@ucsfdotedu",
+        known_hash="30e978c0f72939c9967442e3bc18adbb1d55eba902df4cfa76d4317df5059b08",
     )
 
     # E13.5 atlas, with updated name and URLs
@@ -339,6 +347,7 @@
         atlas_name="admba_3d_e13_5_mouse",
         atlas_link="https://search.kg.ebrains.eu/instances/bdb89f61-8dc4-4255-b4d5-50d470958b58",
         atlas_file_url="https://data.kg.ebrains.eu/zip?container=https://object.cscs.ch/v1/AUTH_4791e0a3b3de43e2840fe46d9dc2b334/ext-d000024_3Drecon-ADMBA-E13pt5_pub",
+        known_hash="7ab6d7fc62a7cce26ed0691716ccd00067df47e1a90ed05ed6a017709455593b",
     )
 
     # E15.5 atlas
@@ -347,6 +356,7 @@
         atlas_name="admba_3d_e15_5_mouse",
         atlas_link="https://search.kg.ebrains.eu/instances/Dataset/51a81ae5-d821-437a-a6d5-9b1f963cfe9b",
         atlas_file_url="https://data.kg.ebrains.eu/zip?container=https://object.cscs.ch/v1/AUTH_4791e0a3b3de43e2840fe46d9dc2b334/ext-d000025_3Drecon-ADMBA-E15pt5_pub",
+        known_hash="b85be368d2460c7193d7ecfbae91acefd88e4022d1d6d4c650d5b3082c56d43b",
     )
 
     # E18.5 atlas
@@ -355,6 +365,7 @@
         atlas_name="admba_3d_e18_5_mouse",
         atlas_link="https://search.kg.ebrains.eu/instances/633b41be-867a-4611-8570-82271aebd516",
         atlas_file_url="https://data.kg.ebrains.eu/zip?container=https://object.cscs.ch/v1/AUTH_4791e0a3b3de43e2840fe46d9dc2b334/ext-d000026_3Drecon-ADMBA-E18pt5_pub",
+        known_hash="18b57e4db2c2d87d5d19295b4167de0c67a24e63061d01098724615beed0192a",
     )
 
     # P4 atlas, which has different resolutions
@@ -364,6 +375,7 @@
         atlas_link="https://search.kg.ebrains.eu/instances/eea3589f-d74b-4988-8f4c-fd9ae8e3a4b3",
         atlas_file_url="https://data.kg.ebrains.eu/zip?container=https://object.cscs.ch/v1/AUTH_4791e0a3b3de43e2840fe46d9dc2b334/ext-d000027_3Drecon-ADMBA-P4_pub",
         resolution=(16.752, 16.752, 20),
+        known_hash="87dd3411737b5fdccffabfddb0ae900d358427156d397dc04668e3442f59b0f2",
     )
 
     # P14 atlas, which has slightly different resolutions
@@ -373,6 +385,7 @@
         atlas_link="https://search.kg.ebrains.eu/instances/114e50aa-156c-4283-af73-11b7f03d287e",
         atlas_file_url="https://data.kg.ebrains.eu/zip?container=https://object.cscs.ch/v1/AUTH_4791e0a3b3de43e2840fe46d9dc2b334/ext-d000028_3Drecon-ADMBA-P14_pub",
         resolution=(16.752, 16.752, 25),
+        known_hash="7c81bdb68493d7a31f865a06c08f430229e04dc5f1c0b85d4434fee8c2d0ebac",
     )
 
     # P28 atlas, which has same resolutions as P14
@@ -382,6 +395,7 @@
         atlas_name="admba_3d_p28_mouse",
         atlas_link="https://search.kg.ebrains.eu/instances/3a1153f0-6779-43bd-9f02-f92700a585a4",
         atlas_file_url="https://data.kg.ebrains.eu/zip?container=https://object.cscs.ch/v1/AUTH_4791e0a3b3de43e2840fe46d9dc2b334/ext-d000029_3Drecon-ADMBA-P28_pub",
+        known_hash="be108596c14957f4b25019f4d23477f9e1abbb3d3e014c112ac42eec7b3686fd",
     )
 
     # P56 atlas, which has different resolutions
@@ -391,6 +405,7 @@
         atlas_link="https://search.kg.ebrains.eu/instances/a7e99105-1ec2-42e2-a53a-7aa0f2b78135",
         atlas_file_url="https://data.kg.ebrains.eu/zip?container=https://object.cscs.ch/v1/AUTH_4791e0a3b3de43e2840fe46d9dc2b334/ext-d000030_3Drecon-ADMBA-P56_pub",
         resolution=(25, 25, 25),
+        known_hash="0bad5f8e4dfa256d5f634834f1f07b58356b16a9165aebe3ff39f3133901986d",
     )
 
     # atlases to create

brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_cord.py

@@ -3,50 +3,53 @@
 import json
 import multiprocessing as mp
 import time
-import zipfile
 from pathlib import Path
 from random import choices
 
 import numpy as np
 import pandas as pd
+import pooch
 import tifffile
 from loguru import logger
 from rich.progress import track
 
-# import sys
-# sys.path.append("./")
 from brainglobe_atlasapi import utils
 from brainglobe_atlasapi.atlas_generation.mesh_utils import (
     Region,
     create_region_mesh,
     inspect_meshes_folder,
 )
 from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data
+from brainglobe_atlasapi.config import DEFAULT_WORKDIR
 from brainglobe_atlasapi.structure_tree_util import get_structures_tree
 
 PARALLEL = True
 TEST = False
 
+ATLAS_NAME = "allen_cord"
+SPECIES = "Mus musculus"
+ATLAS_LINK = "https://data.mendeley.com/datasets/4rrggzv5d5/1"
+CITATION = "Fiederling et al. 2021, https://doi.org/10.1101/2021.05.06.443008"
+ORIENTATION = "asr"
+RESOLUTION = (20, 10, 10)
+ROOT_ID = 250
+ATLAS_FILE_URL = "https://prod-dcd-datasets-cache-zipfiles.s3.eu-west-1.amazonaws.com/4rrggzv5d5-1.zip"
+ATLAS_PACKAGER = "MetaCell LLC, Ltd."
+
 
 def download_atlas_files(download_dir_path: Path, atlas_file_url: str) -> Path:
     utils.check_internet_connection()
 
-    atlas_files_dir = download_dir_path / "atlas_files"
-
-    # only download data if they weren't already downloaded
-    if atlas_files_dir.exists():
-        print("Not downloading atlas since it was downloaded already already")
-        return atlas_files_dir / "SC_P56_Atlas_10x10x20_v5_2020"
-    else:
-        print("Downloading atlas data")
-
-    destination_path = download_dir_path / "atlas_download"
-    utils.retrieve_over_http(atlas_file_url, destination_path)
+    pooch.retrieve(
+        url=atlas_file_url,
+        known_hash="4e8d592c78d1613827fa7bc524f215dc0fe7c7e5049fb31be6d3e4b3822852f7",
+        path=download_dir_path,
+        progressbar=True,
+        processor=pooch.Unzip(extract_dir="."),
+    )
 
-    with zipfile.ZipFile(destination_path, "r") as zip_ref:
-        zip_ref.extractall(atlas_files_dir)
+    atlas_files_dir = download_dir_path / "SC_P56_Atlas_10x10x20_v5_2020"
 
-    atlas_files_dir = atlas_files_dir / "SC_P56_Atlas_10x10x20_v5_2020"
     return atlas_files_dir
 
 
@@ -212,17 +215,6 @@
 
 
 def create_atlas(working_dir):
-    ATLAS_NAME = "allen_cord"
-    SPECIES = "Mus musculus"
-    ATLAS_LINK = "https://data.mendeley.com/datasets/4rrggzv5d5/1"
-    CITATION = (
-        "Fiederling et al. 2021, https://doi.org/10.1101/2021.05.06.443008"
-    )
-    ORIENTATION = "asr"
-    RESOLUTION = (20, 10, 10)
-    ROOT_ID = 250
-    ATLAS_FILE_URL = "https://prod-dcd-datasets-cache-zipfiles.s3.eu-west-1.amazonaws.com/4rrggzv5d5-1.zip"
-    ATLAS_PACKAGER = "MetaCell LLC, Ltd."
 
     download_dir_path = working_dir / "downloads"
     download_dir_path.mkdir(exist_ok=True)
@@ -292,7 +284,7 @@
 
 if __name__ == "__main__":
     # Generated atlas path:
-    bg_root_dir = Path.home() / "brainglobe_workingdir" / "allen_cord_smooth"
+    bg_root_dir = DEFAULT_WORKDIR / ATLAS_NAME
     bg_root_dir.mkdir(exist_ok=True, parents=True)
 
     # generate atlas

brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse.py

@@ -1,7 +1,5 @@
 __version__ = "2"
 
-from pathlib import Path
-
 from allensdk.api.queries.ontologies_api import OntologiesApi
 from allensdk.api.queries.reference_space_api import ReferenceSpaceApi
 from allensdk.core.reference_space_cache import ReferenceSpaceCache
@@ -10,15 +8,17 @@
 
 from brainglobe_atlasapi import descriptors
 from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data
+from brainglobe_atlasapi.config import DEFAULT_WORKDIR
+
+ATLAS_NAME = "allen_mouse"
+SPECIES = "Mus musculus"
+ATLAS_LINK = "http://www.brain-map.org"
+CITATION = "Wang et al 2020, https://doi.org/10.1016/j.cell.2020.04.007"
+ORIENTATION = "asr"
+RES_UM = 25
 
 
 def create_atlas(working_dir, resolution):
-    # Specify information about the atlas:
-    ATLAS_NAME = "allen_mouse"
-    SPECIES = "Mus musculus"
-    ATLAS_LINK = "http://www.brain-map.org"
-    CITATION = "Wang et al 2020, https://doi.org/10.1016/j.cell.2020.04.007"
-    ORIENTATION = "asr"
 
     # Temporary folder for nrrd files download:
     download_dir_path = working_dir / "downloading_path"
@@ -107,9 +107,8 @@
 
 
 if __name__ == "__main__":
-    RES_UM = 25
     # Generated atlas path:
-    bg_root_dir = Path.home() / "brainglobe_workingdir" / "allen_mouse"
+    bg_root_dir = DEFAULT_WORKDIR / ATLAS_NAME
     bg_root_dir.mkdir(exist_ok=True)
 
     create_atlas(bg_root_dir, RES_UM)

brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py

@@ -1,7 +1,7 @@
 __version__ = "0"
+
 import json
 import time
-from pathlib import Path
 
 import meshio as mio
 import nrrd
@@ -20,20 +20,21 @@
     create_region_mesh,
 )
 from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data
+from brainglobe_atlasapi.config import DEFAULT_WORKDIR
 from brainglobe_atlasapi.structure_tree_util import get_structures_tree
 
+ATLAS_NAME = "allen_mouse_bluebrain_barrels"
+SPECIES = "Mus musculus"
+ATLAS_LINK = "http://www.brain-map.org"
+CITATION = "Bolaños-Puchet S., Teska A., et al. (2024).  https://doi.org/10.1162/imag_a_00209"
+ATLAS_PACKAGER = "Axel Bisi"
+ORIENTATION = "asr"
+ROOT_ID = 997
+RES_UM = 25
 
-def create_atlas(working_dir, resolution):
-    # Specify information about the atlas:
-    ATLAS_NAME = "allen_mouse_bluebrain_barrels"
-    SPECIES = "Mus musculus"
-    ATLAS_LINK = "http://www.brain-map.org"
-    CITATION = "Bolaños-Puchet S., Teska A., et al. (2024).  https://doi.org/10.1162/imag_a_00209"
-    ATLAS_PACKAGER = "Axel Bisi"
-    ORIENTATION = "asr"
 
+def create_atlas(working_dir, resolution):
     # Parameters for mesh creation:
-    ROOT_ID = 997
     CLOSING_N_ITERS = 2
     DECIMATE_FRACTION = 0.3
     SMOOTH = True
@@ -365,11 +366,8 @@
 
 
 if __name__ == "__main__":
-    RES_UM = 25
     # Generated atlas path:
-    bg_root_dir = (
-        Path.home() / "brainglobe_workingdir" / "allen_mouse_bluebrain_barrels"
-    )
+    bg_root_dir = DEFAULT_WORKDIR / ATLAS_NAME
     bg_root_dir.mkdir(exist_ok=True)
 
     create_atlas(bg_root_dir, RES_UM)