Merge pull request #77 from BlueBrain/local-to-nexus

New function to facilitate local to nexus upload

bluepyemodel/efeatures_extraction/targets_configuration.py

@@ -296,7 +296,9 @@ def get_related_nexus_ids(self):
         uses = []
         for f in self.files:
             if f.id:
-                uses.append({"id": f.id, "type": "Trace"})
+                f_dict = {"id": f.id, "type": "Trace"}
+                if f_dict not in uses:
+                    uses.append(f_dict)
 
         return {"uses": uses}
 

bluepyemodel/emodel_pipeline/emodel.py

@@ -107,6 +107,10 @@ def __init__(
         self.responses = {}
         self.evaluator = None
 
+    def copy_pdf_dependencies_to_new_path(self, seed):
+        """Copy pdf dependencies to new path using allen notation"""
+        search_pdfs.copy_emodel_pdf_dependencies_to_new_path(self.emodel_metadata, seed)
+
     def build_pdf_dependencies(self, seed):
         """Find all the pdfs associated to an emodel"""
 

bluepyemodel/evaluation/protocols.py

@@ -190,8 +190,9 @@ def _run(
 class ThresholdBasedProtocol(ProtocolWithDependencies):
 
     """Protocol having rheobase-rescaling capabilities. When using ThresholdBasedProtocol,
-    the current amplitude and step amplitude of the stimulus will be ignored and replaced by
-    values obtained from the holding current and rheobase of the cell model respectively."""
+    the holding current amplitude and step amplitude of the stimulus will be ignored and
+    replaced by values obtained from the holding current and rheobase of the cell model
+    respectively."""
 
     def __init__(
         self, name=None, stimulus=None, recordings=None, cvode_active=None, stochasticity=False

bluepyemodel/export_emodel/utils.py

@@ -18,6 +18,7 @@
 
 import logging
 import pathlib
+import shutil
 
 logger = logging.getLogger(__name__)
 
@@ -37,7 +38,11 @@ def get_output_path_from_metadata(output_base_dir, emodel_metadata, seed, use_al
 
 
 def get_output_path(
-    emodel, output_dir=None, output_base_dir="export_emodels_hoc", use_allen_notation=True
+    emodel,
+    output_dir=None,
+    output_base_dir="export_emodels_hoc",
+    use_allen_notation=True,
+    create_dir=True,
 ):
     """Get the output path.
 
@@ -46,13 +51,16 @@ def get_output_path(
         output_dir (str): output directory
         output_base_dir (str): if output_dir is None, export to this directory instead,
             using also emodel metadata in the path
+        use_allen_notation (bool): whether to replace brain region by its Allen notation
+        create_dir (bool): whether to create the output folder if not existent
     """
     if output_dir is None:
         output_dir = get_output_path_from_metadata(
             output_base_dir, emodel.emodel_metadata, emodel.seed, use_allen_notation
         )
     output_path = pathlib.Path(output_dir)
-    output_path.mkdir(parents=True, exist_ok=True)
+    if create_dir:
+        output_path.mkdir(parents=True, exist_ok=True)
 
     return output_path
 
@@ -63,6 +71,30 @@ def get_hoc_file_path(output_path):
     return str(output_path / "model.hoc")
 
 
+def copy_hocs_to_new_output_path(emodel, output_base_dir):
+    """Copy the hocs from the local output path to the new nexus output path."""
+    old_output_path = get_output_path(
+        emodel,
+        output_dir=None,
+        output_base_dir=output_base_dir,
+        use_allen_notation=False,
+        create_dir=False,
+    )
+    output_path_allen = get_output_path(
+        emodel,
+        output_dir=None,
+        output_base_dir=output_base_dir,
+        use_allen_notation=True,
+        create_dir=False,
+    )
+
+    if (
+        not pathlib.Path(get_hoc_file_path(output_path_allen)).is_file()
+        and pathlib.Path(get_hoc_file_path(old_output_path)).is_file()
+    ):
+        shutil.copytree(old_output_path, output_path_allen)
+
+
 def select_emodels(
     emodel_name, emodels, only_validated=False, only_best=True, seeds=None, iteration=None
 ):

bluepyemodel/tools/search_pdfs.py

@@ -19,6 +19,7 @@
 
 import glob
 import logging
+import shutil
 from pathlib import Path
 
 logger = logging.getLogger(__name__)
@@ -39,57 +40,150 @@ def search_figure_path(pathname):
     return str(Path(matches[0]).resolve())
 
 
+def figure_efeatures(emodel, protocol_name, efeature):
+    """Get path to pdf representing the efeature extracted from ephys recordings"""
+
+    path = f"./figures/{emodel}/efeatures_extraction/*{protocol_name}_{efeature}"
+
+    pdf_amp = f"{path}_amp.pdf"
+    pdf_amp_rel = f"{path}_amp_rel.pdf"
+
+    return pdf_amp, pdf_amp_rel
+
+
 def search_figure_efeatures(emodel, protocol_name, efeature):
     """Search for the pdf representing the efeature extracted from ephys recordings"""
 
-    path = f"./figures/{emodel}/efeatures_extraction/*{protocol_name}_{efeature}"
+    pdf_amp, pdf_amp_rel = figure_efeatures(emodel, protocol_name, efeature)
 
-    pdf_amp = search_figure_path(path + "_amp.pdf")
-    pdf_amp_rel = search_figure_path(path + "_amp_rel.pdf")
+    pdf_amp = search_figure_path(pdf_amp)
+    pdf_amp_rel = search_figure_path(pdf_amp_rel)
 
     return pdf_amp, pdf_amp_rel
 
 
-def search_figure_emodel_optimisation(emodel_metadata, seed):
-    """Search for the pdf representing the convergence of the optimisation"""
+def figure_emodel_optimisation(emodel_metadata, seed, use_allen_notation=True):
+    """Get path for the pdf representing the convergence of the optimisation"""
 
-    fname = emodel_metadata.as_string(seed) + ".pdf"
+    fname = f"{emodel_metadata.as_string(seed, use_allen_notation=use_allen_notation)}.pdf"
 
-    pathname = Path("./figures") / emodel_metadata.emodel / fname
+    return Path("./figures") / emodel_metadata.emodel / fname
+
+
+def search_figure_emodel_optimisation(emodel_metadata, seed, use_allen_notation=True):
+    """Search for the pdf representing the convergence of the optimisation"""
+
+    pathname = figure_emodel_optimisation(
+        emodel_metadata, seed, use_allen_notation=use_allen_notation
+    )
 
     return search_figure_path(str(pathname))
 
 
-def search_figure_emodel_traces(emodel_metadata, seed):
-    """Search for the pdf representing the traces of an emodel"""
+def figure_emodel_traces(emodel_metadata, seed, use_allen_notation=True):
+    """Get path for the pdf representing the traces of an emodel"""
 
-    fname = emodel_metadata.as_string(seed) + "__traces.pdf"
+    metadata_str = emodel_metadata.as_string(seed, use_allen_notation=use_allen_notation)
+    fname = f"{metadata_str}__traces.pdf"
 
     pathname = Path("./figures") / emodel_metadata.emodel / "traces" / "all" / fname
     pathname_val = Path("./figures") / emodel_metadata.emodel / "traces" / "validated" / fname
 
+    return pathname, pathname_val
+
+
+def search_figure_emodel_traces(emodel_metadata, seed, use_allen_notation=True):
+    """Search for the pdf representing the traces of an emodel"""
+
+    pathname, pathname_val = figure_emodel_traces(
+        emodel_metadata, seed, use_allen_notation=use_allen_notation
+    )
+
     return [search_figure_path(str(pathname)), search_figure_path(str(pathname_val))]
 
 
-def search_figure_emodel_score(emodel_metadata, seed):
-    """Search for the pdf representing the scores of an emodel"""
+def figure_emodel_score(emodel_metadata, seed, use_allen_notation=True):
+    """Get path for the pdf representing the scores of an emodel"""
 
-    fname = emodel_metadata.as_string(seed) + "__scores.pdf"
+    metadata_str = emodel_metadata.as_string(seed, use_allen_notation=use_allen_notation)
+    fname = f"{metadata_str}__scores.pdf"
 
     pathname = Path("./figures") / emodel_metadata.emodel / "scores" / "all" / fname
     pathname_val = Path("./figures") / emodel_metadata.emodel / "scores" / "validated" / fname
 
+    return pathname, pathname_val
+
+
+def search_figure_emodel_score(emodel_metadata, seed, use_allen_notation=True):
+    """Search for the pdf representing the scores of an emodel"""
+
+    pathname, pathname_val = figure_emodel_score(
+        emodel_metadata, seed, use_allen_notation=use_allen_notation
+    )
+
     return [search_figure_path(str(pathname)), search_figure_path(str(pathname_val))]
 
 
-def search_figure_emodel_parameters(emodel_metadata):
-    """Search for the pdf representing the distribution of the parameters
-    of an emodel"""
+def figure_emodel_parameters(emodel_metadata, use_allen_notation=True):
+    """Get path for the pdf representing the distribution of the parameters of an emodel"""
 
-    fname = emodel_metadata.as_string() + "__parameters_distribution.pdf"
+    metadata_str = emodel_metadata.as_string(use_allen_notation=use_allen_notation)
+    fname = f"{metadata_str}__parameters_distribution.pdf"
 
     pathname = Path("./figures") / emodel_metadata.emodel / "distributions" / "all" / fname
     pathname_val = Path("./figures") / emodel_metadata.emodel / "distributions" / "validated"
     pathname_val = pathname_val / fname
 
+    return pathname, pathname_val
+
+
+def search_figure_emodel_parameters(emodel_metadata, use_allen_notation=True):
+    """Search for the pdf representing the distribution of the parameters
+    of an emodel"""
+
+    pathname, pathname_val = figure_emodel_parameters(
+        emodel_metadata, use_allen_notation=use_allen_notation
+    )
+
     return [search_figure_path(str(pathname)), search_figure_path(str(pathname_val))]
+
+
+def copy_emodel_pdf_dependency_to_new_path(old_path, new_path):
+    """Copy a pdf dependency to new path using allen notation"""
+    if old_path.is_file() and not new_path.is_file():
+        new_path.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy(old_path, new_path)
+
+
+def copy_emodel_pdf_dependencies_to_new_path(emodel_metadata, seed):
+    """Copy dependencies to new path using allen notation"""
+    old_opt_path = figure_emodel_optimisation(emodel_metadata, seed, use_allen_notation=False)
+    new_opt_path = figure_emodel_optimisation(emodel_metadata, seed, use_allen_notation=True)
+    copy_emodel_pdf_dependency_to_new_path(old_opt_path, new_opt_path)
+
+    old_traces_path, old_traces_path_val = figure_emodel_traces(
+        emodel_metadata, seed, use_allen_notation=False
+    )
+    new_traces_path, new_traces_path_val = figure_emodel_traces(
+        emodel_metadata, seed, use_allen_notation=True
+    )
+    copy_emodel_pdf_dependency_to_new_path(old_traces_path, new_traces_path)
+    copy_emodel_pdf_dependency_to_new_path(old_traces_path_val, new_traces_path_val)
+
+    old_score_path, old_score_path_val = figure_emodel_score(
+        emodel_metadata, seed, use_allen_notation=False
+    )
+    new_score_path, new_score_path_val = figure_emodel_score(
+        emodel_metadata, seed, use_allen_notation=True
+    )
+    copy_emodel_pdf_dependency_to_new_path(old_score_path, new_score_path)
+    copy_emodel_pdf_dependency_to_new_path(old_score_path_val, new_score_path_val)
+
+    old_params_path, old_params_path_val = figure_emodel_parameters(
+        emodel_metadata, use_allen_notation=False
+    )
+    new_params_path, new_params_path_val = figure_emodel_parameters(
+        emodel_metadata, use_allen_notation=True
+    )
+    copy_emodel_pdf_dependency_to_new_path(old_params_path, new_params_path)
+    copy_emodel_pdf_dependency_to_new_path(old_params_path_val, new_params_path_val)