From 35747386181c86138bd936584169c1920d6fc8b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaquier=20Aur=C3=A9lien=20Tristan?= Date: Thu, 7 Dec 2023 10:42:01 +0100 Subject: [PATCH 1/6] Add a new test for sAHP extraction --- .gitignore | 2 + bluepyefe/protocol.py | 2 +- setup.py | 2 +- tests/__init__.py | 0 tests/test_extractor.py | 119 ++++++++++++++++++++++++++++++++++++++++ tests/utils.py | 32 +++++++++++ 6 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/utils.py diff --git a/.gitignore b/.gitignore index 45da335..4db5390 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,5 @@ test_run MouseCells/ .ipynb_checkpoints/ coverage.xml +MouseCells_sAHP/ +tests/exp_data/X/ \ No newline at end of file diff --git a/bluepyefe/protocol.py b/bluepyefe/protocol.py index 7bb267e..d1b2176 100644 --- a/bluepyefe/protocol.py +++ b/bluepyefe/protocol.py @@ -193,7 +193,7 @@ def reduce_ecode(self, ecode, operator): amp_rel = operator([c["amp_rel"] for c in params]) mean_param = float(amp_rel) * self.global_rheobase / 100. elif key == "amp2" and self.global_rheobase: - amp_rel2 = operator([c["amp_rel2"] for c in params]) + amp_rel2 = operator([c["amp2_rel"] for c in params]) mean_param = float(amp_rel2) * self.global_rheobase / 100. else: mean_param = operator([numpy.nan if c[key] is None else c[key] for c in params]) diff --git a/setup.py b/setup.py index d33e039..c8fbe74 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ "the eFel library. BluePyEfe outputs protocols and features files in a " "format that can then be used by BluePyOpt for electrical model building " "purposes.", - packages=setuptools.find_packages(), + packages=setuptools.find_packages(exclude=["tests"]), python_requires=">=3.8", include_package_data=True, author="BlueBrain Project, EPFL", diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_extractor.py b/tests/test_extractor.py index 4390594..024f787 100644 --- a/tests/test_extractor.py +++ b/tests/test_extractor.py @@ -6,6 +6,7 @@ import bluepyefe.extract import bluepyefe.tools +from tests.utils import download_sahp_datafiles def get_config(absolute_amplitude=False): @@ -74,6 +75,74 @@ def get_config(absolute_amplitude=False): return files_metadata, bluepyefe.extract.convert_legacy_targets(targets) +def get_sahp_config(absolute_amplitude=False): + download_sahp_datafiles() + + interesting_efeatures = { + "Spikecount": {}, + "mean_frequency": {}, + "ISI_CV": {}, + "AP1_amp": {}, + "AP_width": {}, + } + + files_metadata1 = [] + for file in glob.glob("./tests/exp_data/X/X_sAHP_ch0_*.ibw"): + files_metadata1.append( + { + "i_file": file, + "v_file": file.replace("ch0", "ch1"), + "i_unit": "A", + "v_unit": "V", + "t_unit": "s", + "dt": 0.00025, + "ljp": 14, + "ton": 10, + "tmid": 260, + "tmid2": 360, + "toff": 1360, + } + ) + files_metadata2 = [] + for file in glob.glob("./tests/exp_data/X/X_IDthresh_ch0_*.ibw"): + files_metadata2.append( + { + "i_file": file, + "v_file": file.replace("ch0", "ch1"), + "i_unit": "A", + "v_unit": "V", + "t_unit": "s", + "dt": 0.00025, + "ljp": 14, + } + ) + + files_metadata = { + "MouseNeuron1": {"sAHP": files_metadata1, "IDthresh": files_metadata2}, + } + + if absolute_amplitude: + targets = { + "sAHP": { + "amplitudes": [0.315, 0.225, 0.5, 0.69, 0.41, 0.595], + "tolerances": [0.01], + "efeatures": interesting_efeatures, + "location": "soma", + } + } + + else: + targets = { + "sAHP": { + "amplitudes": [285, 200, 450, 625, 370, 540], + "tolerances": [10.0], + "efeatures": interesting_efeatures, + "location": "soma", + } + } + + return files_metadata, bluepyefe.extract.convert_legacy_targets(targets) + class ExtractorTest(unittest.TestCase): def test_extract(self): @@ -219,6 +288,56 @@ def test_extract_absolute(self): protocols = json.load(fp) self.assertEqual(len(features), len(protocols)) + + def test_extract_sahp(self): + + files_metadata, targets = get_sahp_config() + + cells = bluepyefe.extract.read_recordings(files_metadata=files_metadata) + + cells = bluepyefe.extract.extract_efeatures_at_targets( + cells=cells, targets=targets + ) + + bluepyefe.extract.compute_rheobase(cells, protocols_rheobase=["IDthresh"]) + + self.assertEqual(len(cells), 1) + self.assertEqual(len(cells[0].recordings), 24) + self.assertLess(abs(cells[0].rheobase - 0.1103), 0.01) + self.assertLess(abs(cells[0].recordings[0].amp - 0.0953), 0.01) + self.assertLess(abs(cells[0].recordings[0].amp2 - 0.3153), 0.01) + self.assertLess(abs(cells[0].recordings[0].amp_rel - 86.4), 0.1) + self.assertLess(abs(cells[0].recordings[0].amp2_rel - 285.8), 0.1) + + + protocols = bluepyefe.extract.group_efeatures( + cells, + targets, + use_global_rheobase=True, + protocol_mode="mean" + ) + + _ = bluepyefe.extract.create_feature_protocol_files( + cells=cells, protocols=protocols, output_directory="MouseCells_sAHP" + ) + + for protocol in protocols: + if protocol.name == "sAHP" and protocol.amplitude == 625: + for target in protocol.feature_targets: + if target.efel_feature_name == "Spikecount": + self.assertEqual(target.mean, 6) + break + + bluepyefe.extract.plot_all_recordings_efeatures( + cells, protocols, output_dir="MouseCells_sAHP/" + ) + + with open("MouseCells_sAHP/features.json") as fp: + features = json.load(fp) + with open("MouseCells_sAHP/protocols.json") as fp: + protocols = json.load(fp) + + self.assertEqual(len(features), len(protocols)) if __name__ == "__main__": diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..e49f4dd --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,32 @@ +"""Utils""" + +import urllib.request +import shutil +from pathlib import Path + + +def download_sahp_datafiles(): + """Download data files for sAHP and IDthresh traces""" + output_dir = "./tests/exp_data/X/" + gb_url = "https://raw.githubusercontent.com/BlueBrain/SSCxEModelExamples/main/feature_extraction/input-traces/C060109A1-SR-C1/" + sahp_pathname = "X_sAHP" + sahp_ch = ["ch0", "ch1"] + sahp_numbers = list(range(320, 326)) + idthresh_pathname = "X_IDthresh" + idthresh_ch = ["ch0", "ch1"] + idthresh_numbers = list(range(349, 358)) + list(range(362, 371)) + # https://raw.githubusercontent.com/BlueBrain/SSCxEModelExamples/tree/main/feature_extraction/input-traces/C060109A1-SR-C1/ + # https://github.com/BlueBrain/SSCxEModelExamples/blob/c24f096495150698547741b9a34dab84e0335649/feature_extraction/input-traces/C060109A1-SR-C1/X_IDthresh_ch1_370.ibw + # https://github.com/BlueBrain/SSCxEModelExamples/raw/main/feature_extraction/input-traces/C060109A1-SR-C1/X_IDthresh_ch1_370.ibw + # https://raw.githubusercontent.com/BlueBrain/SSCxEModelExamples/main/feature_extraction/input-traces/C060109A1-SR-C1//X_IDthresh_ch1_370.ibw + + sahp_paths = [f"{sahp_pathname}_{ch}_{n}.ibw" for ch in sahp_ch for n in sahp_numbers] + idthresh_paths = [f"{idthresh_pathname}_{ch}_{n}.ibw" for ch in idthresh_ch for n in idthresh_numbers] + pathnames = sahp_paths + idthresh_paths + + Path(output_dir).mkdir(exist_ok=True, parents=True) + for pathname in pathnames: + output_path = f"{output_dir}{pathname}" + if not Path(output_path).is_file(): + with urllib.request.urlopen(f"{gb_url}{pathname}") as response, open(output_path, "wb") as out_file: + shutil.copyfileobj(response, out_file) \ No newline at end of file From 41643a8f8cf6b06998462d7d102c1d868543921c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaquier=20Aur=C3=A9lien=20Tristan?= Date: Thu, 7 Dec 2023 10:46:58 +0100 Subject: [PATCH 2/6] remove unnecessary stuff --- tests/utils.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index e49f4dd..feecb0d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -15,10 +15,6 @@ def download_sahp_datafiles(): idthresh_pathname = "X_IDthresh" idthresh_ch = ["ch0", "ch1"] idthresh_numbers = list(range(349, 358)) + list(range(362, 371)) - # https://raw.githubusercontent.com/BlueBrain/SSCxEModelExamples/tree/main/feature_extraction/input-traces/C060109A1-SR-C1/ - # https://github.com/BlueBrain/SSCxEModelExamples/blob/c24f096495150698547741b9a34dab84e0335649/feature_extraction/input-traces/C060109A1-SR-C1/X_IDthresh_ch1_370.ibw - # https://github.com/BlueBrain/SSCxEModelExamples/raw/main/feature_extraction/input-traces/C060109A1-SR-C1/X_IDthresh_ch1_370.ibw - # https://raw.githubusercontent.com/BlueBrain/SSCxEModelExamples/main/feature_extraction/input-traces/C060109A1-SR-C1//X_IDthresh_ch1_370.ibw sahp_paths = [f"{sahp_pathname}_{ch}_{n}.ibw" for ch in sahp_ch for n in sahp_numbers] idthresh_paths = [f"{idthresh_pathname}_{ch}_{n}.ibw" for ch in idthresh_ch for n in idthresh_numbers] From fb7a587d0a5567b2383d5a2511f9cbfcc6be7565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaquier=20Aur=C3=A9lien=20Tristan?= Date: Thu, 7 Dec 2023 10:49:51 +0100 Subject: [PATCH 3/6] Also test on python3.12 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 09cec9c..d7753c2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v2 From 48ba8348f0037d4425614d48aac694417db97c58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaquier=20Aur=C3=A9lien=20Tristan?= Date: Thu, 7 Dec 2023 11:14:26 +0100 Subject: [PATCH 4/6] tmp commit for debugging --- bluepyefe/ecode/sAHP.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bluepyefe/ecode/sAHP.py b/bluepyefe/ecode/sAHP.py index 9b6304b..9b6a40c 100644 --- a/bluepyefe/ecode/sAHP.py +++ b/bluepyefe/ecode/sAHP.py @@ -125,6 +125,7 @@ def compute_amp(self, current, config_data, reader_data): self.set_amplitudes_ecode("amp", config_data, reader_data, amp_value) amp2_value = numpy.median(smooth_current[self.tmid : self.tmid2]) - self.hypamp + print(f"amp2 is {amp2_value}") self.set_amplitudes_ecode("amp2", config_data, reader_data, amp2_value) if config_data.get("tend", None) is None: From 8b7c35c7c1c97659b3594661057269b161545059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaquier=20Aur=C3=A9lien=20Tristan?= Date: Thu, 7 Dec 2023 11:19:21 +0100 Subject: [PATCH 5/6] temporarily remove test for python3.12 for debugging --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d7753c2..09cec9c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v2 From d54e131548db87d11fa4816a64d387e0b304e603 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaquier=20Aur=C3=A9lien=20Tristan?= Date: Thu, 7 Dec 2023 11:50:11 +0100 Subject: [PATCH 6/6] fix recording selection in test --- bluepyefe/ecode/sAHP.py | 1 - tests/test_extractor.py | 14 ++++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/bluepyefe/ecode/sAHP.py b/bluepyefe/ecode/sAHP.py index 9b6a40c..9b6304b 100644 --- a/bluepyefe/ecode/sAHP.py +++ b/bluepyefe/ecode/sAHP.py @@ -125,7 +125,6 @@ def compute_amp(self, current, config_data, reader_data): self.set_amplitudes_ecode("amp", config_data, reader_data, amp_value) amp2_value = numpy.median(smooth_current[self.tmid : self.tmid2]) - self.hypamp - print(f"amp2 is {amp2_value}") self.set_amplitudes_ecode("amp2", config_data, reader_data, amp2_value) if config_data.get("tend", None) is None: diff --git a/tests/test_extractor.py b/tests/test_extractor.py index 024f787..d011acf 100644 --- a/tests/test_extractor.py +++ b/tests/test_extractor.py @@ -304,10 +304,16 @@ def test_extract_sahp(self): self.assertEqual(len(cells), 1) self.assertEqual(len(cells[0].recordings), 24) self.assertLess(abs(cells[0].rheobase - 0.1103), 0.01) - self.assertLess(abs(cells[0].recordings[0].amp - 0.0953), 0.01) - self.assertLess(abs(cells[0].recordings[0].amp2 - 0.3153), 0.01) - self.assertLess(abs(cells[0].recordings[0].amp_rel - 86.4), 0.1) - self.assertLess(abs(cells[0].recordings[0].amp2_rel - 285.8), 0.1) + + # amplitude test for one recording + # sort the recordings because they can be in any order, + # and we want to select the same one each time we test + sahp_recs = [rec for rec in cells[0].recordings if rec.protocol_name == "sAHP"] + rec1 = sorted(sahp_recs, key=lambda x: x.amp2)[1] + self.assertLess(abs(rec1.amp - 0.0953), 0.01) + self.assertLess(abs(rec1.amp2 - 0.3153), 0.01) + self.assertLess(abs(rec1.amp_rel - 86.4), 0.1) + self.assertLess(abs(rec1.amp2_rel - 285.8), 0.1) protocols = bluepyefe.extract.group_efeatures(