From 5b8c1b945c86032813c922f57fe2e602aa24cb13 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 16 Jul 2024 07:01:29 +0100 Subject: [PATCH 1/5] use sets as faster --- spynnaker/pyNN/data/spynnaker_data_view.py | 12 ++++++++---- spynnaker/pyNN/utilities/neo_buffer_database.py | 3 +++ unittests/data/test_simulator_data.py | 17 +++++++++-------- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/spynnaker/pyNN/data/spynnaker_data_view.py b/spynnaker/pyNN/data/spynnaker_data_view.py index 9489105a0db..8f4b87d6269 100644 --- a/spynnaker/pyNN/data/spynnaker_data_view.py +++ b/spynnaker/pyNN/data/spynnaker_data_view.py @@ -70,8 +70,8 @@ def _clear(self) -> None: self._min_delay: Optional[float] = None # Using a dict to verify if later could be stored here only self._neurons_per_core_set: Set[Type[AbstractPyNNModel]] = set() - self._populations: List[Population] = [] - self._projections: List[Projection] = [] + self._populations: Set[Population] = set() + self._projections: Set[Projection] = set() self._segment_counter = 0 def _hard_reset(self) -> None: @@ -139,6 +139,8 @@ def iterate_projections(cls) -> Iterator[Projection]: The iteration will be empty if no projections added. + Note: This method is backed by a set so does not guarantee order + :rtype: iterable(Projection) """ return iter(cls.__spy_data._projections) @@ -174,7 +176,7 @@ def add_projection(cls, projection: Projection): "This method should only be called from the Projection init") if not isinstance(projection, Proj): raise TypeError("The projection must be a Projection") - cls.__spy_data._projections.append(projection) + cls.__spy_data._projections.add(projection) @classmethod def iterate_populations(cls) -> Iterator[Population]: @@ -183,6 +185,8 @@ def iterate_populations(cls) -> Iterator[Population]: The iteration will be empty if no populations added. + Note: This method is backed by a set so does not guarantee order + :rtype: iterable(~spynnaker.pyNN.models.populations.Population) """ return iter(cls.__spy_data._populations) @@ -229,7 +233,7 @@ def add_population(cls, population: Population): "This method should only be called from the Population init") first_id = cls.__spy_data._id_counter cls.__spy_data._id_counter += population.size - cls.__spy_data._populations.append(population) + cls.__spy_data._populations.add(population) return first_id, cls.__spy_data._id_counter-1 @classmethod diff --git a/spynnaker/pyNN/utilities/neo_buffer_database.py b/spynnaker/pyNN/utilities/neo_buffer_database.py index 78f7322f081..1fc8cd4bb42 100644 --- a/spynnaker/pyNN/utilities/neo_buffer_database.py +++ b/spynnaker/pyNN/utilities/neo_buffer_database.py @@ -1459,6 +1459,9 @@ def write_metadata(self) -> None: """ Write the current metadata to the database. + The underlying call does not guarantee order + so there order the metadata is added is not consistent, + .. note:: The database must be writable for this to work! """ diff --git a/unittests/data/test_simulator_data.py b/unittests/data/test_simulator_data.py index 374072655dc..72354c087f9 100644 --- a/unittests/data/test_simulator_data.py +++ b/unittests/data/test_simulator_data.py @@ -101,8 +101,9 @@ def test_populations_and_projections(self): self.assertEqual(5, writer._SpynnakerDataWriter__spy_data._id_counter) pop_2 = Population(size=15, cellclass=model) - self.assertListEqual( - [pop_1, pop_2], list(SpynnakerDataView.iterate_populations())) + as_list = list(SpynnakerDataView.iterate_populations()) + self.assertListEqual([pop_1, pop_2], sorted( + SpynnakerDataView.iterate_populations(), key=lambda x: x.label)) self.assertEqual(2, SpynnakerDataView.get_n_populations()) # Hack to check internal data # DO NOT COPY as unsupported @@ -114,8 +115,8 @@ def test_populations_and_projections(self): self.assertEqual(1, SpynnakerDataView.get_n_projections()) pro_2 = Projection( pop_2, pop_1, OneToOneConnector(), receptor_type='excitatory') - self.assertListEqual( - [pro_1, pro_2], list(SpynnakerDataView.iterate_projections())) + self.assertListEqual([pro_1, pro_2], sorted( + SpynnakerDataView.iterate_projections(), key=lambda x: x.label)) self.assertEqual(2, SpynnakerDataView.get_n_projections()) writer.start_run() # Unable to add while running @@ -127,11 +128,11 @@ def test_populations_and_projections(self): writer.finish_run() writer.hard_reset() # population not changed by hard reset - self.assertListEqual( - [pop_1, pop_2], list(SpynnakerDataView.iterate_populations())) + self.assertListEqual([pop_1, pop_2], sorted( + SpynnakerDataView.iterate_populations(), key=lambda x: x.label)) self.assertEqual(2, SpynnakerDataView.get_n_populations()) - self.assertListEqual( - [pro_1, pro_2], list(SpynnakerDataView.iterate_projections())) + self.assertListEqual([pro_1, pro_2], sorted( + SpynnakerDataView.iterate_projections(), key=lambda x: x.label)) self.assertEqual(2, SpynnakerDataView.get_n_projections()) self.assertEqual(20, writer._SpynnakerDataWriter__spy_data._id_counter) with self.assertRaises(TypeError): From dce20a380e6137b3f869bbdc3464f0f96dd7982d Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 16 Jul 2024 08:07:06 +0100 Subject: [PATCH 2/5] guarantee order --- spynnaker/pyNN/utilities/neo_buffer_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spynnaker/pyNN/utilities/neo_buffer_database.py b/spynnaker/pyNN/utilities/neo_buffer_database.py index 1fc8cd4bb42..67329b70e98 100644 --- a/spynnaker/pyNN/utilities/neo_buffer_database.py +++ b/spynnaker/pyNN/utilities/neo_buffer_database.py @@ -515,7 +515,7 @@ def __get_region_metadata(self, rec_id: int) -> Iterable[Tuple[ SELECT region_id, recording_neurons_st, vertex_slice, base_key FROM region_metadata WHERE rec_id = ? - ORDER BY region_metadata_id + ORDER BY region_id, recording_neurons_st, vertex_slice, base_key """, (rec_id,))): vertex_slice = MDSlice.from_string( self._string(row["vertex_slice"])) From 92e20f62af43c205b792f75d0908f73e49460dca Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 16 Jul 2024 08:08:46 +0100 Subject: [PATCH 3/5] flake8 --- spynnaker/pyNN/data/spynnaker_data_view.py | 2 +- unittests/data/test_simulator_data.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/spynnaker/pyNN/data/spynnaker_data_view.py b/spynnaker/pyNN/data/spynnaker_data_view.py index 8f4b87d6269..5576a60ce53 100644 --- a/spynnaker/pyNN/data/spynnaker_data_view.py +++ b/spynnaker/pyNN/data/spynnaker_data_view.py @@ -14,7 +14,7 @@ from __future__ import annotations import logging from typing import ( - Iterator, List, Optional, Set, Tuple, Type, Union, TYPE_CHECKING) + Iterator, Optional, Set, Tuple, Type, Union, TYPE_CHECKING) from spinn_utilities.log import FormatAdapter from spinn_front_end_common.data import FecDataView from spynnaker import _version diff --git a/unittests/data/test_simulator_data.py b/unittests/data/test_simulator_data.py index 72354c087f9..aec2e5dd02f 100644 --- a/unittests/data/test_simulator_data.py +++ b/unittests/data/test_simulator_data.py @@ -101,7 +101,6 @@ def test_populations_and_projections(self): self.assertEqual(5, writer._SpynnakerDataWriter__spy_data._id_counter) pop_2 = Population(size=15, cellclass=model) - as_list = list(SpynnakerDataView.iterate_populations()) self.assertListEqual([pop_1, pop_2], sorted( SpynnakerDataView.iterate_populations(), key=lambda x: x.label)) self.assertEqual(2, SpynnakerDataView.get_n_populations()) From e46ae6ad518843b73519c19c94f062d6650465a3 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 16 Jul 2024 08:30:31 +0100 Subject: [PATCH 4/5] flake8 --- spynnaker/pyNN/utilities/neo_buffer_database.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spynnaker/pyNN/utilities/neo_buffer_database.py b/spynnaker/pyNN/utilities/neo_buffer_database.py index 67329b70e98..be0779aa462 100644 --- a/spynnaker/pyNN/utilities/neo_buffer_database.py +++ b/spynnaker/pyNN/utilities/neo_buffer_database.py @@ -515,7 +515,8 @@ def __get_region_metadata(self, rec_id: int) -> Iterable[Tuple[ SELECT region_id, recording_neurons_st, vertex_slice, base_key FROM region_metadata WHERE rec_id = ? - ORDER BY region_id, recording_neurons_st, vertex_slice, base_key + ORDER BY region_id, recording_neurons_st, vertex_slice, + base_key """, (rec_id,))): vertex_slice = MDSlice.from_string( self._string(row["vertex_slice"])) From 739a18e91b1131933b73ecd479e509a7642b405c Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 16 Jul 2024 08:37:59 +0100 Subject: [PATCH 5/5] flake8 --- spynnaker/pyNN/utilities/neo_buffer_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spynnaker/pyNN/utilities/neo_buffer_database.py b/spynnaker/pyNN/utilities/neo_buffer_database.py index be0779aa462..7e0cf28f829 100644 --- a/spynnaker/pyNN/utilities/neo_buffer_database.py +++ b/spynnaker/pyNN/utilities/neo_buffer_database.py @@ -515,7 +515,7 @@ def __get_region_metadata(self, rec_id: int) -> Iterable[Tuple[ SELECT region_id, recording_neurons_st, vertex_slice, base_key FROM region_metadata WHERE rec_id = ? - ORDER BY region_id, recording_neurons_st, vertex_slice, + ORDER BY region_id, recording_neurons_st, vertex_slice, base_key """, (rec_id,))): vertex_slice = MDSlice.from_string(