diff --git a/spynnaker/pyNN/models/recorder.py b/spynnaker/pyNN/models/recorder.py index 94c0b4f9f2..65f422d685 100644 --- a/spynnaker/pyNN/models/recorder.py +++ b/spynnaker/pyNN/models/recorder.py @@ -271,18 +271,30 @@ def csv_neo_block( wrote_metadata = db.csv_block_metadata( csv_file, pop_label, annotations) if wrote_metadata: - db.csv_segment( - csv_file, pop_label, variables, view_indexes) + db.csv_segment(csv_file, pop_label, variables, + view_indexes, allow_missing=True) - with NeoBufferDatabase() as db: - if SpynnakerDataView.is_reset_last(): + if SpynnakerDataView.is_reset_last(): + if wrote_metadata: logger.warning( "Due to the call directly after reset, " "the data will only contain {} segments", SpynnakerDataView.get_segment_counter() - 1) + return else: - db.csv_segment( - csv_file, pop_label, variables, view_indexes) + raise ConfigurationException( + f"Unable to write data for {pop_label}") + + with NeoBufferDatabase() as db: + if not wrote_metadata: + wrote_metadata = db.csv_block_metadata( + csv_file, pop_label, annotations) + if wrote_metadata: + db.csv_segment(csv_file, pop_label, variables, + view_indexes, allow_missing=False) + else: + raise ConfigurationException( + f"Unable to write data for {pop_label}") def __append_current_segment( self, block: neo.Block, variables: Names, diff --git a/spynnaker/pyNN/utilities/neo_buffer_database.py b/spynnaker/pyNN/utilities/neo_buffer_database.py index 1086cad607..a21a423b43 100644 --- a/spynnaker/pyNN/utilities/neo_buffer_database.py +++ b/spynnaker/pyNN/utilities/neo_buffer_database.py @@ -1079,7 +1079,8 @@ def get_spike_counts( self.__get_segment_info() metadata = self.__get_recording_metadata(pop_label, SPIKES) if metadata is None: - return {} + raise ConfigurationException( + f"{pop_label} did not record spikes") (rec_id, _, buffered_type, _, _, pop_size, _, n_colour_bits) = metadata @@ -1157,7 +1158,7 @@ def __add_data( def __read_and_csv_data( self, pop_label: str, variable: str, csv_writer: CSVWriter, - view_indexes: ViewIndices, t_stop: float): + view_indexes: ViewIndices, t_stop: float, allow_missing: bool): """ Reads the data for one variable and adds it to the CSV file. @@ -1173,10 +1174,16 @@ def __read_and_csv_data( :param view_indexes: :type view_indexes: None, ~numpy.array or list(int) :param float t_stop: + :param allow_missing: Flag to say if data for missing variable + should raise an exception """ metadata = self.__get_recording_metadata(pop_label, variable) if metadata is None: - return + if allow_missing: + return + else: + raise ConfigurationException( + f"No data for {pop_label=} {variable=}") (rec_id, data_type, buffer_type, t_start, sampling_interval_ms, pop_size, units, n_colour_bits) = metadata @@ -1269,7 +1276,7 @@ def get_full_block( def csv_segment( self, csv_file: str, pop_label: str, variables: Names, - view_indexes: ViewIndices = None): + view_indexes: ViewIndices, allow_missing: bool): """ Writes the data including metadata to a CSV file. @@ -1291,7 +1298,7 @@ def csv_segment( If the recording metadata not setup correctly """ if not os.path.isfile(csv_file): - raise SpynnakerException("PLease call csv_block_metadata first") + raise SpynnakerException("Please call csv_block_metadata first") with open(csv_file, 'a', newline='', encoding="utf-8") as csvfile: csv_writer = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) @@ -1302,8 +1309,8 @@ def csv_segment( csv_writer, segment_number, rec_datetime) for variable in self.__clean_variables(variables, pop_label): - self.__read_and_csv_data( - pop_label, variable, csv_writer, view_indexes, t_stop) + self.__read_and_csv_data(pop_label, variable, csv_writer, + view_indexes, t_stop, allow_missing) def csv_block_metadata( self, csv_file: str, pop_label: str, diff --git a/spynnaker_integration_tests/test_various/test_recording_later_additions.py b/spynnaker_integration_tests/test_various/test_recording_later_additions.py index a52882d76f..9d0f0536a4 100644 --- a/spynnaker_integration_tests/test_various/test_recording_later_additions.py +++ b/spynnaker_integration_tests/test_various/test_recording_later_additions.py @@ -12,14 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. - +import os +from spinn_front_end_common.utilities.exceptions import ConfigurationException import pyNN.spiNNaker as sim from spinnaker_testbase import BaseTestCase class TestRecordingLaterAdditions(BaseTestCase): - def do_simple(self): + def do_later_additions(self): sim.setup(timestep=1.0) sim.set_number_of_neurons_per_core(sim.IF_curr_exp, 100) @@ -31,19 +32,58 @@ def do_simple(self): synapse_type=sim.StaticSynapse(weight=5, delay=1)) pop_a.record(["spikes", "v"]) + pop_b = sim.Population(1, sim.IF_curr_exp(), label="pop_b") + sim.Projection(input_pop, pop_b, sim.OneToOneConnector(), + synapse_type=sim.StaticSynapse(weight=5, delay=1)) + pop_c = sim.Population(1, sim.IF_curr_exp(), label="pop_c") sim.Projection(input_pop, pop_c, sim.OneToOneConnector(), synapse_type=sim.StaticSynapse(weight=5, delay=1)) pop_c.record(["spikes"]) + sim.run(10) + + # Did not record all these should fail + with self.assertRaises(ConfigurationException): + pop_b.get_data(variables=["spikes", "v"]) + with self.assertRaises(ConfigurationException): + pop_b.get_spike_counts() + with self.assertRaises(ConfigurationException): + pop_b.write_data("test_b_bad.csv", ["spikes", "v"]) + with self.assertRaises(ConfigurationException): + pop_c.get_data(variables=["spikes", "v"]) + with self.assertRaises(ConfigurationException): + pop_c.write_data("test_c_bad.csv", ["spikes", "v"]) + sim.reset() - pop_b = sim.Population(1, sim.IF_curr_exp(), label="pop_b") - sim.Projection(input_pop, pop_b, sim.OneToOneConnector(), - synapse_type=sim.StaticSynapse(weight=5, delay=1)) - pop_b.record(["spikes", "v"]) + # No recording from previous segment so should fail + with self.assertRaises(ConfigurationException): + pop_b.get_data(variables=["spikes", "v"]) + with self.assertRaises(ConfigurationException): + pop_b.write_data("test_b_bad.csv", ["spikes", "v"]) + # Only includes this segment so should fail + with self.assertRaises(ConfigurationException): + pop_b.get_spike_counts() + # Some recording from previous will wok + pop_c.get_data(variables=["spikes", "v"]) + pop_c.write_data("test_c_1.csv", ["spikes", "v"]) + pop_b.record(["spikes", "v"]) pop_c.record(["v"]) + + # No recording from previous segment so should fail + with self.assertRaises(ConfigurationException): + pop_b.get_data(variables=["spikes", "v"]) + with self.assertRaises(ConfigurationException): + pop_b.write_data("test_b_bad.csv", ["spikes", "v"]) + # Only includes this segment so should fail + with self.assertRaises(ConfigurationException): + pop_b.get_spike_counts() + # Some recording from previous will wok + pop_c.get_data(variables=["spikes", "v"]) + pop_c.write_data("test_c_1.csv", ["spikes", "v"]) + sim.run(20) neo_a = pop_a.get_data(variables=["spikes", "v"]) @@ -116,5 +156,18 @@ def do_simple(self): pop_c.write_data("test_c.csv", ["spikes", "v"]) sim.end() - def test_simple(self): - self.runsafe(self.do_simple) + def cleanup(self, file): + try: + if os.path.exists(file): + os.remove(file) + except Exception: + pass + + def test_later_additions(self): + self.cleanup("test_a.csv") + self.cleanup("test_b.csv") + self.cleanup("test_b_bad.csv") + self.cleanup("test_c.csv") + self.cleanup("test_c_1.csv") + self.cleanup("test_c_bad.csv") + self.runsafe(self.do_later_additions) diff --git a/unittests/test_pop_views_assembly/test_csv.py b/unittests/test_pop_views_assembly/test_csv.py index a8285702a1..c4d9fd080b 100644 --- a/unittests/test_pop_views_assembly/test_csv.py +++ b/unittests/test_pop_views_assembly/test_csv.py @@ -62,7 +62,8 @@ def test_write(self): with NeoBufferDatabase(my_buffer) as db: db.csv_block_metadata( my_csv, "pop_1", annotations={"foo": 12, "bar": 34}) - db.csv_segment(my_csv, "pop_1", variables="all") + db.csv_segment(my_csv, "pop_1", variables="all", + view_indexes=None, allow_missing=False) neo = NeoCsv().read_csv(my_csv) # All annotations converted to String and not back @@ -88,7 +89,7 @@ def test_view(self): db.csv_block_metadata(my_csv, "pop_1", annotations=None) db.csv_segment( my_csv, "pop_1", variables=["spikes", "v"], - view_indexes=[2, 4, 7, 8]) + view_indexes=[2, 4, 7, 8], allow_missing=False) neo = NeoCsv().read_csv(my_csv) @@ -109,7 +110,8 @@ def test_over_view(self): my_csv = os.path.join(my_dir, "test_over_view.csv") with NeoBufferDatabase(my_buffer) as db: db.csv_block_metadata(my_csv, "pop_1", annotations=None) - db.csv_segment(my_csv, "pop_1", variables="all") + db.csv_segment(my_csv, "pop_1", variables="all", + view_indexes=None, allow_missing=False) neo = NeoCsv().read_csv(my_csv) spikes = neo_convertor.convert_spikes(neo) @@ -129,8 +131,8 @@ def test_over_sub_view(self): my_csv = os.path.join(my_dir, "test_over_sub_view.csv") with NeoBufferDatabase(my_buffer) as db: db.csv_block_metadata(my_csv, "pop_1", annotations=None) - db.csv_segment( - my_csv, "pop_1", variables="all", view_indexes=[2, 4]) + db.csv_segment(my_csv, "pop_1", variables="all", + view_indexes=[2, 4], allow_missing=False) neo = NeoCsv().read_csv(my_csv) spikes = neo_convertor.convert_spikes(neo) @@ -150,8 +152,8 @@ def test_no_intersection(self): my_csv = os.path.join(my_dir, "test_no_intersection.csv") with NeoBufferDatabase(my_buffer) as db: db.csv_block_metadata(my_csv, "pop_1", annotations=None) - db.csv_segment( - my_csv, "pop_1", variables="all", view_indexes=[4, 6]) + db.csv_segment(my_csv, "pop_1", variables="all", + view_indexes=[4, 6], allow_missing=False) neo = NeoCsv().read_csv(my_csv) spikes = neo_convertor.convert_spikes(neo) @@ -170,7 +172,8 @@ def test_rewiring(self): my_csv = os.path.join(my_dir, "test_rewiring.csv") with NeoBufferDatabase(my_buffer) as db: db.csv_block_metadata(my_csv, "pop_1", annotations=None) - db.csv_segment(my_csv, "pop_1", variables="all") + db.csv_segment(my_csv, "pop_1", variables="all", + view_indexes=None, allow_missing=False) neo = NeoCsv().read_csv(my_csv) formation_events = neo.segments[0].events[0] elimination_events = neo.segments[0].events[1]