From 137b43a74067154c872c294df698ab40509197bc Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 5 May 2021 09:34:04 +0200 Subject: [PATCH 1/6] Add marker_detections.csv to surface export --- .../surface_tracker/background_tasks.py | 14 ++++++++++++++ .../surface_tracker/surface_tracker_offline.py | 1 + 2 files changed, 15 insertions(+) diff --git a/pupil_src/shared_modules/surface_tracker/background_tasks.py b/pupil_src/shared_modules/surface_tracker/background_tasks.py index 1573db71c5..70585c1427 100644 --- a/pupil_src/shared_modules/surface_tracker/background_tasks.py +++ b/pupil_src/shared_modules/surface_tracker/background_tasks.py @@ -221,6 +221,7 @@ def get_export_proxy( gaze_positions, fixations, camera_model, + marker_cache, mp_context, ): exporter = Exporter( @@ -231,6 +232,7 @@ def get_export_proxy( gaze_positions, fixations, camera_model, + marker_cache, ) proxy = background_helper.IPC_Logging_Task_Proxy( "Offline Surface Tracker Exporter", @@ -250,6 +252,7 @@ def __init__( gaze_positions, fixations, camera_model, + marker_cache, ): self.export_range = export_range self.metrics_dir = os.path.join(export_dir, "surfaces") @@ -260,6 +263,7 @@ def __init__( self.camera_model = camera_model self.gaze_on_surfaces = None self.fixations_on_surfaces = None + self.marker_cache = marker_cache def save_surface_statisics_to_file(self): logger.info("exporting metrics to {}".format(self.metrics_dir)) @@ -277,6 +281,7 @@ def save_surface_statisics_to_file(self): self.fixations_on_surfaces, ) = self._map_gaze_and_fixations() + self._export_marker_detections() self._export_surface_visibility() self._export_surface_gaze_distribution() self._export_surface_events() @@ -338,6 +343,15 @@ def _map_gaze_and_fixations(self): return gaze_on_surface, fixations_on_surface + def _export_marker_detections(self): + file_path = os.path.join(self.metrics_dir, "marker_detections.csv") + with open(file_path, "w", encoding="utf-8", newline="") as csv_file: + csv_writer = csv.writer(csv_file, delimiter=",") + csv_writer.writerow(("world_index", "marker_uid")) + for idx, markers in enumerate(self.marker_cache): + for m in markers: + csv_writer.writerow((idx, m.uid)) + def _export_surface_visibility(self): with open( os.path.join(self.metrics_dir, "surface_visibility.csv"), diff --git a/pupil_src/shared_modules/surface_tracker/surface_tracker_offline.py b/pupil_src/shared_modules/surface_tracker/surface_tracker_offline.py index 86cdbc4550..6d66a2a52d 100644 --- a/pupil_src/shared_modules/surface_tracker/surface_tracker_offline.py +++ b/pupil_src/shared_modules/surface_tracker/surface_tracker_offline.py @@ -567,6 +567,7 @@ def on_notify(self, notification): self.g_pool.gaze_positions, self.g_pool.fixations, self.camera_model, + self.marker_cache, mp_context, ) self.export_proxies.add(proxy) From 4325b6deb096265f3aea3fce65f22eae7714ef91 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Mon, 10 May 2021 21:48:45 +0200 Subject: [PATCH 2/6] Avoid multiprocessing serialization by saving marker cache to temp file before export --- .../surface_tracker/background_tasks.py | 33 ++++++++++++------- .../surface_tracker_offline.py | 13 +++++++- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/pupil_src/shared_modules/surface_tracker/background_tasks.py b/pupil_src/shared_modules/surface_tracker/background_tasks.py index 70585c1427..0a36f6105f 100644 --- a/pupil_src/shared_modules/surface_tracker/background_tasks.py +++ b/pupil_src/shared_modules/surface_tracker/background_tasks.py @@ -19,6 +19,7 @@ import background_helper import player_methods +import file_methods logger = logging.getLogger(__name__) @@ -221,7 +222,7 @@ def get_export_proxy( gaze_positions, fixations, camera_model, - marker_cache, + marker_cache_path, mp_context, ): exporter = Exporter( @@ -232,7 +233,7 @@ def get_export_proxy( gaze_positions, fixations, camera_model, - marker_cache, + marker_cache_path, ) proxy = background_helper.IPC_Logging_Task_Proxy( "Offline Surface Tracker Exporter", @@ -252,7 +253,7 @@ def __init__( gaze_positions, fixations, camera_model, - marker_cache, + marker_cache_path, ): self.export_range = export_range self.metrics_dir = os.path.join(export_dir, "surfaces") @@ -263,7 +264,7 @@ def __init__( self.camera_model = camera_model self.gaze_on_surfaces = None self.fixations_on_surfaces = None - self.marker_cache = marker_cache + self.marker_cache_path = marker_cache_path def save_surface_statisics_to_file(self): logger.info("exporting metrics to {}".format(self.metrics_dir)) @@ -344,13 +345,23 @@ def _map_gaze_and_fixations(self): return gaze_on_surface, fixations_on_surface def _export_marker_detections(self): - file_path = os.path.join(self.metrics_dir, "marker_detections.csv") - with open(file_path, "w", encoding="utf-8", newline="") as csv_file: - csv_writer = csv.writer(csv_file, delimiter=",") - csv_writer.writerow(("world_index", "marker_uid")) - for idx, markers in enumerate(self.marker_cache): - for m in markers: - csv_writer.writerow((idx, m.uid)) + + # Load the temporary marker cache created by the offline surface tracker + marker_cache = file_methods.Persistent_Dict(self.marker_cache_path) + marker_cache = marker_cache["marker_cache"] + + try: + file_path = os.path.join(self.metrics_dir, "marker_detections.csv") + with open(file_path, "w", encoding="utf-8", newline="") as csv_file: + csv_writer = csv.writer(csv_file, delimiter=",") + csv_writer.writerow(("world_index", "marker_uid")) + for idx, markers in enumerate(marker_cache): + for m in markers: + csv_writer.writerow((idx, m.uid)) + finally: + # Delete the temporary marker cache created by the offline surface tracker + os.remove(self.marker_cache_path) + self.marker_cache_path = None def _export_surface_visibility(self): with open( diff --git a/pupil_src/shared_modules/surface_tracker/surface_tracker_offline.py b/pupil_src/shared_modules/surface_tracker/surface_tracker_offline.py index 6d66a2a52d..9094085155 100644 --- a/pupil_src/shared_modules/surface_tracker/surface_tracker_offline.py +++ b/pupil_src/shared_modules/surface_tracker/surface_tracker_offline.py @@ -14,6 +14,7 @@ import multiprocessing import os import platform +import tempfile import time import typing as T @@ -559,6 +560,16 @@ def on_notify(self, notification): break elif notification["subject"] == "should_export": + # Create new marker cache temporary file + # Backgroud exporter is responsible of removing the temporary file when finished + file_handle, marker_cache_path = tempfile.mkstemp() + os.close(file_handle) # https://bugs.python.org/issue42830 + + # Save marker cache into the new temporary file + temp_marker_cache = file_methods.Persistent_Dict(marker_cache_path) + temp_marker_cache["marker_cache"] = self.marker_cache + temp_marker_cache.save() + proxy = background_tasks.get_export_proxy( notification["export_dir"], notification["range"], @@ -567,7 +578,7 @@ def on_notify(self, notification): self.g_pool.gaze_positions, self.g_pool.fixations, self.camera_model, - self.marker_cache, + marker_cache_path, mp_context, ) self.export_proxies.add(proxy) From 647be448fd68d16168231a2d1304c148a19674bc Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Tue, 11 May 2021 15:27:39 +0200 Subject: [PATCH 3/6] Treat empty files as non-existent in Persistent_Dict --- pupil_src/shared_modules/file_methods.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/file_methods.py b/pupil_src/shared_modules/file_methods.py index 3afcf892c8..e11a4bca45 100644 --- a/pupil_src/shared_modules/file_methods.py +++ b/pupil_src/shared_modules/file_methods.py @@ -42,7 +42,9 @@ def __init__(self, file_path, *args, **kwargs): super().__init__(*args, **kwargs) self.file_path = os.path.expanduser(file_path) try: - self.update(**load_object(self.file_path, allow_legacy=False)) + if os.path.getsize(file_path) > 0: + # Only try to load object if file is not empty + self.update(**load_object(self.file_path, allow_legacy=False)) except IOError: logger.debug( f"Session settings file '{self.file_path}' not found." From bcc7560ad155a4bfb474665f7c8ca4545b4f7fdc Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Tue, 11 May 2021 15:28:13 +0200 Subject: [PATCH 4/6] Correctly deserialize Surface_Marker in background exporter --- .../shared_modules/surface_tracker/background_tasks.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pupil_src/shared_modules/surface_tracker/background_tasks.py b/pupil_src/shared_modules/surface_tracker/background_tasks.py index 0a36f6105f..25552b0102 100644 --- a/pupil_src/shared_modules/surface_tracker/background_tasks.py +++ b/pupil_src/shared_modules/surface_tracker/background_tasks.py @@ -21,6 +21,9 @@ import player_methods import file_methods +from .surface_marker import Surface_Marker + + logger = logging.getLogger(__name__) @@ -355,8 +358,8 @@ def _export_marker_detections(self): with open(file_path, "w", encoding="utf-8", newline="") as csv_file: csv_writer = csv.writer(csv_file, delimiter=",") csv_writer.writerow(("world_index", "marker_uid")) - for idx, markers in enumerate(marker_cache): - for m in markers: + for idx, serialized_markers in enumerate(marker_cache): + for m in map(Surface_Marker.deserialize, serialized_markers): csv_writer.writerow((idx, m.uid)) finally: # Delete the temporary marker cache created by the offline surface tracker From 640751271f4bb1095c026c4c3c9114e92713027b Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Tue, 11 May 2021 18:55:25 +0200 Subject: [PATCH 5/6] Add surface marker corner coordinates to export --- .../surface_tracker/background_tasks.py | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/pupil_src/shared_modules/surface_tracker/background_tasks.py b/pupil_src/shared_modules/surface_tracker/background_tasks.py index 25552b0102..236c7bc3c5 100644 --- a/pupil_src/shared_modules/surface_tracker/background_tasks.py +++ b/pupil_src/shared_modules/surface_tracker/background_tasks.py @@ -357,10 +357,31 @@ def _export_marker_detections(self): file_path = os.path.join(self.metrics_dir, "marker_detections.csv") with open(file_path, "w", encoding="utf-8", newline="") as csv_file: csv_writer = csv.writer(csv_file, delimiter=",") - csv_writer.writerow(("world_index", "marker_uid")) + csv_writer.writerow( + ( + "world_index", + "marker_uid", + "corner_0_x", + "corner_0_y", + "corner_1_x", + "corner_1_y", + "corner_2_x", + "corner_2_y", + "corner_3_x", + "corner_3_y", + ) + ) for idx, serialized_markers in enumerate(marker_cache): for m in map(Surface_Marker.deserialize, serialized_markers): - csv_writer.writerow((idx, m.uid)) + flat_corners = [x for c in m.verts_px for x in c[0]] + assert len(flat_corners) == 8 # sanity check + csv_writer.writerow( + ( + idx, + m.uid, + *flat_corners, + ) + ) finally: # Delete the temporary marker cache created by the offline surface tracker os.remove(self.marker_cache_path) From f0d43c26ba3170e89bb098351c0a22132847fb97 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 12 May 2021 10:13:52 +0200 Subject: [PATCH 6/6] Attempt to optimize memory usage in surface tracking exporter --- .../surface_tracker/background_tasks.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/surface_tracker/background_tasks.py b/pupil_src/shared_modules/surface_tracker/background_tasks.py index 236c7bc3c5..893a76a80e 100644 --- a/pupil_src/shared_modules/surface_tracker/background_tasks.py +++ b/pupil_src/shared_modules/surface_tracker/background_tasks.py @@ -285,7 +285,6 @@ def save_surface_statisics_to_file(self): self.fixations_on_surfaces, ) = self._map_gaze_and_fixations() - self._export_marker_detections() self._export_surface_visibility() self._export_surface_gaze_distribution() self._export_surface_events() @@ -307,6 +306,17 @@ def save_surface_statisics_to_file(self): "Saved surface gaze and fixation data for '{}'".format(surface.name) ) + # Cleanup surface related data to release memory + self.surfaces = None + self.fixations = None + self.gaze_positions = None + self.gaze_on_surfaces = None + self.fixations_on_surfaces = None + + # Perform marker export *after* surface data is released + # to avoid holding everything in memory all at once. + self._export_marker_detections() + logger.info("Done exporting reference surface data.") return