Merge pull request #1577 from papr/surface-tracker-apriltag

v1.15 Fixes

pupil_src/shared_modules/head_pose_tracker/worker/detection_worker.py

@@ -29,7 +29,8 @@
 def get_markers_data(detection, img_size, timestamp):
     return {
         "id": detection.tag_id,
-        "verts": detection.corners[::-1].tolist(),
+        # verts: Corners need to be listed counter-clockwise
+        "verts": detection.corners.tolist(),
         "centroid": normalize(detection.center, img_size, flip_y=True),
         "timestamp": timestamp,
     }

pupil_src/shared_modules/surface_tracker/background_tasks.py

@@ -537,6 +537,8 @@ def _export_fixations_on_surface(self, fixations_on_surf, surface, surface_name)
             csv_writer = csv.writer(csv_file, delimiter=",")
             csv_writer.writerow(
                 (
+                    "world_timestamp",
+                    "world_index",
                     "fixation_id",
                     "start_timestamp",
                     "duration",
@@ -548,21 +550,22 @@ def _export_fixations_on_surface(self, fixations_on_surf, surface, surface_name)
                     "on_surf",
                 )
             )
-            # flatten `fixations_on_surf[world_frame_idx][frame_event_idx]`
-            # to `fixations_on_surf[global_event_idx]`
-            fixations_on_surf = itertools.chain.from_iterable(fixations_on_surf)
-            without_duplicates = {fix["id"]: fix for fix in fixations_on_surf}.values()
-            for fix in without_duplicates:
-                csv_writer.writerow(
-                    (
-                        fix["id"],
-                        fix["timestamp"],
-                        fix["duration"],
-                        fix["dispersion"],
-                        fix["norm_pos"][0],
-                        fix["norm_pos"][1],
-                        fix["norm_pos"][0] * surface.real_world_size["x"],
-                        fix["norm_pos"][1] * surface.real_world_size["y"],
-                        fix["on_surf"],
-                    )
-                )
+            for world_idx, fixs_for_frame in enumerate(fixations_on_surf):
+                world_idx += self.export_range[0]
+                if fixs_for_frame:
+                    for fix in fixs_for_frame:
+                        csv_writer.writerow(
+                            (
+                                self.world_timestamps[world_idx],
+                                world_idx,
+                                fix["id"],
+                                fix["timestamp"],
+                                fix["duration"],
+                                fix["dispersion"],
+                                fix["norm_pos"][0],
+                                fix["norm_pos"][1],
+                                fix["norm_pos"][0] * surface.real_world_size["x"],
+                                fix["norm_pos"][1] * surface.real_world_size["y"],
+                                fix["on_surf"],
+                            )
+                        )

pupil_src/shared_modules/surface_tracker/surface.py

@@ -552,6 +552,7 @@ def get_placeholder_heatmap(self):
 
     def save_to_dict(self):
         return {
+            "version": self.version,
             "name": self.name,
             "real_world_size": self.real_world_size,
             "reg_markers": [

pupil_src/shared_modules/surface_tracker/surface_marker_detector.py

@@ -28,8 +28,8 @@
 
 @enum.unique
 class Surface_Marker_Detector_Mode(enum.Enum):
-    SQUARE_MARKER = 'square_marker'
-    APRILTAG_MARKER = 'apriltag_marker'
+    SQUARE_MARKER = "square_marker"
+    APRILTAG_MARKER = "apriltag_marker"
 
     @staticmethod
     def all_supported_cases() -> typing.Set["Surface_Marker_Detector_Mode"]:
@@ -42,7 +42,9 @@ def from_marker(marker: Surface_Marker) -> "Surface_Marker_Detector_Mode":
             return Surface_Marker_Detector_Mode.SQUARE_MARKER
         if marker_type == Surface_Marker_Type.APRILTAG_V3:
             return Surface_Marker_Detector_Mode.APRILTAG_MARKER
-        raise ValueError(f"Can't map marker of type '{marker_type}' to a detection mode")
+        raise ValueError(
+            f"Can't map marker of type '{marker_type}' to a detection mode"
+        )
 
     @property
     def label(self) -> str:
@@ -95,11 +97,15 @@ def marker_detector_modes(self, value: typing.Set[Surface_Marker_Detector_Mode])
         pass
 
     @abc.abstractmethod
-    def detect_markers_iter(self, gray_img, frame_index: int) -> typing.Iterable[Surface_Marker]:
+    def detect_markers_iter(
+        self, gray_img, frame_index: int
+    ) -> typing.Iterable[Surface_Marker]:
         pass
 
     def detect_markers(self, gray_img, frame_index: int) -> typing.List[Surface_Marker]:
-        return list(self.detect_markers_iter(gray_img=gray_img, frame_index=frame_index))
+        return list(
+            self.detect_markers_iter(gray_img=gray_img, frame_index=frame_index)
+        )
 
     def _surface_marker_filter(self, marker: Surface_Marker) -> bool:
         return self.marker_min_perimeter <= marker.perimeter
@@ -115,9 +121,7 @@ def __init__(
         **kwargs,
     ):
         self.__marker_min_perimeter = (
-            marker_min_perimeter
-            if marker_min_perimeter is not ...
-            else 60
+            marker_min_perimeter if marker_min_perimeter is not ... else 60
         )
         self.__robust_detection = (
             square_marker_robust_detection
@@ -165,7 +169,9 @@ def marker_detector_modes(self) -> typing.Set[Surface_Marker_Detector_Mode]:
     def marker_detector_modes(self, value: typing.Set[Surface_Marker_Detector_Mode]):
         self.__marker_detector_modes = value
 
-    def detect_markers_iter(self, gray_img, frame_index: int) -> typing.Iterable[Surface_Marker]:
+    def detect_markers_iter(
+        self, gray_img, frame_index: int
+    ) -> typing.Iterable[Surface_Marker]:
         if Surface_Marker_Detector_Mode.SQUARE_MARKER not in self.marker_detector_modes:
             return []
 
@@ -226,9 +232,7 @@ def __init__(
         self.debug = debug
 
     def to_dict(self):
-        d = {
-            "families": " ".join(self.families),
-        }
+        d = {"families": " ".join(self.families)}
         if self.nthreads is not ...:
             d["nthreads"] = self.nthreads
         if self.quad_decimate is not ...:
@@ -245,7 +249,6 @@ def to_dict(self):
 
 
 class Surface_Apriltag_V3_Marker_Detector(Surface_Base_Marker_Detector):
-
     def __getstate__(self):
         return (
             self.__detector_params,
@@ -283,11 +286,7 @@ def __init__(
             decode_sharpening=apriltag_decode_sharpening,
             debug=apriltag_debug,
         )
-        state = (
-            detector_params,
-            marker_min_perimeter,
-            marker_detector_modes,
-        )
+        state = (detector_params, marker_min_perimeter, marker_detector_modes)
         self.__setstate__(state)
 
     @property
@@ -322,8 +321,13 @@ def marker_detector_modes(self) -> typing.Set[Surface_Marker_Detector_Mode]:
     def marker_detector_modes(self, value: typing.Set[Surface_Marker_Detector_Mode]):
         self.__marker_detector_modes = value
 
-    def detect_markers_iter(self, gray_img, frame_index: int) -> typing.Iterable[Surface_Marker]:
-        if Surface_Marker_Detector_Mode.APRILTAG_MARKER not in self.marker_detector_modes:
+    def detect_markers_iter(
+        self, gray_img, frame_index: int
+    ) -> typing.Iterable[Surface_Marker]:
+        if (
+            Surface_Marker_Detector_Mode.APRILTAG_MARKER
+            not in self.marker_detector_modes
+        ):
             return []
         markers = self._detector.detect(img=gray_img)
         markers = map(Surface_Marker.from_apriltag_v3_detection, markers)
@@ -408,10 +412,16 @@ def marker_detector_modes(self, value: typing.Set[Surface_Marker_Detector_Mode])
         self.__square_detector.marker_detector_modes = value
         self.__apriltag_detector.marker_detector_modes = value
 
-    def detect_markers_iter(self, gray_img, frame_index: int) -> typing.Iterable[Surface_Marker]:
+    def detect_markers_iter(
+        self, gray_img, frame_index: int
+    ) -> typing.Iterable[Surface_Marker]:
         return itertools.chain(
-            self.__square_detector.detect_markers_iter(gray_img=gray_img, frame_index=frame_index),
-            self.__apriltag_detector.detect_markers_iter(gray_img=gray_img, frame_index=frame_index),
+            self.__square_detector.detect_markers_iter(
+                gray_img=gray_img, frame_index=frame_index
+            ),
+            self.__apriltag_detector.detect_markers_iter(
+                gray_img=gray_img, frame_index=frame_index
+            ),
         )
 
 

pupil_src/shared_modules/surface_tracker/surface_tracker.py

@@ -23,7 +23,10 @@
 import file_methods
 
 from . import gui
-from .surface_marker_detector import Surface_Marker_Detector, Surface_Marker_Detector_Mode
+from .surface_marker_detector import (
+    Surface_Marker_Detector,
+    Surface_Marker_Detector_Mode,
+)
 
 
 class Surface_Tracker(Plugin, metaclass=ABCMeta):
@@ -35,7 +38,15 @@ class Surface_Tracker(Plugin, metaclass=ABCMeta):
     icon_chr = chr(0xEC07)
     icon_font = "pupil_icons"
 
-    def __init__(self, g_pool, marker_min_perimeter: int = 60, inverted_markers: bool = False):
+    def __init__(
+        self,
+        g_pool,
+        marker_min_perimeter: int = 60,
+        inverted_markers: bool = False,
+        marker_detector_mode: typing.Union[
+            Surface_Marker_Detector_Mode, str
+        ] = Surface_Marker_Detector_Mode.APRILTAG_MARKER,
+    ):
         super().__init__(g_pool)
 
         self.current_frame = None
@@ -47,7 +58,14 @@ def __init__(self, g_pool, marker_min_perimeter: int = 60, inverted_markers: boo
         self._last_mouse_pos = (0.0, 0.0)
         self.gui = gui.GUI(self)
 
-        marker_detector_modes = {Surface_Marker_Detector_Mode.APRILTAG_MARKER}
+        if isinstance(marker_detector_mode, str):
+            # Here we ensure that we pass a proper Surface_Marker_Detector_Mode
+            # instance to Surface_Marker_Detector:
+            marker_detector_mode = Surface_Marker_Detector_Mode(marker_detector_mode)
+
+        # Even though the Surface_Marker_Detector class supports multiple detector
+        # modes at once, we only want one mode to be active during surface tracking.
+        marker_detector_modes = {marker_detector_mode}
         self._marker_min_perimeter = marker_min_perimeter
         self.marker_min_confidence = 0.0
         self.marker_detector = Surface_Marker_Detector(
@@ -77,6 +95,16 @@ def marker_detector_modes(self) -> typing.Set[Surface_Marker_Detector_Mode]:
     def marker_detector_modes(self, value: typing.Set[Surface_Marker_Detector_Mode]):
         self.marker_detector.marker_detector_modes = value
 
+    @property
+    def active_marker_detector_mode(self) -> Surface_Marker_Detector_Mode:
+        assert len(self.marker_detector_modes) == 1
+        return next(iter(self.marker_detector_modes))
+
+    @active_marker_detector_mode.setter
+    def active_marker_detector_mode(self, mode: Surface_Marker_Detector_Mode):
+        self.marker_detector_modes = {mode}
+        self.notify_all({"subject": "surface_tracker.marker_detection_params_changed"})
+
     @property
     def marker_min_perimeter(self) -> int:
         return self._marker_min_perimeter
@@ -209,18 +237,12 @@ def set_robust_detection(val):
                 {"subject": "surface_tracker.marker_detection_params_changed"}
             )
 
-        def marker_detector_mode_getter() -> Surface_Marker_Detector_Mode:
-            assert len(self.marker_detector_modes) == 1
-            return next(iter(self.marker_detector_modes))
-
-        def marker_detector_mode_setter(mode: Surface_Marker_Detector_Mode):
-            self.marker_detector_modes = {mode}
-            self.notify_all(
-                {"subject": "surface_tracker.marker_detection_params_changed"}
-            )
-
-        supported_surface_marker_detector_modes = Surface_Marker_Detector_Mode.all_supported_cases()
-        supported_surface_marker_detector_modes = sorted(supported_surface_marker_detector_modes, key=lambda m: m.value)
+        supported_surface_marker_detector_modes = (
+            Surface_Marker_Detector_Mode.all_supported_cases()
+        )
+        supported_surface_marker_detector_modes = sorted(
+            supported_surface_marker_detector_modes, key=lambda m: m.value
+        )
 
         advanced_menu = pyglui.ui.Growing_Menu("Marker Detection Parameters")
         advanced_menu.collapsed = True
@@ -253,14 +275,11 @@ def marker_detector_mode_setter(mode: Surface_Marker_Detector_Mode):
         )
         advanced_menu.append(
             pyglui.ui.Selector(
-                "marker_detector_mode",
-                None, #self,
+                "active_marker_detector_mode",
+                self,
                 label="Marker Detector Mode",
-                getter=marker_detector_mode_getter,
-                setter=marker_detector_mode_setter,
                 labels=[mode.label for mode in supported_surface_marker_detector_modes],
                 selection=[mode for mode in supported_surface_marker_detector_modes],
-                # selection_getter=marker_detection_mode_selection_getter,
             )
         )
         self.menu.append(advanced_menu)
@@ -406,7 +425,9 @@ def _update_markers(self, frame):
         pass
 
     def _detect_markers(self, frame):
-        markers = self.marker_detector.detect_markers(gray_img=frame.gray, frame_index=frame.index)
+        markers = self.marker_detector.detect_markers(
+            gray_img=frame.gray, frame_index=frame.index
+        )
         markers = self._remove_duplicate_markers(markers)
         self.markers_unfiltered = markers
         self.markers = self._filter_markers(markers)
@@ -522,6 +543,7 @@ def get_init_dict(self):
         return {
             "marker_min_perimeter": self.marker_min_perimeter,
             "inverted_markers": self.inverted_markers,
+            "marker_detector_mode": self.active_marker_detector_mode.value,
         }
 
     def save_surface_definitions_to_file(self):

pupil_src/shared_modules/surface_tracker/surface_tracker_offline.py

@@ -54,8 +54,8 @@ class Surface_Tracker_Offline(Surface_Tracker, Analysis_Plugin_Base):
     order = 0.2
     TIMELINE_LINE_HEIGHT = 16
 
-    def __init__(self, g_pool, marker_min_perimeter=60, inverted_markers=False):
-        super().__init__(g_pool, marker_min_perimeter, inverted_markers)
+    def __init__(self, g_pool, *args, **kwargs):
+        super().__init__(g_pool, *args, **kwargs)
 
         self.MARKER_CACHE_VERSION = 3
         # Also add very small detected markers to cache and filter cache afterwards
@@ -108,12 +108,16 @@ def _init_marker_detection_modes(self):
         # Filter out non-filled frames where the cache entry is None.
         # Chain the remaining entries (which are lists) to get a flat sequence.
         filled_out_marker_cache = filter(lambda x: x is not None, self.marker_cache)
-        cached_surface_marker_sequence = itertools.chain.from_iterable(filled_out_marker_cache)
+        cached_surface_marker_sequence = itertools.chain.from_iterable(
+            filled_out_marker_cache
+        )
 
         # Get the first surface marker from the sequence, and set the detection mode according to it.
         first_cached_surface_marker = next(cached_surface_marker_sequence, None)
         if first_cached_surface_marker is not None:
-            marker_detector_mode = Surface_Marker_Detector_Mode.from_marker(first_cached_surface_marker)
+            marker_detector_mode = Surface_Marker_Detector_Mode.from_marker(
+                first_cached_surface_marker
+            )
             self.marker_detector_modes = {marker_detector_mode}
 
     def _init_marker_cache(self):
@@ -333,21 +337,6 @@ def _start_gaze_buffer_filler(self, all_gaze_events, all_world_timestamps, secti
             mp_context,
         )
 
-    def _start_fixation_buffer_filler(
-        self, all_fixation_events, all_world_timestamps, section
-    ):
-        if self.fixations_on_surf_buffer_filler is not None:
-            self.fixations_on_surf_buffer_filler.cancel()
-        self.fixations_on_surf_buffer = []
-        self.fixations_on_surf_buffer_filler = background_tasks.background_gaze_on_surface(
-            self.surfaces,
-            section,
-            all_world_timestamps,
-            all_fixation_events,
-            self.camera_model,
-            mp_context,
-        )
-
     def gl_display(self):
         if self.timeline:
             self.timeline.refresh()

pupil_src/shared_modules/surface_tracker/surface_tracker_online.py

@@ -28,15 +28,11 @@ class Surface_Tracker_Online(Surface_Tracker):
     necessary computation is done per frame.
     """
 
-    def __init__(self, g_pool, marker_min_perimeter=60, inverted_markers=False):
+    def __init__(self, g_pool, *args, **kwargs):
         self.freeze_scene = False
         self.frozen_scene_frame = None
         self.frozen_scene_tex = None
-        super().__init__(
-            g_pool,
-            marker_min_perimeter=marker_min_perimeter,
-            inverted_markers=inverted_markers,
-        )
+        super().__init__(g_pool, *args, **kwargs)
 
         self.menu = None
         self.button = None