Merge pull request #1489 from papr/generic_overlay

Video Overlays

pupil_src/launchables/player.py

@@ -97,7 +97,6 @@ def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_versio
         from vis_fixation import Vis_Fixation
 
         # from vis_scan_path import Vis_Scan_Path
-        from vis_eye_video_overlay import Vis_Eye_Video_Overlay
         from seek_control import Seek_Control
         from offline_surface_tracker import Offline_Surface_Tracker
 
@@ -120,6 +119,7 @@ def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_versio
         from video_export.plugins.eye_video_exporter import Eye_Video_Exporter
         from video_export.plugins.world_video_exporter import World_Video_Exporter
         from video_capture import File_Source
+        from video_overlay.plugins import Video_Overlay, Eye_Overlay
 
         assert VersionFormat(pyglui_version) >= VersionFormat(
             "1.23"
@@ -141,7 +141,8 @@ def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_versio
             Vis_Light_Points,
             Vis_Cross,
             Vis_Watermark,
-            Vis_Eye_Video_Overlay,
+            Eye_Overlay,
+            Video_Overlay,
             # Vis_Scan_Path,
             Offline_Fixation_Detector,
             Offline_Blink_Detection,

pupil_src/shared_modules/player_methods.py

@@ -260,8 +260,7 @@ def transparent_image_overlay(pos, overlay_img, img, alpha):
     )
     try:
         cv2.addWeighted(overlay_img, alpha, img[roi], 1.0 - alpha, 0, img[roi])
-    except:
+    except (TypeError, cv2.error):
         logger.debug(
             "transparent_image_overlay was outside of the world image and was not drawn"
         )
-    pass

pupil_src/shared_modules/video_export/plugins/world_video_exporter.py

@@ -73,6 +73,8 @@ def _precomputed_eye_data_for_range(self, export_range):
         pre_computed = {
             "gaze": self.g_pool.gaze_positions,
             "pupil": self.g_pool.pupil_positions,
+            "pupil_by_id_0": self.g_pool.pupil_positions_by_id[0],
+            "pupil_by_id_1": self.g_pool.pupil_positions_by_id[1],
             "fixations": self.g_pool.fixations,
         }
 
@@ -116,9 +118,9 @@ def _export_world_video(
     # Plug-ins
     from plugin import Plugin_List, import_runtime_plugins
     from video_capture import EndofVideoError, File_Source
+    from video_overlay.plugins import Video_Overlay, Eye_Overlay
     from vis_circle import Vis_Circle
     from vis_cross import Vis_Cross
-    from vis_eye_video_overlay import Vis_Eye_Video_Overlay
     from vis_light_points import Vis_Light_Points
     from vis_polyline import Vis_Polyline
     from vis_scan_path import Vis_Scan_Path
@@ -139,7 +141,8 @@ def _export_world_video(
                 Vis_Light_Points,
                 Vis_Watermark,
                 Vis_Scan_Path,
-                Vis_Eye_Video_Overlay,
+                Eye_Overlay,
+                Video_Overlay,
             ],
             key=lambda x: x.__name__,
         )
@@ -231,6 +234,10 @@ def _export_world_video(
             ]
 
         g_pool.pupil_positions = pm.Bisector(**pre_computed_eye_data["pupil"])
+        g_pool.pupil_positions_by_id = (
+            pm.Bisector(**pre_computed_eye_data["pupil_by_id_0"]),
+            pm.Bisector(**pre_computed_eye_data["pupil_by_id_1"]),
+        )
         g_pool.gaze_positions = pm.Bisector(**pre_computed_eye_data["gaze"])
         g_pool.fixations = pm.Affiliator(**pre_computed_eye_data["fixations"])
 

pupil_src/shared_modules/video_overlay/controllers/overlay_manager.py

@@ -0,0 +1,54 @@
+import abc
+from storage import SingleFileStorage
+from video_overlay.models.config import Configuration
+from video_overlay.workers.overlay_renderer import OverlayRenderer
+
+
+class OverlayManager(SingleFileStorage):
+    def __init__(self, rec_dir, plugin):
+        super().__init__(rec_dir, plugin)
+        self._overlays = []
+        self._load_from_disk()
+        self._patch_on_cleanup(plugin)
+
+    @property
+    def _storage_file_name(self):
+        return "video_overlays.msgpack"
+
+    @property
+    def _item_class(self):
+        return Configuration
+
+    def add(self, item):
+        overlay = OverlayRenderer(item)
+        self._overlays.append(overlay)
+
+    def delete(self, item):
+        for overlay in self._overlays.copy():
+            if overlay.config is item:
+                self._overlays.remove(overlay)
+
+    @property
+    def items(self):
+        yield from (overlay.config for overlay in self._overlays)
+
+    @property
+    def overlays(self):
+        return self._overlays
+
+    @property
+    def most_recent(self):
+        return self._overlays[-1]
+
+    def remove_overlay(self, overlay):
+        self._overlays.remove(overlay)
+        self.save_to_disk()
+
+    def _patch_on_cleanup(self, plugin):
+        """Patches cleanup observer to trigger on get_init_dict().
+
+        Save current settings to disk on get_init_dict() instead of cleanup().
+        This ensures that the World Video Exporter loads the most recent settings.
+        """
+        plugin.remove_observer("cleanup", self._on_cleanup)
+        plugin.add_observer("get_init_dict", self._on_cleanup)

pupil_src/shared_modules/video_overlay/models/config.py

@@ -0,0 +1,56 @@
+from storage import StorageItem
+
+from video_overlay.utils.constraints import (
+    ConstraintedPosition,
+    ConstraintedValue,
+    BooleanConstraint,
+    InclusiveConstraint,
+)
+
+
+class Configuration(StorageItem):
+    version = 0
+
+    def __init__(
+        self,
+        video_path=None,
+        origin_x=0,
+        origin_y=0,
+        scale=0.6,
+        alpha=0.8,
+        hflip=False,
+        vflip=False,
+    ):
+        self.video_path = video_path
+        self.origin = ConstraintedPosition(origin_x, origin_y)
+        self.scale = ConstraintedValue(scale, InclusiveConstraint(low=0.2, high=1.0))
+        self.alpha = ConstraintedValue(alpha, InclusiveConstraint(low=0.1, high=1.0))
+        self.hflip = ConstraintedValue(hflip, BooleanConstraint())
+        self.vflip = ConstraintedValue(vflip, BooleanConstraint())
+
+    @property
+    def as_tuple(self):
+        return (
+            self.video_path,
+            self.origin.x.value,
+            self.origin.y.value,
+            self.scale.value,
+            self.alpha.value,
+            self.hflip.value,
+            self.vflip.value,
+        )
+
+    @staticmethod
+    def from_tuple(tuple_):
+        return Configuration(*tuple_)
+
+    def as_dict(self):
+        return {
+            "video_path": self.video_path,
+            "origin_x": self.origin.x.value,
+            "origin_y": self.origin.y.value,
+            "scale": self.scale.value,
+            "alpha": self.alpha.value,
+            "hflip": self.hflip.value,
+            "vflip": self.vflip.value,
+        }

pupil_src/shared_modules/video_overlay/plugins/__init__.py

@@ -0,0 +1,2 @@
+from video_overlay.plugins.generic_overlay import Video_Overlay
+from video_overlay.plugins.eye_overlay import Eye_Overlay

pupil_src/shared_modules/video_overlay/plugins/eye_overlay.py

@@ -0,0 +1,117 @@
+import os
+import glob
+
+import player_methods as pm
+from plugin import Plugin
+from observable import Observable
+
+from video_overlay.workers.overlay_renderer import EyeOverlayRenderer
+from video_overlay.models.config import Configuration
+from video_overlay.ui.management import UIManagementEyes
+from video_overlay.utils.constraints import ConstraintedValue, BooleanConstraint
+
+
+class Eye_Overlay(Observable, Plugin):
+    icon_chr = chr(0xEC02)
+    icon_font = "pupil_icons"
+
+    def __init__(
+        self,
+        g_pool,
+        scale=0.6,
+        alpha=0.8,
+        show_ellipses=True,
+        eye0_config=None,
+        eye1_config=None,
+    ):
+        super().__init__(g_pool)
+        eye0_config = eye0_config or {"vflip": True, "origin_x": 210, "origin_y": 60}
+        eye1_config = eye1_config or {"hflip": True, "origin_x": 10, "origin_y": 60}
+
+        self.current_frame_ts = None
+        self.show_ellipses = ConstraintedValue(show_ellipses, BooleanConstraint())
+        self._scale = scale
+        self._alpha = alpha
+
+        self.eye0 = self._setup_eye(0, eye0_config)
+        self.eye1 = self._setup_eye(1, eye1_config)
+
+    def recent_events(self, events):
+        if "frame" in events:
+            frame = events["frame"]
+            self.current_frame_ts = frame.timestamp
+            for overlay in (self.eye0, self.eye1):
+                overlay.draw_on_frame(frame)
+
+    @property
+    def scale(self):
+        return self._scale
+
+    @scale.setter
+    def scale(self, val):
+        self._scale = val
+        self.eye0.config.scale.value = val
+        self.eye1.config.scale.value = val
+
+    @property
+    def alpha(self):
+        return self._alpha
+
+    @alpha.setter
+    def alpha(self, val):
+        self._alpha = val
+        self.eye0.config.alpha.value = val
+        self.eye1.config.alpha.value = val
+
+    def init_ui(self):
+        self.add_menu()
+        self.menu.label = "Eye Video Overlays"
+        self.ui = UIManagementEyes(self, self.menu, (self.eye0, self.eye1))
+
+    def deinit_ui(self):
+        self.ui.teardown()
+        self.remove_menu()
+
+    def _setup_eye(self, eye_id, prefilled_config):
+        video_path = self._video_path_for_eye(eye_id)
+        prefilled_config["video_path"] = video_path
+        prefilled_config["scale"] = self.scale
+        prefilled_config["alpha"] = self.alpha
+        config = Configuration(**prefilled_config)
+        overlay = EyeOverlayRenderer(
+            config, self.show_ellipses, self.make_current_pupil_datum_getter(eye_id)
+        )
+        return overlay
+
+    def _video_path_for_eye(self, eye_id):
+        rec_dir = self.g_pool.rec_dir
+        video_file_pattern = "eye{}.*".format(eye_id)
+        video_path_pattern = os.path.join(rec_dir, video_file_pattern)
+        try:
+            video_path_candidates = glob.iglob(video_path_pattern)
+            return next(video_path_candidates)
+        except StopIteration:
+            return "/not/found/eye{}.mp4".format(eye_id)
+
+    def get_init_dict(self):
+        return {
+            "scale": self.scale,
+            "alpha": self.alpha,
+            "show_ellipses": self.show_ellipses.value,
+            "eye0_config": self.eye0.config.as_dict(),
+            "eye1_config": self.eye1.config.as_dict(),
+        }
+
+    def make_current_pupil_datum_getter(self, eye_id):
+        def _pupil_getter():
+            try:
+                pupil_data = self.g_pool.pupil_positions_by_id[eye_id]
+                closest_pupil_idx = pm.find_closest(
+                    pupil_data.data_ts, self.current_frame_ts
+                )
+                current_datum = pupil_data.data[closest_pupil_idx]
+                return current_datum
+            except (IndexError, ValueError):
+                return None
+
+        return _pupil_getter

pupil_src/shared_modules/video_overlay/plugins/generic_overlay.py

@@ -0,0 +1,69 @@
+import collections
+import os
+
+from observable import Observable
+from plugin import Plugin
+from video_capture.utils import VIDEO_EXTS
+
+from video_overlay.models.config import Configuration
+from video_overlay.controllers.overlay_manager import OverlayManager
+from video_overlay.ui.management import UIManagementGeneric
+from video_overlay.ui.interactions import current_mouse_pos
+
+
+class Video_Overlay(Observable, Plugin):
+    icon_chr = "O"
+
+    def __init__(self, g_pool):
+        super().__init__(g_pool)
+        self.manager = OverlayManager(g_pool.rec_dir, self)
+
+    def recent_events(self, events):
+        if "frame" in events:
+            frame = events["frame"]
+            for overlay in self.manager.overlays:
+                overlay.draw_on_frame(frame)
+
+    def on_drop(self, paths):
+        multi_drop_offset = 10
+        inital_drop = current_mouse_pos(
+            self.g_pool.main_window,
+            self.g_pool.camera_render_size,
+            self.g_pool.capture.frame_size,
+        )
+        valid_paths = [p for p in paths if self.valid_path(p)]
+        for video_path in valid_paths:
+            self._add_overlay_to_storage(video_path, inital_drop)
+            inital_drop = (
+                inital_drop[0] + multi_drop_offset,
+                inital_drop[1] + multi_drop_offset,
+            )
+        # event only consumed if at least one valid file was present
+        return bool(valid_paths)
+
+    @staticmethod
+    def valid_path(path):
+        # splitext()[1] always starts with `.`
+        ext = os.path.splitext(path)[1][1:]
+        return ext in VIDEO_EXTS
+
+    def _add_overlay_to_storage(self, video_path, initial_pos=(0, 0)):
+        config = Configuration(
+            video_path, origin_x=initial_pos[0], origin_y=initial_pos[1]
+        )
+        self.manager.add(config)
+        self.manager.save_to_disk()
+        self._overlay_added_to_storage(self.manager.most_recent)
+
+    def _overlay_added_to_storage(self, overlay):
+        pass  # observed to create menus and draggables
+
+    def init_ui(self):
+        self.add_menu()
+        self.menu.label = "Generic Video Overlays"
+        self.ui = UIManagementGeneric(self, self.menu, self.manager.overlays)
+        self.ui.add_observer("remove_overlay", self.manager.remove_overlay)
+
+    def deinit_ui(self):
+        self.ui.teardown()
+        self.remove_menu()