diff --git a/pupil_src/launchables/player.py b/pupil_src/launchables/player.py index e6f6875dde..5135571032 100644 --- a/pupil_src/launchables/player.py +++ b/pupil_src/launchables/player.py @@ -97,7 +97,6 @@ def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_versio from vis_fixation import Vis_Fixation # from vis_scan_path import Vis_Scan_Path - from vis_eye_video_overlay import Vis_Eye_Video_Overlay from seek_control import Seek_Control from offline_surface_tracker import Offline_Surface_Tracker @@ -120,6 +119,7 @@ def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_versio from video_export.plugins.eye_video_exporter import Eye_Video_Exporter from video_export.plugins.world_video_exporter import World_Video_Exporter from video_capture import File_Source + from video_overlay.plugins import Video_Overlay, Eye_Overlay assert VersionFormat(pyglui_version) >= VersionFormat( "1.23" @@ -141,7 +141,8 @@ def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_versio Vis_Light_Points, Vis_Cross, Vis_Watermark, - Vis_Eye_Video_Overlay, + Eye_Overlay, + Video_Overlay, # Vis_Scan_Path, Offline_Fixation_Detector, Offline_Blink_Detection, diff --git a/pupil_src/shared_modules/player_methods.py b/pupil_src/shared_modules/player_methods.py index 13c8267a28..24cea03e90 100644 --- a/pupil_src/shared_modules/player_methods.py +++ b/pupil_src/shared_modules/player_methods.py @@ -260,8 +260,7 @@ def transparent_image_overlay(pos, overlay_img, img, alpha): ) try: cv2.addWeighted(overlay_img, alpha, img[roi], 1.0 - alpha, 0, img[roi]) - except: + except (TypeError, cv2.error): logger.debug( "transparent_image_overlay was outside of the world image and was not drawn" ) - pass diff --git a/pupil_src/shared_modules/video_export/plugins/world_video_exporter.py b/pupil_src/shared_modules/video_export/plugins/world_video_exporter.py index 6cb6ba728a..f1809d5338 100644 --- a/pupil_src/shared_modules/video_export/plugins/world_video_exporter.py +++ b/pupil_src/shared_modules/video_export/plugins/world_video_exporter.py @@ -73,6 +73,8 @@ def _precomputed_eye_data_for_range(self, export_range): pre_computed = { "gaze": self.g_pool.gaze_positions, "pupil": self.g_pool.pupil_positions, + "pupil_by_id_0": self.g_pool.pupil_positions_by_id[0], + "pupil_by_id_1": self.g_pool.pupil_positions_by_id[1], "fixations": self.g_pool.fixations, } @@ -116,9 +118,9 @@ def _export_world_video( # Plug-ins from plugin import Plugin_List, import_runtime_plugins from video_capture import EndofVideoError, File_Source + from video_overlay.plugins import Video_Overlay, Eye_Overlay from vis_circle import Vis_Circle from vis_cross import Vis_Cross - from vis_eye_video_overlay import Vis_Eye_Video_Overlay from vis_light_points import Vis_Light_Points from vis_polyline import Vis_Polyline from vis_scan_path import Vis_Scan_Path @@ -139,7 +141,8 @@ def _export_world_video( Vis_Light_Points, Vis_Watermark, Vis_Scan_Path, - Vis_Eye_Video_Overlay, + Eye_Overlay, + Video_Overlay, ], key=lambda x: x.__name__, ) @@ -231,6 +234,10 @@ def _export_world_video( ] g_pool.pupil_positions = pm.Bisector(**pre_computed_eye_data["pupil"]) + g_pool.pupil_positions_by_id = ( + pm.Bisector(**pre_computed_eye_data["pupil_by_id_0"]), + pm.Bisector(**pre_computed_eye_data["pupil_by_id_1"]), + ) g_pool.gaze_positions = pm.Bisector(**pre_computed_eye_data["gaze"]) g_pool.fixations = pm.Affiliator(**pre_computed_eye_data["fixations"]) diff --git a/pupil_src/shared_modules/video_overlay/__init__.py b/pupil_src/shared_modules/video_overlay/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pupil_src/shared_modules/video_overlay/controllers/overlay_manager.py b/pupil_src/shared_modules/video_overlay/controllers/overlay_manager.py new file mode 100644 index 0000000000..fd99b8755a --- /dev/null +++ b/pupil_src/shared_modules/video_overlay/controllers/overlay_manager.py @@ -0,0 +1,54 @@ +import abc +from storage import SingleFileStorage +from video_overlay.models.config import Configuration +from video_overlay.workers.overlay_renderer import OverlayRenderer + + +class OverlayManager(SingleFileStorage): + def __init__(self, rec_dir, plugin): + super().__init__(rec_dir, plugin) + self._overlays = [] + self._load_from_disk() + self._patch_on_cleanup(plugin) + + @property + def _storage_file_name(self): + return "video_overlays.msgpack" + + @property + def _item_class(self): + return Configuration + + def add(self, item): + overlay = OverlayRenderer(item) + self._overlays.append(overlay) + + def delete(self, item): + for overlay in self._overlays.copy(): + if overlay.config is item: + self._overlays.remove(overlay) + + @property + def items(self): + yield from (overlay.config for overlay in self._overlays) + + @property + def overlays(self): + return self._overlays + + @property + def most_recent(self): + return self._overlays[-1] + + def remove_overlay(self, overlay): + self._overlays.remove(overlay) + self.save_to_disk() + + def _patch_on_cleanup(self, plugin): + """Patches cleanup observer to trigger on get_init_dict(). + + Save current settings to disk on get_init_dict() instead of cleanup(). + This ensures that the World Video Exporter loads the most recent settings. + """ + plugin.remove_observer("cleanup", self._on_cleanup) + plugin.add_observer("get_init_dict", self._on_cleanup) diff --git a/pupil_src/shared_modules/video_overlay/models/config.py b/pupil_src/shared_modules/video_overlay/models/config.py new file mode 100644 index 0000000000..106b098c28 --- /dev/null +++ b/pupil_src/shared_modules/video_overlay/models/config.py @@ -0,0 +1,56 @@ +from storage import StorageItem + +from video_overlay.utils.constraints import ( + ConstraintedPosition, + ConstraintedValue, + BooleanConstraint, + InclusiveConstraint, +) + + +class Configuration(StorageItem): + version = 0 + + def __init__( + self, + video_path=None, + origin_x=0, + origin_y=0, + scale=0.6, + alpha=0.8, + hflip=False, + vflip=False, + ): + self.video_path = video_path + self.origin = ConstraintedPosition(origin_x, origin_y) + self.scale = ConstraintedValue(scale, InclusiveConstraint(low=0.2, high=1.0)) + self.alpha = ConstraintedValue(alpha, InclusiveConstraint(low=0.1, high=1.0)) + self.hflip = ConstraintedValue(hflip, BooleanConstraint()) + self.vflip = ConstraintedValue(vflip, BooleanConstraint()) + + @property + def as_tuple(self): + return ( + self.video_path, + self.origin.x.value, + self.origin.y.value, + self.scale.value, + self.alpha.value, + self.hflip.value, + self.vflip.value, + ) + + @staticmethod + def from_tuple(tuple_): + return Configuration(*tuple_) + + def as_dict(self): + return { + "video_path": self.video_path, + "origin_x": self.origin.x.value, + "origin_y": self.origin.y.value, + "scale": self.scale.value, + "alpha": self.alpha.value, + "hflip": self.hflip.value, + "vflip": self.vflip.value, + } diff --git a/pupil_src/shared_modules/video_overlay/plugins/__init__.py b/pupil_src/shared_modules/video_overlay/plugins/__init__.py new file mode 100644 index 0000000000..e34f43ab19 --- /dev/null +++ b/pupil_src/shared_modules/video_overlay/plugins/__init__.py @@ -0,0 +1,2 @@ +from video_overlay.plugins.generic_overlay import Video_Overlay +from video_overlay.plugins.eye_overlay import Eye_Overlay diff --git a/pupil_src/shared_modules/video_overlay/plugins/eye_overlay.py b/pupil_src/shared_modules/video_overlay/plugins/eye_overlay.py new file mode 100644 index 0000000000..c76dc7ccff --- /dev/null +++ b/pupil_src/shared_modules/video_overlay/plugins/eye_overlay.py @@ -0,0 +1,117 @@ +import os +import glob + +import player_methods as pm +from plugin import Plugin +from observable import Observable + +from video_overlay.workers.overlay_renderer import EyeOverlayRenderer +from video_overlay.models.config import Configuration +from video_overlay.ui.management import UIManagementEyes +from video_overlay.utils.constraints import ConstraintedValue, BooleanConstraint + + +class Eye_Overlay(Observable, Plugin): + icon_chr = chr(0xEC02) + icon_font = "pupil_icons" + + def __init__( + self, + g_pool, + scale=0.6, + alpha=0.8, + show_ellipses=True, + eye0_config=None, + eye1_config=None, + ): + super().__init__(g_pool) + eye0_config = eye0_config or {"vflip": True, "origin_x": 210, "origin_y": 60} + eye1_config = eye1_config or {"hflip": True, "origin_x": 10, "origin_y": 60} + + self.current_frame_ts = None + self.show_ellipses = ConstraintedValue(show_ellipses, BooleanConstraint()) + self._scale = scale + self._alpha = alpha + + self.eye0 = self._setup_eye(0, eye0_config) + self.eye1 = self._setup_eye(1, eye1_config) + + def recent_events(self, events): + if "frame" in events: + frame = events["frame"] + self.current_frame_ts = frame.timestamp + for overlay in (self.eye0, self.eye1): + overlay.draw_on_frame(frame) + + @property + def scale(self): + return self._scale + + @scale.setter + def scale(self, val): + self._scale = val + self.eye0.config.scale.value = val + self.eye1.config.scale.value = val + + @property + def alpha(self): + return self._alpha + + @alpha.setter + def alpha(self, val): + self._alpha = val + self.eye0.config.alpha.value = val + self.eye1.config.alpha.value = val + + def init_ui(self): + self.add_menu() + self.menu.label = "Eye Video Overlays" + self.ui = UIManagementEyes(self, self.menu, (self.eye0, self.eye1)) + + def deinit_ui(self): + self.ui.teardown() + self.remove_menu() + + def _setup_eye(self, eye_id, prefilled_config): + video_path = self._video_path_for_eye(eye_id) + prefilled_config["video_path"] = video_path + prefilled_config["scale"] = self.scale + prefilled_config["alpha"] = self.alpha + config = Configuration(**prefilled_config) + overlay = EyeOverlayRenderer( + config, self.show_ellipses, self.make_current_pupil_datum_getter(eye_id) + ) + return overlay + + def _video_path_for_eye(self, eye_id): + rec_dir = self.g_pool.rec_dir + video_file_pattern = "eye{}.*".format(eye_id) + video_path_pattern = os.path.join(rec_dir, video_file_pattern) + try: + video_path_candidates = glob.iglob(video_path_pattern) + return next(video_path_candidates) + except StopIteration: + return "/not/found/eye{}.mp4".format(eye_id) + + def get_init_dict(self): + return { + "scale": self.scale, + "alpha": self.alpha, + "show_ellipses": self.show_ellipses.value, + "eye0_config": self.eye0.config.as_dict(), + "eye1_config": self.eye1.config.as_dict(), + } + + def make_current_pupil_datum_getter(self, eye_id): + def _pupil_getter(): + try: + pupil_data = self.g_pool.pupil_positions_by_id[eye_id] + closest_pupil_idx = pm.find_closest( + pupil_data.data_ts, self.current_frame_ts + ) + current_datum = pupil_data.data[closest_pupil_idx] + return current_datum + except (IndexError, ValueError): + return None + + return _pupil_getter diff --git a/pupil_src/shared_modules/video_overlay/plugins/generic_overlay.py b/pupil_src/shared_modules/video_overlay/plugins/generic_overlay.py new file mode 100644 index 0000000000..f361ec98dc --- /dev/null +++ b/pupil_src/shared_modules/video_overlay/plugins/generic_overlay.py @@ -0,0 +1,69 @@ +import collections +import os + +from observable import Observable +from plugin import Plugin +from video_capture.utils import VIDEO_EXTS + +from video_overlay.models.config import Configuration +from video_overlay.controllers.overlay_manager import OverlayManager +from video_overlay.ui.management import UIManagementGeneric +from video_overlay.ui.interactions import current_mouse_pos + + +class Video_Overlay(Observable, Plugin): + icon_chr = "O" + + def __init__(self, g_pool): + super().__init__(g_pool) + self.manager = OverlayManager(g_pool.rec_dir, self) + + def recent_events(self, events): + if "frame" in events: + frame = events["frame"] + for overlay in self.manager.overlays: + overlay.draw_on_frame(frame) + + def on_drop(self, paths): + multi_drop_offset = 10 + inital_drop = current_mouse_pos( + self.g_pool.main_window, + self.g_pool.camera_render_size, + self.g_pool.capture.frame_size, + ) + valid_paths = [p for p in paths if self.valid_path(p)] + for video_path in valid_paths: + self._add_overlay_to_storage(video_path, inital_drop) + inital_drop = ( + inital_drop[0] + multi_drop_offset, + inital_drop[1] + multi_drop_offset, + ) + # event only consumed if at least one valid file was present + return bool(valid_paths) + + @staticmethod + def valid_path(path): + # splitext()[1] always starts with `.` + ext = os.path.splitext(path)[1][1:] + return ext in VIDEO_EXTS + + def _add_overlay_to_storage(self, video_path, initial_pos=(0, 0)): + config = Configuration( + video_path, origin_x=initial_pos[0], origin_y=initial_pos[1] + ) + self.manager.add(config) + self.manager.save_to_disk() + self._overlay_added_to_storage(self.manager.most_recent) + + def _overlay_added_to_storage(self, overlay): + pass # observed to create menus and draggables + + def init_ui(self): + self.add_menu() + self.menu.label = "Generic Video Overlays" + self.ui = UIManagementGeneric(self, self.menu, self.manager.overlays) + self.ui.add_observer("remove_overlay", self.manager.remove_overlay) + + def deinit_ui(self): + self.ui.teardown() + self.remove_menu() diff --git a/pupil_src/shared_modules/video_overlay/ui/interactions.py b/pupil_src/shared_modules/video_overlay/ui/interactions.py new file mode 100644 index 0000000000..39550e1b29 --- /dev/null +++ b/pupil_src/shared_modules/video_overlay/ui/interactions.py @@ -0,0 +1,58 @@ +from glfw import getHDPIFactor, glfwGetCurrentContext, glfwGetCursorPos, GLFW_PRESS +from methods import normalize, denormalize + + +class Draggable: + __slots__ = ("overlay", "drag_offset") + + def __init__(self, overlay): + self.drag_offset = None + self.overlay = overlay + + def on_click(self, pos, button, action): + if not self.overlay.valid_video_loaded: + return False # click event has not been consumed + + click_engaged = action == GLFW_PRESS + if click_engaged and self._in_bounds(pos): + self.drag_offset = self._calculate_offset(pos) + return True + self.drag_offset = None + return False + + def on_pos(self, pos): + if self.overlay.valid_video_loaded and self.drag_offset: + self.overlay.config.origin.x.value = int(pos[0] + self.drag_offset[0]) + self.overlay.config.origin.y.value = int(pos[1] + self.drag_offset[1]) + + def _in_bounds(self, pos): + curr_x, curr_y = pos + origin = self.overlay.config.origin + width, height = self._effective_overlay_frame_size() + x_in_bounds = origin.x.value < curr_x < origin.x.value + width + y_in_bounds = origin.y.value < curr_y < origin.y.value + height + return x_in_bounds and y_in_bounds + + def _calculate_offset(self, pos): + curr_x, curr_y = pos + origin = self.overlay.config.origin + x_offset = origin.x.value - curr_x + y_offset = origin.y.value - curr_y + return (x_offset, y_offset) + + def _effective_overlay_frame_size(self): + overlay_scale = self.overlay.config.scale.value + overlay_width, overlay_height = self.overlay.video.source.frame_size + overlay_width = round(overlay_width * overlay_scale) + overlay_height = round(overlay_height * overlay_scale) + return overlay_width, overlay_height + + +def current_mouse_pos(window, camera_render_size, frame_size): + hdpi_fac = getHDPIFactor(window) + x, y = glfwGetCursorPos(glfwGetCurrentContext()) + pos = x * hdpi_fac, y * hdpi_fac + pos = normalize(pos, camera_render_size) + # Position in img pixels + pos = denormalize(pos, frame_size) + return (int(pos[0]), int(pos[1])) diff --git a/pupil_src/shared_modules/video_overlay/ui/management.py b/pupil_src/shared_modules/video_overlay/ui/management.py new file mode 100644 index 0000000000..5a9e6a55b2 --- /dev/null +++ b/pupil_src/shared_modules/video_overlay/ui/management.py @@ -0,0 +1,123 @@ +import abc +import weakref +from collections import OrderedDict +from pyglui import ui + +from observable import Observable + +from video_overlay.ui.interactions import Draggable +from video_overlay.ui.menu import GenericOverlayMenuRenderer, EyesOverlayMenuRenderer + + +class UIManagement(Observable, abc.ABC): + def __init__(self, plugin, parent_menu, existing_overlays): + self._parent_menu = parent_menu + self._menu_renderers = {} + # Insert order is important for correct dragging behavior + self._draggables = OrderedDict() + + self._add_menu_with_general_elements() + self._add_menu_for_existing_overlays(existing_overlays) + self._add_draggable_for_existing_overlays(existing_overlays) + + plugin.add_observer("on_click", self._on_click) + plugin.add_observer("on_pos", self._on_pos) + + @abc.abstractmethod + def _add_menu_with_general_elements(self): + raise NotImplementedError + + @abc.abstractmethod + def _add_overlay_menu(self, overlay): + raise NotImplementedError + + def teardown(self): + del self._parent_menu[:] + del self._parent_menu + self._draggables.clear() + self._menu_renderers.clear() + + def _on_click(self, *args, **kwargs): + # iterate over draggables in reverse order since last element is drawn on top + any(d.on_click(*args, **kwargs) for d in reversed(self._draggables.values())) + + def _on_pos(self, *args, **kwargs): + any(d.on_pos(*args, **kwargs) for d in reversed(self._draggables.values())) + + def _add_menu_for_existing_overlays(self, existing_overlays): + for overlay in existing_overlays: + self._add_overlay_menu(overlay) + + def _add_draggable_for_existing_overlays(self, existing_overlays): + for overlay in existing_overlays: + self._add_overlay_draggable(overlay) + + def _add_overlay_draggable(self, overlay): + draggable = Draggable(overlay) + self._draggables[overlay] = draggable + + +class UIManagementGeneric(UIManagement): + def __init__(self, plugin, parent_menu, existing_overlays): + super().__init__(plugin, parent_menu, existing_overlays) + plugin.add_observer("_overlay_added_to_storage", self._add_overlay_menu) + plugin.add_observer("_overlay_added_to_storage", self._add_overlay_draggable) + + def _add_menu_with_general_elements(self): + self._parent_menu.append( + ui.Info_Text( + "This plugin is able to overlay videos with synchronized timestamps." + ) + ) + self._parent_menu.append( + ui.Info_Text( + "Drag and drop such videos onto the " + "main Player window in order to load them" + ) + ) + self._parent_menu.append(ui.Separator()) + + def _add_overlay_menu(self, overlay): + renderer = GenericOverlayMenuRenderer(overlay) + renderer.add_observer("remove_button_clicked", self.remove_overlay) + self._menu_renderers[overlay] = renderer + self._parent_menu.append(renderer.menu) + + def remove_overlay(self, overlay): + renderer = self._menu_renderers[overlay] + renderer.remove_all_observers("remove_button_clicked") + del self._menu_renderers[overlay] + del self._draggables[overlay] + self._parent_menu.remove(renderer.menu) + + +class UIManagementEyes(UIManagement): + def __init__(self, plugin, parent_menu, existing_overlays): + self.plugin = weakref.ref(plugin) + super().__init__(plugin, parent_menu, existing_overlays) + + def _add_menu_with_general_elements(self): + self._parent_menu.append( + ui.Info_Text( + "Show the eye video overlaid on top of the world video. " + "Eye 0 is usually the right eye." + ) + ) + self._parent_menu.append( + ui.Slider( + "alpha", self.plugin(), min=0.1, step=0.05, max=1.0, label="Opacity" + ) + ) + self._parent_menu.append( + ui.Slider( + "scale", self.plugin(), min=0.2, step=0.05, max=1.0, label="Video Scale" + ) + ) + self._parent_menu.append( + ui.Switch("value", self.plugin().show_ellipses, label="Visualize Ellipses") + ) + + def _add_overlay_menu(self, overlay): + renderer = EyesOverlayMenuRenderer(overlay) + self._menu_renderers[overlay] = renderer + self._parent_menu.append(renderer.menu) diff --git a/pupil_src/shared_modules/video_overlay/ui/menu.py b/pupil_src/shared_modules/video_overlay/ui/menu.py new file mode 100644 index 0000000000..996f897cd6 --- /dev/null +++ b/pupil_src/shared_modules/video_overlay/ui/menu.py @@ -0,0 +1,108 @@ +import abc +import os +import weakref + +from pyglui import ui + +from observable import Observable + + +def make_scale_slider(config): + return ui.Slider( + "value", + config.scale, + label="Scale", + min=config.scale.constraint.low, + max=config.scale.constraint.high, + step=0.05, + ) + + +def make_alpha_slider(config): + return ui.Slider( + "value", + config.alpha, + label="Opacity", + min=config.alpha.constraint.low, + max=config.alpha.constraint.high, + step=0.05, + ) + + +def make_hflip_switch(config): + return ui.Switch("value", config.hflip, label="Flip horizontally") + + +def make_vflip_switch(config): + return ui.Switch("value", config.vflip, label="Flip vertically") + + +class OverlayMenuRenderer(Observable, abc.ABC): + def __init__(self, overlay): + self.overlay = weakref.ref(overlay) + video_basename = os.path.basename(self.overlay().config.video_path) + self.menu = ui.Growing_Menu(video_basename) + self.menu.collapsed = True + self.update_menu() + + def update_menu(self): + if self.overlay().valid_video_loaded: + self.menu[:] = self._generic_overlay_elements() + else: + self.menu[:] = self._not_valid_video_elements() + self._append_remove_button() + + def _append_remove_button(self): + pass # do not show remove button by default + + @abc.abstractmethod + def _generic_overlay_elements(self): + raise NotImplementedError + + @abc.abstractmethod + def _not_valid_video_elements(self): + raise NotImplementedError + + +class GenericOverlayMenuRenderer(OverlayMenuRenderer): + def _append_remove_button(self): + self.menu.append(ui.Button("Remove overlay", self._remove)) + + def _remove(self): + self.remove_button_clicked(self.overlay()) + + def remove_button_clicked(self, overlay): + pass # observable + + def _generic_overlay_elements(self): + config = self.overlay().config + return ( + ui.Info_Text("Loaded video: {}".format(config.video_path)), + make_scale_slider(config), + make_alpha_slider(config), + make_hflip_switch(config), + make_vflip_switch(config), + ) + + def _not_valid_video_elements(self): + video_path = self.overlay().config.video_path + return ( + ui.Info_Text("No valid overlay video found at {}".format(video_path)), + ui.Info_Text( + "Valid overlay videos conform to the Pupil data format and " + "their timestamps are in sync with the opened recording." + ), + ) + + +class EyesOverlayMenuRenderer(OverlayMenuRenderer): + def _generic_overlay_elements(self): + config = self.overlay().config + return (make_hflip_switch(config), make_vflip_switch(config)) + + def _not_valid_video_elements(self): + video_path = self.overlay().config.video_path + video_name = os.path.basename(video_path) + return ( + ui.Info_Text("{} was not recorded or cannot be found.".format(video_name)), + ) diff --git a/pupil_src/shared_modules/video_overlay/utils/constraints.py b/pupil_src/shared_modules/video_overlay/utils/constraints.py new file mode 100644 index 0000000000..ecad3fce04 --- /dev/null +++ b/pupil_src/shared_modules/video_overlay/utils/constraints.py @@ -0,0 +1,69 @@ +import abc + +INF = float("inf") + + +class BaseConstraint(metaclass=abc.ABCMeta): + @abc.abstractmethod + def apply_to(self, value): + raise NotImplementedError + + +class NoConstraint(BaseConstraint): + def apply_to(self, value): + return value + + +class InclusiveConstraint(BaseConstraint): + __slots__ = ("low", "high") + + def __init__(self, *, low=-INF, high=INF): + self.low = low + self.high = high + + def apply_to(self, value): + return min(max(self.low, value), self.high) + + +class BooleanConstraint(BaseConstraint): + def apply_to(self, value): + return bool(value) + + +class ConstraintedValue: + __slots__ = ("_val", "_constraint") + + def __init__(self, value, constraint=NoConstraint()): + self._constraint = constraint + self._val = value + + @property + def value(self): + return self.constraint.apply_to(self._val) + + @value.setter + def value(self, new_val): + self._val = new_val + + @property + def constraint(self): + return self._constraint + + @constraint.setter + def constraint(self, new_constraint): + self._constraint = new_constraint + + @constraint.deleter + def constraint(self): + self.constraint = NoConstraint() + + +class ConstraintedPosition: + __slots__ = ("x", "y") + + def __init__(self, x, y): + self.x = ConstraintedValue(x) + self.y = ConstraintedValue(y) + + def __str__(self): + return "(x={}, y={})".format(self.x.value, self.y.value) diff --git a/pupil_src/shared_modules/video_overlay/utils/image_manipulation.py b/pupil_src/shared_modules/video_overlay/utils/image_manipulation.py new file mode 100644 index 0000000000..e8a7d162cd --- /dev/null +++ b/pupil_src/shared_modules/video_overlay/utils/image_manipulation.py @@ -0,0 +1,90 @@ +import abc + +import cv2 +import numpy as np + + +class ImageManipulator(metaclass=abc.ABCMeta): + @abc.abstractmethod + def apply_to(self, image, parameter): + raise NotImplementedError + + +class ScaleTransform(ImageManipulator): + def apply_to(self, image, parameter): + """parameter: scale factor as float""" + return cv2.resize(image, (0, 0), fx=parameter, fy=parameter) + + +class HorizontalFlip(ImageManipulator): + def apply_to(self, image, parameter): + """parameter: boolean indicating if image should be flipped""" + return np.fliplr(image) if parameter else image + + +class VerticalFlip(ImageManipulator): + def apply_to(self, image, parameter): + """parameter: boolean indicating if image should be flipped""" + return np.flipud(image) if parameter else image + + +class PupilRenderer(ImageManipulator): + __slots__ = "pupil_getter" + + def __init__(self, pupil_getter): + self.pupil_getter = pupil_getter + + def apply_to(self, image, parameter): + """parameter: boolean indicating if pupil should be rendered""" + if parameter: + pupil_position = self.pupil_getter() + if pupil_position: + self.render_pupil(image, pupil_position) + return image + + def render_pupil(self, image, pupil_position): + el = pupil_position["ellipse"] + conf = int( + pupil_position.get( + "model_confidence", pupil_position.get("confidence", 0.1) + ) + * 255 + ) + el_points = self.get_ellipse_points((el["center"], el["axes"], el["angle"])) + cv2.polylines( + image, + [np.asarray(el_points, dtype="i")], + True, + (0, 0, 255, conf), + thickness=1, + ) + cv2.circle( + image, + (int(el["center"][0]), int(el["center"][1])), + 5, + (0, 0, 255, conf), + thickness=-1, + ) + + @staticmethod + def get_ellipse_points(e, num_pts=10): + c1 = e[0][0] + c2 = e[0][1] + a = e[1][0] + b = e[1][1] + angle = e[2] + + steps = np.linspace(0, 2 * np.pi, num=num_pts, endpoint=False) + rot = cv2.getRotationMatrix2D((0, 0), -angle, 1) + + pts1 = a / 2.0 * np.cos(steps) + pts2 = b / 2.0 * np.sin(steps) + pts = np.column_stack((pts1, pts2, np.ones(pts1.shape[0]))) + + pts_rot = np.matmul(rot, pts.T) + pts_rot = pts_rot.T + + pts_rot[:, 0] += c1 + pts_rot[:, 1] += c2 + + return pts_rot diff --git a/pupil_src/shared_modules/video_overlay/workers/frame_fetcher.py b/pupil_src/shared_modules/video_overlay/workers/frame_fetcher.py new file mode 100644 index 0000000000..bf2acdb0a9 --- /dev/null +++ b/pupil_src/shared_modules/video_overlay/workers/frame_fetcher.py @@ -0,0 +1,44 @@ +import logging + +import player_methods as pm + +from video_capture import File_Source, EndofVideoError + +logger = logging.getLogger(__name__) + + +class _Empty: + """Replacement for actual g_pool object""" + + +class FrameFetcher: + __slots__ = ("source", "current_frame") + + def __init__(self, video_path): + try: + self.source = File_Source(_Empty(), source_path=video_path, timing=None) + except AssertionError as err: + raise FileNotFoundError(video_path) from err + self.current_frame = self.source.get_frame() + + def initialised(self): + return self.source.initialised + + def closest_frame_to_ts(self, ts): + closest_idx = pm.find_closest(self.source.timestamps, ts) + return self.frame_for_idx(closest_idx) + + def frame_for_idx(self, requested_frame_idx): + if requested_frame_idx != self.current_frame.index: + if requested_frame_idx == self.source.get_frame_index() + 2: + # if we just need to seek by one frame, + # its faster to just read one and and throw it away. + self.source.get_frame() + if requested_frame_idx != self.source.get_frame_index() + 1: + self.source.seek_to_frame(int(requested_frame_idx)) + + try: + self.current_frame = self.source.get_frame() + except EndofVideoError: + logger.info("End of video {}.".format(self.source.source_path)) + return self.current_frame diff --git a/pupil_src/shared_modules/video_overlay/workers/overlay_renderer.py b/pupil_src/shared_modules/video_overlay/workers/overlay_renderer.py new file mode 100644 index 0000000000..e52831cb14 --- /dev/null +++ b/pupil_src/shared_modules/video_overlay/workers/overlay_renderer.py @@ -0,0 +1,65 @@ +import logging +from collections import OrderedDict + +import player_methods as pm +from observable import Observable + +import video_overlay.utils.image_manipulation as IM +from video_overlay.utils.constraints import InclusiveConstraint +from video_overlay.models.config import Configuration +from video_overlay.workers.frame_fetcher import FrameFetcher + +logger = logging.getLogger(__name__) + + +class OverlayRenderer: + def __init__(self, config): + self.config = config + self.attempt_to_load_video() + self.pipeline = self.setup_pipeline() + + def attempt_to_load_video(self): + try: + self.video = FrameFetcher(self.config.video_path) + self.valid_video_loaded = True + except FileNotFoundError: + logger.debug("Could not load overlay: {}".format(self.config.video_path)) + self.valid_video_loaded = False + return self.valid_video_loaded + + def setup_pipeline(self): + return [ + (self.config.scale, IM.ScaleTransform()), + (self.config.hflip, IM.HorizontalFlip()), + (self.config.vflip, IM.VerticalFlip()), + ] + + def draw_on_frame(self, target_frame): + if not self.valid_video_loaded: + return + overlay_frame = self.video.closest_frame_to_ts(target_frame.timestamp) + overlay_image = overlay_frame.img + for param, manipulation in self.pipeline: + overlay_image = manipulation.apply_to(overlay_image, param.value) + + self._adjust_origin_constraint(target_frame.img, overlay_image) + self._render_overlay(target_frame.img, overlay_image) + + def _adjust_origin_constraint(self, target_image, overlay_image): + max_x = target_image.shape[1] - overlay_image.shape[1] + max_y = target_image.shape[0] - overlay_image.shape[0] + self.config.origin.x.constraint = InclusiveConstraint(low=0, high=max_x) + self.config.origin.y.constraint = InclusiveConstraint(low=0, high=max_y) + + def _render_overlay(self, target_image, overlay_image): + overlay_origin = (self.config.origin.x.value, self.config.origin.y.value) + pm.transparent_image_overlay( + overlay_origin, overlay_image, target_image, self.config.alpha.value + ) + + +class EyeOverlayRenderer(OverlayRenderer): + def __init__(self, config, should_render_pupil_data, pupil_getter): + super().__init__(config) + pupil_renderer = (should_render_pupil_data, IM.PupilRenderer(pupil_getter)) + self.pipeline.insert(0, pupil_renderer)