diff --git a/deployment/deploy_capture/bundle.spec b/deployment/deploy_capture/bundle.spec index a8bf2433e1..494903681f 100644 --- a/deployment/deploy_capture/bundle.spec +++ b/deployment/deploy_capture/bundle.spec @@ -10,7 +10,7 @@ import sys import numpy import pkg_resources -from PyInstaller.utils.hooks import collect_submodules +from PyInstaller.utils.hooks import collect_submodules, collect_data_files hidden_imports = [] hidden_imports += collect_submodules("av") @@ -40,6 +40,8 @@ if not glfw_path.exists(): glfw_path = pathlib.Path(pkg_resources.resource_filename("glfw", glfw_name)) glfw_binaries = [(glfw_path.name, str(glfw_path), "BINARY")] +data_files_pye3d = collect_data_files("pye3d") + if platform.system() == "Darwin": sys.path.append(".") from version import pupil_version @@ -52,6 +54,7 @@ if platform.system() == "Darwin": hookspath=None, runtime_hooks=None, excludes=["matplotlib"], + datas=data_files_pye3d, ) pyz = PYZ(a.pure) exe = EXE( @@ -104,6 +107,7 @@ elif platform.system() == "Linux": hookspath=None, runtime_hooks=None, excludes=["matplotlib"], + datas=data_files_pye3d, ) pyz = PYZ(a.pure) @@ -172,7 +176,7 @@ elif platform.system() == "Windows": ["../../pupil_src/main.py"], pathex=["../../pupil_src/shared_modules/", str(external_libs_path)], binaries=None, - datas=None, + datas=data_files_pye3d, hiddenimports=hidden_imports, hookspath=None, runtime_hooks=None, diff --git a/deployment/deploy_player/bundle.spec b/deployment/deploy_player/bundle.spec index 773b9e185a..9e4b2cd912 100644 --- a/deployment/deploy_player/bundle.spec +++ b/deployment/deploy_player/bundle.spec @@ -10,7 +10,7 @@ import sys import numpy import pkg_resources -from PyInstaller.utils.hooks import collect_submodules +from PyInstaller.utils.hooks import collect_submodules, collect_data_files hidden_imports = [] hidden_imports += collect_submodules("av") @@ -40,6 +40,8 @@ if not glfw_path.exists(): glfw_path = pathlib.Path(pkg_resources.resource_filename("glfw", glfw_name)) glfw_binaries = [(glfw_path.name, str(glfw_path), "BINARY")] +data_files_pye3d = collect_data_files("pye3d") + if platform.system() == "Darwin": sys.path.append(".") from version import pupil_version @@ -53,6 +55,7 @@ if platform.system() == "Darwin": hookspath=None, runtime_hooks=None, excludes=["matplotlib"], + datas=data_files_pye3d, ) pyz = PYZ(a.pure) @@ -106,6 +109,7 @@ elif platform.system() == "Linux": hookspath=None, runtime_hooks=None, excludes=["matplotlib"], + datas=data_files_pye3d, ) pyz = PYZ(a.pure) @@ -177,6 +181,7 @@ elif platform.system() == "Windows": hookspath=None, runtime_hooks=None, excludes=["matplotlib"], + datas=data_files_pye3d, ) pyz = PYZ(a.pure) diff --git a/deployment/deploy_service/bundle.spec b/deployment/deploy_service/bundle.spec index f9a3a3e8bf..f91fb8bd2b 100644 --- a/deployment/deploy_service/bundle.spec +++ b/deployment/deploy_service/bundle.spec @@ -10,7 +10,7 @@ import sys import numpy import pkg_resources -from PyInstaller.utils.hooks import collect_submodules +from PyInstaller.utils.hooks import collect_submodules, collect_data_files hidden_imports = [] hidden_imports += collect_submodules("av") @@ -30,6 +30,8 @@ if not glfw_path.exists(): glfw_path = pathlib.Path(pkg_resources.resource_filename("glfw", glfw_name)) glfw_binaries = [(glfw_path.name, str(glfw_path), "BINARY")] +data_files_pye3d = collect_data_files("pye3d") + if platform.system() == "Darwin": sys.path.append(".") from version import pupil_version @@ -43,6 +45,7 @@ if platform.system() == "Darwin": hookspath=None, runtime_hooks=None, excludes=["matplotlib"], + datas=data_files_pye3d, ) pyz = PYZ(a.pure) exe = EXE( @@ -89,6 +92,7 @@ elif platform.system() == "Linux": hookspath=None, runtime_hooks=None, excludes=["matplotlib"], + datas=data_files_pye3d, ) pyz = PYZ(a.pure) @@ -151,7 +155,7 @@ elif platform.system() == "Windows": ["../../pupil_src/main.py"], pathex=["../../pupil_src/shared_modules/", str(external_libs_path)], binaries=None, - datas=None, + datas=data_files_pye3d, hiddenimports=hidden_imports, hookspath=None, runtime_hooks=None, diff --git a/docs/dependencies-macos.md b/docs/dependencies-macos.md index 14ae3a2385..91956cdaef 100644 --- a/docs/dependencies-macos.md +++ b/docs/dependencies-macos.md @@ -46,8 +46,6 @@ brew install portaudio # tbb is included by default with https://github.com/Homebrew/homebrew-core/pull/20101 brew install opencv brew install glew -# dependencies for 2d_3d c++ detector -brew install ceres-solver ``` ## libuvc diff --git a/docs/dependencies-ubuntu17.md b/docs/dependencies-ubuntu17.md index 2375c9b66a..7cfccb1358 100644 --- a/docs/dependencies-ubuntu17.md +++ b/docs/dependencies-ubuntu17.md @@ -119,35 +119,7 @@ sudo udevadm trigger ## 3D Eye Model Dependencies ```sh -sudo apt install -y libgoogle-glog-dev libatlas-base-dev libeigen3-dev -``` - -## Ceres - -You will need to build Ceres from source: -```sh -sudo add-apt-repository ppa:bzindovic/suitesparse-bugfix-1319687 -sudo apt-get update -sudo apt-get install libsuitesparse-dev -``` - -If `add-apt-repository` is not found, install the following package first: -```sh -sudo apt-get install software-properties-common -``` - -The build and install the Ceres solver: -```sh -git clone https://ceres-solver.googlesource.com/ceres-solver -cd ceres-solver -git checkout 1.14.0 -mkdir build && cd build -cmake .. -DBUILD_SHARED_LIBS=ON -make -j3 -make test -sudo make install -sudo sh -c 'echo "/usr/local/lib" > /etc/ld.so.conf.d/ceres.conf' -sudo ldconfig +sudo apt install -y libeigen3-dev ``` ### Install Python Libraries diff --git a/docs/dependencies-ubuntu18.md b/docs/dependencies-ubuntu18.md index 00e2fc79e6..281dc64f5a 100644 --- a/docs/dependencies-ubuntu18.md +++ b/docs/dependencies-ubuntu18.md @@ -12,12 +12,8 @@ sudo apt install -y pkg-config git cmake build-essential nasm wget python3-setup # ffmpeg >= 3.2 sudo apt install -y libavformat-dev libavcodec-dev libavdevice-dev libavutil-dev libswscale-dev libavresample-dev ffmpeg x264 x265 libportaudio2 portaudio19-dev -# OpenCV >= 3 -sudo apt install -y python3-opencv libopencv-dev - -# 3D Eye model dependencies -sudo apt install -y libgoogle-glog-dev libatlas-base-dev libeigen3-dev -sudo apt install -y libceres-dev +# OpenCV >= 3 + Eigen +sudo apt install -y python3-opencv libopencv-dev libeigen3-dev ``` ## Turbojpeg diff --git a/pupil_src/launchables/eye.py b/pupil_src/launchables/eye.py index 13f3216ccb..6ec1c9b451 100644 --- a/pupil_src/launchables/eye.py +++ b/pupil_src/launchables/eye.py @@ -65,6 +65,7 @@ def eye( hide_ui=False, debug=False, pub_socket_hwm=None, + parent_application="capture", ): """reads eye video and detects the pupil. @@ -190,7 +191,7 @@ def interrupt_handler(sig, frame): g_pool.debug = debug g_pool.user_dir = user_dir g_pool.version = version - g_pool.app = "capture" + g_pool.app = parent_application g_pool.eye_id = eye_id g_pool.process = f"eye{eye_id}" g_pool.timebase = timebase @@ -223,7 +224,7 @@ def load_runtime_pupil_detection_plugins(): continue yield plugin - default_2d, default_3d, available_detectors = available_detector_plugins() + available_detectors = available_detector_plugins() runtime_detectors = list(load_runtime_pupil_detection_plugins()) plugins = ( manager_classes @@ -253,13 +254,10 @@ def load_runtime_pupil_detection_plugins(): # TODO: extend with plugins (default_capture_name, default_capture_settings), ("UVC_Manager", {}), - # Detectors needs to be loaded first to set `g_pool.pupil_detector` - (default_2d.__name__, {}), - (default_3d.__name__, {}), + *[(p.__name__, {}) for p in available_detectors], ("NDSI_Manager", {}), ("HMD_Streaming_Manager", {}), ("File_Manager", {}), - ("PupilDetectorManager", {}), ("Roi", {}), ] @@ -814,10 +812,10 @@ def window_should_update(): plugin.alive = False g_pool.plugins.clean() - glfw.destroy_window(main_window) - g_pool.gui.terminate() - glfw.terminate() - logger.info("Process shutting down.") + glfw.destroy_window(main_window) + g_pool.gui.terminate() + glfw.terminate() + logger.info("Process shutting down.") def eye_profiled( @@ -833,6 +831,7 @@ def eye_profiled( hide_ui=False, debug=False, pub_socket_hwm=None, + parent_application="capture", ): import cProfile import subprocess @@ -840,7 +839,23 @@ def eye_profiled( from .eye import eye cProfile.runctx( - "eye(timebase, is_alive_flag,ipc_pub_url,ipc_sub_url,ipc_push_url, user_dir, version, eye_id, overwrite_cap_settings, hide_ui, debug)", + ( + "eye(" + "timebase, " + "is_alive_flag, " + "ipc_pub_url, " + "ipc_sub_url, " + "ipc_push_url, " + "user_dir, " + "version, " + "eye_id, " + "overwrite_cap_settings, " + "hide_ui, " + "debug, " + "pub_socket_hwm, " + "parent_application, " + ")" + ), { "timebase": timebase, "is_alive_flag": is_alive_flag, @@ -854,6 +869,7 @@ def eye_profiled( "hide_ui": hide_ui, "debug": debug, "pub_socket_hwm": pub_socket_hwm, + "parent_application": parent_application, }, locals(), "eye{}.pstats".format(eye_id), diff --git a/pupil_src/launchables/world.py b/pupil_src/launchables/world.py index a037b580db..3008f1afc5 100644 --- a/pupil_src/launchables/world.py +++ b/pupil_src/launchables/world.py @@ -36,6 +36,7 @@ def world( Reacts to notifications: ``eye_process.started`` ``start_plugin`` + ``should_stop`` Emits notifications: ``eye_process.should_start`` @@ -516,6 +517,8 @@ def handle_notifications(noti): ) elif subject == "world_process.adapt_window_size": set_window_size() + elif subject == "world_process.should_stop": + glfw.set_window_should_close(main_window, True) width, height = session_settings.get( "window_size", (1280 + icon_bar_width, 720) diff --git a/pupil_src/main.py b/pupil_src/main.py index 0266c1ce53..64da537b14 100644 --- a/pupil_src/main.py +++ b/pupil_src/main.py @@ -310,6 +310,7 @@ def log_loop(ipc_sub_url, log_level_debug): parsed_args.hide_ui, parsed_args.debug, n.get("pub_socket_hwm"), + parsed_args.app, # parent_application ), ).start() elif "notify.player_process.should_start" in topic: @@ -392,6 +393,14 @@ def log_loop(ipc_sub_url, log_level_debug): "doc": launcher.__doc__, } ) + elif "notify.launcher_process.should_stop" in topic: + if parsed_args.app == "capture": + cmd_push.notify({"subject": "world_process.should_stop"}) + elif parsed_args.app == "service": + cmd_push.notify({"subject": "service_process.should_stop"}) + elif parsed_args.app == "player": + cmd_push.notify({"subject": "player_process.should_stop"}) + else: if not active_children(): break diff --git a/pupil_src/shared_modules/blink_detection.py b/pupil_src/shared_modules/blink_detection.py index ec85c3e6fa..7469ee08f0 100644 --- a/pupil_src/shared_modules/blink_detection.py +++ b/pupil_src/shared_modules/blink_detection.py @@ -160,9 +160,6 @@ def recent_events(self, events={}): blink_type = "offset" confidence = min(abs(filter_response), 1.0) # clamp conf. value at 1. - logger.debug( - "Blink {} detected with confidence {:0.3f}".format(blink_type, confidence) - ) # Add info to events blink_entry = { "topic": "blinks", diff --git a/pupil_src/shared_modules/calibration_choreography/controller/gui_window.py b/pupil_src/shared_modules/calibration_choreography/controller/gui_window.py index 363947f330..159038da83 100644 --- a/pupil_src/shared_modules/calibration_choreography/controller/gui_window.py +++ b/pupil_src/shared_modules/calibration_choreography/controller/gui_window.py @@ -128,10 +128,12 @@ def close(self): @contextlib.contextmanager def drawing_context(self): if self.__gl_handle is None: + yield None return if glfw.window_should_close(self.__gl_handle): self.close() + yield None return with self._switch_to_current_context(): diff --git a/pupil_src/shared_modules/calibration_choreography/controller/marker_window_controller.py b/pupil_src/shared_modules/calibration_choreography/controller/marker_window_controller.py index a37aeb62c1..04cfcca783 100644 --- a/pupil_src/shared_modules/calibration_choreography/controller/marker_window_controller.py +++ b/pupil_src/shared_modules/calibration_choreography/controller/marker_window_controller.py @@ -279,10 +279,11 @@ def draw_window(self): return with self.__window.drawing_context() as gl_context: - self.__draw_circle_marker( - position=marker_position, is_valid=is_valid, alpha=marker_alpha - ) - self.__draw_status_text(clicks_needed=clicks_needed) + if gl_context: + self.__draw_circle_marker( + position=marker_position, is_valid=is_valid, alpha=marker_alpha + ) + self.__draw_status_text(clicks_needed=clicks_needed) else: raise UnhandledMarkerWindowStateError(self.__state) diff --git a/pupil_src/shared_modules/file_methods.py b/pupil_src/shared_modules/file_methods.py index 81b3c4800c..af2934908a 100644 --- a/pupil_src/shared_modules/file_methods.py +++ b/pupil_src/shared_modules/file_methods.py @@ -23,9 +23,10 @@ import msgpack import numpy as np + assert ( - msgpack.version[1] == 5 -), "msgpack out of date, please upgrade to version (0, 5, 6 ) or later." + msgpack.version[0] == 1 +), "msgpack out of date, please upgrade to version (1, 0, 0)" logger = logging.getLogger(__name__) @@ -77,7 +78,7 @@ def load_object(file_path, allow_legacy=True): with file_path.open("rb") as fh: try: gc.disable() # speeds deserialization up. - data = msgpack.unpack(fh, raw=False) + data = msgpack.unpack(fh, strict_map_key=False) except Exception as e: if not allow_legacy: raise e @@ -118,7 +119,9 @@ def __init__(self, directory=""): def __enter__(self): self.file_handle = open(self.file_loc, "rb") - self.unpacker = msgpack.Unpacker(self.file_handle, raw=False, use_list=False) + self.unpacker = msgpack.Unpacker( + self.file_handle, use_list=False, strict_map_key=False + ) self.num_key_value_pairs = self.unpacker.read_map_header() self._skipped = True return self @@ -143,7 +146,9 @@ def load_pldata_file(directory, topic): topics = collections.deque() data_ts = np.load(ts_file) with open(msgpack_file, "rb") as fh: - for topic, payload in msgpack.Unpacker(fh, raw=False, use_list=False): + for topic, payload in msgpack.Unpacker( + fh, use_list=False, strict_map_key=False + ): data.append(Serialized_Dict(msgpack_bytes=payload)) topics.append(topic) except FileNotFoundError: @@ -235,10 +240,10 @@ def _deser(self): if not self._data: self._data = msgpack.unpackb( self._ser_data, - raw=False, use_list=False, object_hook=self.unpacking_object_hook, ext_hook=self.unpacking_ext_hook, + strict_map_key=False, ) self._cache_ref.pop(0).purge_cache() self._cache_ref.append(self) @@ -360,7 +365,6 @@ def unpacking_ext_hook(self, code, data): return msgpack.unpackb( self._ser_data, - raw=False, use_list=False, ext_hook=unpacking_ext_hook, ) diff --git a/pupil_src/shared_modules/gaze_mapping/gazer_3d/gazer_headset.py b/pupil_src/shared_modules/gaze_mapping/gazer_3d/gazer_headset.py index d68a1b8dd7..788eaab67d 100644 --- a/pupil_src/shared_modules/gaze_mapping/gazer_3d/gazer_headset.py +++ b/pupil_src/shared_modules/gaze_mapping/gazer_3d/gazer_headset.py @@ -293,8 +293,8 @@ def _predict_single(self, x): nearest_intersection_point *= -1.0 g = { - "eye_centers_3d": {0: s0_center.tolist(), 1: s1_center.tolist()}, - "gaze_normals_3d": {0: s0_normal.tolist(), 1: s1_normal.tolist()}, + "eye_centers_3d": {"0": s0_center.tolist(), "1": s1_center.tolist()}, + "gaze_normals_3d": {"0": s0_normal.tolist(), "1": s1_normal.tolist()}, "gaze_point_3d": nearest_intersection_point.tolist(), } diff --git a/pupil_src/shared_modules/pupil_detector_plugins/__init__.py b/pupil_src/shared_modules/pupil_detector_plugins/__init__.py index 0635d8a7db..14e71a5b62 100644 --- a/pupil_src/shared_modules/pupil_detector_plugins/__init__.py +++ b/pupil_src/shared_modules/pupil_detector_plugins/__init__.py @@ -12,24 +12,32 @@ import traceback import typing as T +import pupil_detectors +from version_utils import parse_version + from .detector_2d_plugin import Detector2DPlugin -from .detector_3d_plugin import Detector3DPlugin -from .detector_base_plugin import PupilDetectorPlugin, EVENT_KEY +from .detector_base_plugin import EVENT_KEY, PupilDetectorPlugin logger = logging.getLogger(__name__) +required_version_str = "2.0.0" +if parse_version(pupil_detectors.__version__) < parse_version(required_version_str): + msg = ( + f"This version of Pupil requires pupil_detectors >= {required_version_str}." + f" You are running with pupil_detectors == {pupil_detectors.__version__}." + f" Please upgrade to a newer version!" + ) + logger.error(msg) + raise RuntimeError(msg) + -def available_detector_plugins() -> T.Tuple[ - PupilDetectorPlugin, PupilDetectorPlugin, T.List[PupilDetectorPlugin] -]: - """Load and list available plugins, including default +def available_detector_plugins() -> T.List[T.Type[PupilDetectorPlugin]]: + """Load and list available plugins - Returns tuple of default2D, default3D, and list of all detectors. + Returns list of all detectors. """ - all_plugins = [Detector2DPlugin, Detector3DPlugin] - default2D = Detector2DPlugin - default3D = Detector3DPlugin + all_plugins: T.List[T.Type[PupilDetectorPlugin]] = [Detector2DPlugin] try: from .pye3d_plugin import Pye3DPlugin @@ -39,6 +47,5 @@ def available_detector_plugins() -> T.Tuple[ else: logger.info("Using refraction corrected 3D pupil detector.") all_plugins.append(Pye3DPlugin) - default3D = Pye3DPlugin - return default2D, default3D, all_plugins + return all_plugins diff --git a/pupil_src/shared_modules/pupil_detector_plugins/detector_2d_plugin.py b/pupil_src/shared_modules/pupil_detector_plugins/detector_2d_plugin.py index 3ea8ffab95..04687c475d 100644 --- a/pupil_src/shared_modules/pupil_detector_plugins/detector_2d_plugin.py +++ b/pupil_src/shared_modules/pupil_detector_plugins/detector_2d_plugin.py @@ -28,27 +28,38 @@ from methods import normalize from plugin import Plugin -from .detector_base_plugin import PropertyProxy, PupilDetectorPlugin +from .detector_base_plugin import PupilDetectorPlugin from .visualizer_2d import draw_pupil_outline logger = logging.getLogger(__name__) class Detector2DPlugin(PupilDetectorPlugin): - uniqueness = "by_class" - icon_font = "pupil_icons" - icon_chr = chr(0xEC18) + + pupil_detection_identifier = "2d" + pupil_detection_method = "2d c++" label = "C++ 2d detector" - identifier = "2d" + icon_font = "pupil_icons" + icon_chr = chr(0xEC18) order = 0.100 + @property + def pretty_class_name(self): + return "Pupil Detector 2D" + + @property + def pupil_detector(self) -> DetectorBase: + return self.detector_2d + def __init__( - self, g_pool=None, namespaced_properties=None, detector_2d: Detector2D = None + self, + g_pool=None, + properties=None, + detector_2d: Detector2D = None, ): super().__init__(g_pool=g_pool) - self.detector_2d = detector_2d or Detector2D(namespaced_properties or {}) - self.proxy = PropertyProxy(self.detector_2d) + self.detector_2d = detector_2d or Detector2D(properties or {}) def detect(self, frame, **kwargs): # convert roi-plugin to detector roi @@ -60,28 +71,26 @@ def detect(self, frame, **kwargs): color_img=debug_img, roi=roi, ) - eye_id = self.g_pool.eye_id - location = result["location"] - result["norm_pos"] = normalize( - location, (frame.width, frame.height), flip_y=True + + norm_pos = normalize( + result["location"], (frame.width, frame.height), flip_y=True ) - result["timestamp"] = frame.timestamp - result["topic"] = f"pupil.{eye_id}.{self.identifier}" - result["id"] = eye_id - result["method"] = "2d c++" - return result - @property - def pupil_detector(self) -> DetectorBase: - return self.detector_2d + # Create basic pupil datum + datum = self.create_pupil_datum( + norm_pos=norm_pos, + diameter=result["diameter"], + confidence=result["confidence"], + timestamp=frame.timestamp, + ) - @property - def pretty_class_name(self): - return "Pupil Detector 2D" + # Fill out 2D model data + datum["ellipse"] = {} + datum["ellipse"]["axes"] = result["ellipse"]["axes"] + datum["ellipse"]["angle"] = result["ellipse"]["angle"] + datum["ellipse"]["center"] = result["ellipse"]["center"] - def gl_display(self): - if self._recent_detection_result: - draw_pupil_outline(self._recent_detection_result, color_rgb=(0, 0.5, 1)) + return datum def init_ui(self): super().init_ui() @@ -95,8 +104,8 @@ def init_ui(self): self.menu.append(info) self.menu.append( ui.Slider( - "2d.intensity_range", - self.proxy, + "intensity_range", + self.pupil_detector_properties, label="Pupil intensity range", min=0, max=60, @@ -105,8 +114,8 @@ def init_ui(self): ) self.menu.append( ui.Slider( - "2d.pupil_size_min", - self.proxy, + "pupil_size_min", + self.pupil_detector_properties, label="Pupil min", min=1, max=250, @@ -115,11 +124,21 @@ def init_ui(self): ) self.menu.append( ui.Slider( - "2d.pupil_size_max", - self.proxy, + "pupil_size_max", + self.pupil_detector_properties, label="Pupil max", min=50, max=400, step=1, ) ) + + def gl_display(self): + if self._recent_detection_result: + draw_pupil_outline(self._recent_detection_result, color_rgb=(0, 0.5, 1)) + + def on_resolution_change(self, old_size, new_size): + properties = self.pupil_detector.get_properties() + properties["pupil_size_max"] *= new_size[0] / old_size[0] + properties["pupil_size_min"] *= new_size[0] / old_size[0] + self.pupil_detector.update_properties(properties) diff --git a/pupil_src/shared_modules/pupil_detector_plugins/detector_3d_plugin.py b/pupil_src/shared_modules/pupil_detector_plugins/detector_3d_plugin.py deleted file mode 100644 index c51cf9388e..0000000000 --- a/pupil_src/shared_modules/pupil_detector_plugins/detector_3d_plugin.py +++ /dev/null @@ -1,204 +0,0 @@ -""" -(*)~--------------------------------------------------------------------------- -Pupil - eye tracking platform -Copyright (C) 2012-2020 Pupil Labs - -Distributed under the terms of the GNU -Lesser General Public License (LGPL v3.0). -See COPYING and COPYING.LESSER for license details. ----------------------------------------------------------------------------~(*) -""" -import logging -from version_utils import parse_version - -import pupil_detectors -from pupil_detectors import Detector3D, DetectorBase, Roi -from pyglui import ui -from pyglui.cygl.utils import draw_gl_texture - -from gl_utils import ( - adjust_gl_view, - basic_gl_setup, - clear_gl_screen, - make_coord_system_norm_based, - make_coord_system_pixel_based, -) -from methods import normalize -from plugin import Plugin - -from .detector_base_plugin import PropertyProxy, PupilDetectorPlugin -from .visualizer_2d import draw_eyeball_outline, draw_pupil_outline -from .visualizer_3d import Eye_Visualizer - -logger = logging.getLogger(__name__) - -if parse_version(pupil_detectors.__version__) < parse_version("1.1.1"): - msg = ( - f"This version of Pupil requires pupil_detectors >= 1.1.1." - f" You are running with pupil_detectors == {pupil_detectors.__version__}." - f" Please upgrade to a newer version!" - ) - logger.error(msg) - raise RuntimeError(msg) - - -class Detector3DPlugin(PupilDetectorPlugin): - uniqueness = "by_class" - icon_font = "pupil_icons" - icon_chr = chr(0xEC19) - - label = "C++ 3d detector" - identifier = "3d" - order = 0.101 - - def __init__( - self, g_pool=None, namespaced_properties=None, detector_3d: Detector3D = None - ): - super().__init__(g_pool=g_pool) - detector = detector_3d or Detector3D(namespaced_properties or {}) - self._initialize(detector) - - @property - def detector_3d(self): - return self._detector_internal - - def _initialize(self, detector: Detector3D): - # initialize plugin with a detector instance, safe to call multiple times - self._detector_internal = detector - self.proxy = PropertyProxy(self.detector_3d) - - # In case of re-initialization, we need to close the debug window or else we - # leak the opengl window. We can open the new one again afterwards. - try: - debug_window_was_open = self.is_debug_window_open - except AttributeError: - # debug window does not exist yet - debug_window_was_open = False - if debug_window_was_open: - self.debug_window_close() - self.debugVisualizer3D = Eye_Visualizer( - self.g_pool, self.detector_3d.focal_length() - ) - if debug_window_was_open: - self.debug_window_open() - - self._last_focal_length = self.detector_3d.focal_length() - if self.ui_available: - # ui was wrapped around old detector, need to re-init for new one - self._reinit_ui() - - def _process_focal_length_changes(self): - focal_length = self.g_pool.capture.intrinsics.focal_length - if focal_length != self._last_focal_length: - logger.debug( - f"Focal length change detected: {focal_length}." - " Re-initializing 3D detector." - ) - # reinitialize detector with same properties but updated focal length - properties = self.detector_3d.get_properties() - new_detector = Detector3D(properties=properties, focal_length=focal_length) - self._initialize(new_detector) - - def detect(self, frame, **kwargs): - self._process_focal_length_changes() - - # convert roi-plugin to detector roi - roi = Roi(*self.g_pool.roi.bounds) - - debug_img = frame.bgr if self.g_pool.display_mode == "algorithm" else None - result = self.detector_3d.detect( - gray_img=frame.gray, - timestamp=frame.timestamp, - color_img=debug_img, - roi=roi, - debug=self.is_debug_window_open, - internal_raw_2d_data=kwargs.get("internal_raw_2d_data", None), - ) - - eye_id = self.g_pool.eye_id - location = result["location"] - result["norm_pos"] = normalize( - location, (frame.width, frame.height), flip_y=True - ) - result["topic"] = f"pupil.{eye_id}.{self.identifier}" - result["id"] = eye_id - result["method"] = "3d c++" - return result - - @property - def pupil_detector(self) -> DetectorBase: - return self.detector_3d - - ### PupilDetectorPlugin API - - @classmethod - def parse_pretty_class_name(cls) -> str: - return "Pupil Detector 3D" - - def init_ui(self): - super().init_ui() - self._reinit_ui() - - def _reinit_ui(self): - self.menu.elements.clear() - self.menu.label = self.pretty_class_name - self.menu.append( - ui.Info_Text( - "Open the debug window to see a visualization of the 3D pupil detection." - ) - ) - self.menu.append(ui.Button("Reset 3D model", self.reset_model)) - self.menu.append(ui.Button("Open debug window", self.debug_window_toggle)) - model_sensitivity_slider = ui.Slider( - "3d.model_sensitivity", - self.proxy, - label="Model sensitivity", - min=0.990, - max=1.0, - step=0.0001, - ) - model_sensitivity_slider.display_format = "%0.4f" - self.menu.append(model_sensitivity_slider) - self.menu.append( - ui.Switch("3d.model_is_frozen", self.proxy, label="Freeze model") - ) - - def gl_display(self): - self.debug_window_update() - if self._recent_detection_result: - draw_eyeball_outline(self._recent_detection_result) - draw_pupil_outline(self._recent_detection_result) - - def cleanup(self): - self.debug_window_close() # if we change detectors, be sure debug window is also closed - - # Public - - def reset_model(self): - self.detector_3d.reset_model() - - # Debug window management - - @property - def is_debug_window_open(self) -> bool: - return self.debugVisualizer3D.window is not None - - def debug_window_toggle(self): - if not self.is_debug_window_open: - self.debug_window_open() - else: - self.debug_window_close() - - def debug_window_open(self): - if not self.is_debug_window_open: - self.debugVisualizer3D.open_window() - - def debug_window_close(self): - if self.is_debug_window_open: - self.debugVisualizer3D.close_window() - - def debug_window_update(self): - if self.is_debug_window_open: - self.debugVisualizer3D.update_window( - self.g_pool, self.detector_3d.debug_result - ) diff --git a/pupil_src/shared_modules/pupil_detector_plugins/detector_base_plugin.py b/pupil_src/shared_modules/pupil_detector_plugins/detector_base_plugin.py index c7ca6d2e24..d494dea028 100644 --- a/pupil_src/shared_modules/pupil_detector_plugins/detector_base_plugin.py +++ b/pupil_src/shared_modules/pupil_detector_plugins/detector_base_plugin.py @@ -10,6 +10,7 @@ """ import abc import logging +import re import traceback import typing as T @@ -21,43 +22,83 @@ EVENT_KEY = "pupil_detection_results" +PUPIL_DETECTOR_NOTIFICATION_SUBJECT_PATTERN = "^pupil_detector\.(?P[a-z_]+)$" -class PropertyProxy: + +class DetectorPropertyProxy: """Wrapper around detector properties for easy UI coupling.""" def __init__(self, detector): self.__dict__["detector"] = detector - def __getattr__(self, namespaced_key): - namespace, key = namespaced_key.split(".") - return self.detector.get_properties()[namespace][key] + def __getattr__(self, key): + return self.detector.get_properties()[key] - def __setattr__(self, namespaced_key, value): - namespace, key = namespaced_key.split(".") - self.detector.update_properties({namespace: {key: value}}) + def __setattr__(self, key, value): + self.detector.update_properties({key: value}) class PupilDetectorPlugin(Plugin): + label = "Unnamed" # Used in eye -> general settings as selector - # Used to select correct detector on set_pupil_detection_enabled: - identifier = "unnamed" - order = 0.1 + + pupil_detection_identifier = "unnamed" + pupil_detection_method = "undefined" @property @abc.abstractmethod def pupil_detector(self) -> DetectorBase: pass + @property + def pupil_detector_properties(self) -> DetectorPropertyProxy: + attr_name = "__pupil_detector_properties" + if not hasattr(self, attr_name): + property_proxy = DetectorPropertyProxy(self.pupil_detector) + setattr(self, attr_name, property_proxy) + return getattr(self, attr_name) + + @abc.abstractmethod + def detect(self, frame, **kwargs): + pass + + def create_pupil_datum(self, norm_pos, diameter, confidence, timestamp) -> dict: + """""" + eye_id = self.g_pool.eye_id + + # TODO: Assert arguments are valid + + # Create basic pupil datum with required fields + datum = {} + datum["id"] = eye_id + datum["topic"] = f"pupil.{eye_id}.{self.pupil_detection_identifier}" + datum["method"] = f"{self.pupil_detection_method}" + datum["norm_pos"] = norm_pos + datum["diameter"] = diameter + datum["confidence"] = confidence + datum["timestamp"] = timestamp + return datum + + ### Plugin API + + uniqueness = "by_class" + order = 0.1 + + @property + def enabled(self) -> bool: + return self._enabled + + @enabled.setter + def enabled(self, is_on: bool): + self._enabled = is_on + for elem in self.menu: + elem.read_only = not self.enabled + def __init__(self, g_pool): super().__init__(g_pool) g_pool.pupil_detector = self self._recent_detection_result = None - self._notification_handler = { - "pupil_detector.broadcast_properties": self.handle_broadcast_properties_notification, - "pupil_detector.set_property": self.handle_set_property_notification, - "set_pupil_detection_enabled": self.handle_set_pupil_detection_enabled_notification, - } - self._last_frame_size = None + self._enabled = True def init_ui(self): @@ -66,141 +107,182 @@ def init_ui(self): def deinit_ui(self): self.remove_menu() - @property - def enabled(self): - return self._enabled - - @enabled.setter - def enabled(self, value): - self._enabled = value - for elem in self.menu: - elem.read_only = not self.enabled - def recent_events(self, event): - if not self.enabled: - self._recent_detection_result = None - return + frame = event.get("frame", None) - frame = event.get("frame") - if not frame: + if not frame or not self.enabled: self._recent_detection_result = None return - frame_size = (frame.width, frame.height) - if frame_size != self._last_frame_size: - if self._last_frame_size is not None: - self.on_resolution_change(self._last_frame_size, frame_size) - self._last_frame_size = frame_size + # Detect if resolution changed + self.__update_frame_size_if_changed(new_frame_size=(frame.width, frame.height)) - # TODO: The following sections with internal_2d_raw_data are for allowing dual - # detectors until pye3d is finished. Then we should cleanup this section! + # Get results of previous detectors + # TODO: This is currently used by Pye3D to get the results of the 2D detection + previous_detection_results = event.get(EVENT_KEY, []) - # this is only revelant when running the 3D detector, for 2D we just ignore the - # additional parameter detection_result = self.detect( frame=frame, - internal_raw_2d_data=event.get("internal_2d_raw_data", None), # TODO: workaround to get 2D data into pye3D for now - previous_detection_results=event.get(EVENT_KEY, []), + previous_detection_results=previous_detection_results, ) - # if we are running the 2D detector, we might get internal data that we don't - # want published, so we remove it from the dict - if "internal_2d_raw_data" in detection_result: - event["internal_2d_raw_data"] = detection_result.pop("internal_2d_raw_data") - - if EVENT_KEY in event: - event[EVENT_KEY].append(detection_result) - else: - event[EVENT_KEY] = [detection_result] + # Append the new detection result to the previous results + event[EVENT_KEY] = previous_detection_results + [detection_result] + # Save the most recent detection result for visualization self._recent_detection_result = detection_result - @abc.abstractmethod - def detect(self, frame, **kwargs): + def on_notify(self, notification): + + subject_match = re.match( + PUPIL_DETECTOR_NOTIFICATION_SUBJECT_PATTERN, notification["subject"] + ) + + if subject_match: + # { + # "subject": "pupil_detector.", + # "eye_id": , # optinal + # "detector_plugin_class_name": , # optional + # ... + # } + + action = subject_match["action"] + + this_eye_id = self.g_pool.eye_id + this_plugin_class_name = type(self).__name__ + + if "eye_id" in notification and notification["eye_id"] != this_eye_id: + # Eye id doesn't match current process eye id - ignoring notification + # NOTE: Missing value behaves like a match + return + + if ( + "detector_plugin_class_name" in notification + and notification["detector_plugin_class_name"] != this_plugin_class_name + ): + # Detector plugin class name doesn't match current plugin - ignoring notification + # NOTE: Missing value behaves like a match + return + + if action == "set_enabled": + self.__safely_update_enabled(notification) + + elif action == "set_roi": + self.__safely_update_roi(notification) + + elif action == "set_properties": + self.__safely_update_properties(notification) + + elif action == "broadcast_properties": + self.notify_all( + { + "subject": f"pupil_detector.properties.{this_eye_id}.{this_plugin_class_name}", + "values": self.pupil_detector.get_properties(), + } + ) + + def on_resolution_change(self, old_size, new_size): pass - def on_notify(self, notification): - subject = notification["subject"] - for subject_prefix, handler in self._notification_handler.items(): - if subject.startswith(subject_prefix): - handler(notification) - - def handle_broadcast_properties_notification(self, notification): - target_process = notification.get("target", self.g_pool.process) - should_respond = target_process == self.g_pool.process - if should_respond: - props = self.namespaced_detector_properties() - properties_broadcast = { - "subject": f"pupil_detector.properties.{self.g_pool.eye_id}", - **props, # add properties to broadcast - } - self.notify_all(properties_broadcast) - - def handle_set_property_notification(self, notification): - target_process = notification.get("target", self.g_pool.process) - if target_process != self.g_pool.process: + ### Private helpers + + def __safely_update_enabled(self, notification: dict): + try: + value = notification["value"] + except KeyError: + logger.error(f"Malformed notification: missing 'value' key") + logger.debug(traceback.format_exc()) + return + + if not isinstance(value, bool): + logger.error("Enabled value must be a bool") + logger.error(f"Invalid value: {value}") + return + + self.enabled = value + logger.debug(f"enabled set to {value}") + + def __safely_update_roi(self, notification): + try: + value = notification["value"] + except KeyError: + logger.error(f"Malformed notification: missing 'value' key") + logger.debug(traceback.format_exc()) return try: - property_name = notification["name"] - property_value = notification["value"] - subject_components = notification["subject"].split(".") - # len(pupil_detector.properties) is at least 2 due to the subject prefix - # being pupil_detector.set_property. The third component is the optional - # namespace of the property. - if len(subject_components) > 2: - namespace = subject_components[2] - self.pupil_detector.update_properties( - {namespace: {property_name: property_value}} - ) - elif property_name == "roi": - # Modify the ROI with the values sent over network - - try: - minX, minY, maxX, maxY = property_value - except (ValueError, TypeError) as err: - # NOTE: ValueError gets throws when length of the tuple does not - # match. TypeError gets thrown when it is not a tuple. - raise ValueError( - "ROI needs to be 4 integers: (minX, minY, maxX, maxY)" - ) from err - - # Apply very strict error checking here, although roi deal with invalid - # values, so the user gets immediate feedback and does not wonder why - # something did not work as expected. - width, height = self.g_pool.roi.frame_size - if not ((0 <= minX < maxX < width) and (0 <= minY < maxY <= height)): - raise ValueError( - "Received ROI with invalid dimensions!" - f" (minX={minX}, minY={minY}, maxX={maxX}, maxY={maxY})" - f" for frame size ({width} x {height})" - ) - - self.g_pool.roi.bounds = (minX, minY, maxX, maxY) - - else: - raise KeyError( - "Notification subject does not " - "specifiy detector type nor modify ROI." - ) - logger.debug(f"'{property_name}' property set to {property_value}") + minX, minY, maxX, maxY = value + minX, minY, maxX, maxY = int(minX), int(minY), int(maxX), int(maxY) + except (ValueError, TypeError) as err: + # NOTE: + # - TypeError gets thrown when 'value' is not a tuple + # - TypeError gets thrown when 'minX', 'minY', 'maxX', or 'maxY' are not 'int's + # - ValueError gets throws when length of the tuple 'value' is not 4 + logger.error("ROI needs to be 4 integers: (minX, minY, maxX, maxY)") + logger.error(f"Invalid value: {value}") + logger.debug(traceback.format_exc()) + return + + # Apply very strict error checking here, although roi deal with invalid + # values, so the user gets immediate feedback and does not wonder why + # something did not work as expected. + width, height = self.g_pool.roi.frame_size + if not ((0 <= minX < maxX < width) and (0 <= minY < maxY <= height)): + logger.error( + "Received ROI with invalid dimensions!" + f" (minX={minX}, minY={minY}, maxX={maxX}, maxY={maxY})" + f" for frame size ({width} x {height})" + ) + logger.error(f"Invalid value: {value}") + return + + self.g_pool.roi.bounds = (minX, minY, maxX, maxY) + logger.debug(f"roi set to {value}") + + def __safely_update_properties(self, notification): + try: + plugin_class_name = notification["detector_plugin_class_name"] + except KeyError: + logger.error( + f"Malformed notification: missing 'detector_plugin_class_name' key" + ) + logger.debug(traceback.format_exc()) + return + + if not isinstance(plugin_class_name, str): + logger.error("Detector plugin class name must be a string") + logger.error(f"Invalid value: {plugin_class_name}") + return + + if plugin_class_name != type(self).__name__: + # Detector plugin class name doesn't match current plugin - ignoring notification + return + + try: + values = notification["values"] except KeyError: - logger.error("Malformed notification received") + logger.error(f"Malformed notification: missing 'values' key") + logger.debug(traceback.format_exc()) + return + + if not isinstance(values, dict): + logger.error(f"Invalid value: {values}") logger.debug(traceback.format_exc()) + return + + try: + self.pupil_detector.update_properties(values) except (ValueError, TypeError): - logger.error("Invalid property or value") + logger.error(f"Invalid value: {values}") logger.debug(traceback.format_exc()) - def handle_set_pupil_detection_enabled_notification(self, notification): - is_on = notification["value"] - self.enabled = is_on + def __update_frame_size_if_changed(self, new_frame_size): + attr_name = "__last_frame_size" - def namespaced_detector_properties(self) -> dict: - return self.pupil_detector.get_properties() + old_frame_size = getattr(self, attr_name, None) - def on_resolution_change(self, old_size, new_size): - properties = self.pupil_detector.get_properties() - properties["2d"]["pupil_size_max"] *= new_size[0] / old_size[0] - properties["2d"]["pupil_size_min"] *= new_size[0] / old_size[0] - self.pupil_detector.update_properties(properties) + if new_frame_size != old_frame_size: + if old_frame_size is not None: + self.on_resolution_change(old_frame_size, new_frame_size) + setattr(self, attr_name, new_frame_size) diff --git a/pupil_src/shared_modules/pupil_detector_plugins/pye3d_plugin.py b/pupil_src/shared_modules/pupil_detector_plugins/pye3d_plugin.py index ca9919fd63..0ba61c30ce 100644 --- a/pupil_src/shared_modules/pupil_detector_plugins/pye3d_plugin.py +++ b/pupil_src/shared_modules/pupil_detector_plugins/pye3d_plugin.py @@ -11,8 +11,9 @@ import logging import pye3d -from pye3d.detector_3d import Detector3D, CameraModel +from pye3d.detector_3d import Detector3D, CameraModel, DetectorMode from pyglui import ui +from methods import normalize from .detector_base_plugin import PupilDetectorPlugin from .visualizer_2d import draw_eyeball_outline, draw_pupil_outline, draw_ellipse @@ -21,33 +22,59 @@ logger = logging.getLogger(__name__) version_installed = getattr(pye3d, "__version__", "0.0.1") -version_supported = "0.0.1" +version_supported = "0.0.4" if version_installed != version_supported: logger.info( f"Requires pye3d version {version_supported} " f"(Installed: {version_installed})" ) - raise ImportError + raise ImportError("Unsupported version found") class Pye3DPlugin(PupilDetectorPlugin): - uniqueness = "by_class" - icon_font = "pupil_icons" - icon_chr = chr(0xEC19) + pupil_detection_identifier = "3d" + # pupil_detection_method implemented as variable label = "Pye3D" - identifier = "3d" + icon_font = "pupil_icons" + icon_chr = chr(0xEC19) order = 0.101 - def __init__(self, g_pool=None): + @property + def pupil_detector(self): + return self.detector + + def __init__( + self, + g_pool=None, + ): super().__init__(g_pool=g_pool) self.camera = CameraModel( focal_length=self.g_pool.capture.intrinsics.focal_length, resolution=self.g_pool.capture.intrinsics.resolution, ) - self.detector = Detector3D(camera=self.camera) + async_apps = ("capture", "service") + mode = ( + DetectorMode.asynchronous + if g_pool.app in async_apps + else DetectorMode.blocking + ) + logger.debug(f"Running {mode.name} in {g_pool.app}") + self.detector = Detector3D(camera=self.camera, long_term_mode=mode) + + method_suffix = { + DetectorMode.asynchronous: "real-time", + DetectorMode.blocking: "post-hoc", + } + self.pupil_detection_method = f"pye3d {pye3d.__version__} {method_suffix[mode]}" + self.debugVisualizer3D = Eye_Visualizer(self.g_pool, self.camera.focal_length) + self.__debug_window_button = None + + def get_init_dict(self): + init_dict = super().get_init_dict() + return init_dict def _process_camera_changes(self): camera = CameraModel( @@ -84,21 +111,33 @@ def detect(self, frame, **kwargs): datum_2d = datum break else: - # TODO: Should we handle this more gracefully? Can this even happen? What - # could we return in that case? - raise RuntimeError("No 2D detection result! Needed for pye3D!") + logger.warning( + "Required 2d pupil detection input not available. " + "Returning default pye3d datum." + ) + return self.create_pupil_datum( + norm_pos=[0.5, 0.5], + diameter=0.0, + confidence=0.0, + timestamp=frame.timestamp, + ) result = self.detector.update_and_detect( datum_2d, frame.gray, debug=self.is_debug_window_open ) - eye_id = self.g_pool.eye_id - result["timestamp"] = frame.timestamp - result["topic"] = f"pupil.{eye_id}.{self.identifier}" - result["id"] = eye_id - result["method"] = "3d c++" + norm_pos = normalize( + result["location"], (frame.width, frame.height), flip_y=True + ) + template = self.create_pupil_datum( + norm_pos=norm_pos, + diameter=result["diameter"], + confidence=result["confidence"], + timestamp=frame.timestamp, + ) + template.update(result) - return result + return template def on_notify(self, notification): super().on_notify(notification) @@ -120,12 +159,30 @@ def init_ui(self): super().init_ui() self.menu.label = self.pretty_class_name + help_text = ( + f"pye3d {pye3d.__version__} - a model-based 3d pupil detector with corneal " + "refraction correction. Read more about the detector in our docs website." + ) + self.menu.append(ui.Info_Text(help_text)) + self.menu.append(ui.Info_Text("Visualizations:")) + self.menu.append(ui.Info_Text(" Green circle: eye model outline.")) + self.menu.append(ui.Info_Text(" Blue ellipse: 2d pupil detection.")) + self.menu.append(ui.Info_Text(" Red ellipse: 3d pupil detection.")) + self.menu.append(ui.Separator()) self.menu.append(ui.Button("Reset 3D model", self.reset_model)) - self.menu.append(ui.Button("Toggle debug window", self.debug_window_toggle)) + self.__debug_window_button = ui.Button( + self.__debug_window_button_label, self.debug_window_toggle + ) - # self.menu.append( - # ui.Switch(TODO, label="Freeze model") - # ) + help_text = ( + "The 3d model automatically updates in the background. Freeze the model to " + "turn off automatic model updates. Refer to the docs website for details. " + ) + self.menu.append(ui.Info_Text(help_text)) + self.menu.append( + ui.Switch("is_long_term_model_frozen", self.detector, label="Freeze model") + ) + self.menu.append(self.__debug_window_button) def gl_display(self): self.debug_window_update() @@ -154,9 +211,12 @@ def gl_display(self): rgba=(0, 1, 0, 1), thickness=2, ) + if self.__debug_window_button: + self.__debug_window_button.label = self.__debug_window_button_label def cleanup(self): - self.debug_window_close() # if we change detectors, be sure debug window is also closed + # if we change detectors, be sure debug window is also closed + self.debug_window_close() # Public @@ -165,6 +225,13 @@ def reset_model(self): # Debug window management + @property + def __debug_window_button_label(self) -> str: + if not self.is_debug_window_open: + return "Open debug window" + else: + return "Close debug window" + @property def is_debug_window_open(self) -> bool: return self.debugVisualizer3D.window is not None diff --git a/pupil_src/shared_modules/pupil_detector_plugins/visualizer_3d.py b/pupil_src/shared_modules/pupil_detector_plugins/visualizer_3d.py deleted file mode 100644 index c45d286f58..0000000000 --- a/pupil_src/shared_modules/pupil_detector_plugins/visualizer_3d.py +++ /dev/null @@ -1,307 +0,0 @@ -""" -(*)~--------------------------------------------------------------------------- -Pupil - eye tracking platform -Copyright (C) 2012-2020 Pupil Labs - -Distributed under the terms of the GNU -Lesser General Public License (LGPL v3.0). -See COPYING and COPYING.LESSER for license details. ----------------------------------------------------------------------------~(*) -""" -import math - -import numpy as np -from gl_utils.trackball import Trackball -from OpenGL.GL import ( - GL_LINES, - GL_MODELVIEW, - GL_TRIANGLE_FAN, - glLoadMatrixf, - glMatrixMode, - glPopMatrix, - glPushMatrix, -) -from pyglui.cygl import utils as glutils -from pyglui.cygl.utils import RGBA -from visualizer import Visualizer - - -def get_perpendicular_vector(v): - """ Finds an arbitrary perpendicular vector to *v*.""" - # http://codereview.stackexchange.com/questions/43928/algorithm-to-get-an-arbitrary-perpendicular-vector - # for two vectors (x, y, z) and (a, b, c) to be perpendicular, - # the following equation has to be fulfilled - # 0 = ax + by + cz - - # x = y = z = 0 is not an acceptable solution - if v[0] == v[1] == v[2] == 0: - logger.error("zero-vector") - - # If one dimension is zero, this can be solved by setting that to - # non-zero and the others to zero. Example: (4, 2, 0) lies in the - # x-y-Plane, so (0, 0, 1) is orthogonal to the plane. - if v[0] == 0: - return np.array((1, 0, 0)) - if v[1] == 0: - return np.array((0, 1, 0)) - if v[2] == 0: - return np.array((0, 0, 1)) - - # arbitrarily set a = b = 1 - # then the equation simplifies to - # c = -(x + y)/z - return np.array([1, 1, -1.0 * (v[0] + v[1]) / v[2]]) - - -class Eye_Visualizer(Visualizer): - def __init__(self, g_pool, focal_length): - super().__init__(g_pool, "Debug Visualizer", False) - - self.focal_length = focal_length - self.image_width = 640 # right values are assigned in update - self.image_height = 480 - - camera_fov = math.degrees( - 2.0 * math.atan(self.image_height / (2.0 * self.focal_length)) - ) - self.trackball = Trackball(camera_fov) - - ############## MATRIX FUNCTIONS ############################## - - def get_anthropomorphic_matrix(self): - temp = np.identity(4) - temp[2, 2] *= -1 - return temp - - def get_adjusted_pixel_space_matrix(self, scale): - # returns a homoegenous matrix - temp = self.get_anthropomorphic_matrix() - temp[3, 3] *= scale - return temp - - def get_image_space_matrix(self, scale=1.0): - temp = self.get_adjusted_pixel_space_matrix(scale) - temp[1, 1] *= -1 # image origin is top left - temp[0, 3] = -self.image_width / 2.0 - temp[1, 3] = self.image_height / 2.0 - temp[2, 3] = -self.focal_length - return temp.T - - def get_pupil_transformation_matrix( - self, circle_normal, circle_center, circle_scale=1.0 - ): - """ - OpenGL matrix convention for typical GL software - with positive Y=up and positive Z=rearward direction - RT = right - UP = up - BK = back - POS = position/translation - US = uniform scale - - float transform[16]; - - [0] [4] [8 ] [12] - [1] [5] [9 ] [13] - [2] [6] [10] [14] - [3] [7] [11] [15] - - [RT.x] [UP.x] [BK.x] [POS.x] - [RT.y] [UP.y] [BK.y] [POS.y] - [RT.z] [UP.z] [BK.z] [POS.Z] - [ ] [ ] [ ] [US ] - """ - temp = self.get_anthropomorphic_matrix() - right = temp[:3, 0] - up = temp[:3, 1] - back = temp[:3, 2] - translation = temp[:3, 3] - back[:] = np.array(circle_normal) - back[2] *= -1 # our z axis is inverted - - if np.linalg.norm(back) != 0: - back[:] /= np.linalg.norm(back) - right[:] = get_perpendicular_vector(back) / np.linalg.norm( - get_perpendicular_vector(back) - ) - up[:] = np.cross(right, back) / np.linalg.norm(np.cross(right, back)) - right[:] *= circle_scale - back[:] *= circle_scale - up[:] *= circle_scale - translation[:] = np.array(circle_center) - translation[2] *= -1 - return temp.T - - ############## DRAWING FUNCTIONS ############################## - - def draw_debug_info(self, result): - models = result["models"] - eye = models[0]["sphere"] - direction = result["circle"][1] - pupil_radius = result["circle"][2] - - status = " Eyeball center : X: %.2fmm Y: %.2fmm Z: %.2fmm\n Pupil direction: X: %.2f Y: %.2f Z: %.2f\n Pupil Diameter: %.2fmm\n " % ( - eye[0][0], - eye[0][1], - eye[0][2], - direction[0], - direction[1], - direction[2], - pupil_radius * 2, - ) - - self.glfont.push_state() - self.glfont.set_color_float((0, 0, 0, 1)) - - self.glfont.draw_multi_line_text(5, 20, status) - - # draw model info for each model - delta_y = 20 - for model in models: - modelStatus = ( - "Model: %d \n" % model["model_id"], - " age: %.1fs\n" - % (self.g_pool.get_timestamp() - model["birth_timestamp"]), - " maturity: %.3f\n" % model["maturity"], - " solver fit: %.6f\n" % model["solver_fit"], - " confidence: %.6f\n" % model["confidence"], - " performance: %.6f\n" % model["performance"], - " perf.Grad.: %.3e\n" % model["performance_gradient"], - ) - modeltext = "".join(modelStatus) - self.glfont.draw_multi_line_text( - self.window_size[0] - 200, delta_y, modeltext - ) - - delta_y += 160 - - self.glfont.pop_state() - - def draw_circle( - self, - circle_center, - circle_normal, - circle_radius, - color=RGBA(1.1, 0.2, 0.8), - num_segments=20, - ): - vertices = [] - vertices.append((0, 0, 0)) # circle center - - # create circle vertices in the xy plane - for i in np.linspace(0.0, 2.0 * math.pi, num_segments): - x = math.sin(i) - y = math.cos(i) - z = 0 - vertices.append((x, y, z)) - - glPushMatrix() - glMatrixMode(GL_MODELVIEW) - glLoadMatrixf( - self.get_pupil_transformation_matrix( - circle_normal, circle_center, circle_radius - ) - ) - glutils.draw_polyline( - (vertices), color=color, line_type=GL_TRIANGLE_FAN - ) # circle - glutils.draw_polyline( - [(0, 0, 0), (0, 0, 4)], color=RGBA(0, 0, 0), line_type=GL_LINES - ) # normal - glPopMatrix() - - def update_window(self, g_pool, result): - - if not result: - return - - if not self.window: - return - - self.begin_update_window() - - self.image_width, self.image_height = g_pool.capture.frame_size - - latest_circle = result["circle"] - predicted_circle = result["predicted_circle"] - edges = result["edges"] - sphere_models = result["models"] - - self.clear_gl_screen() - self.trackball.push() - - # 2. in pixel space draw video frame - glLoadMatrixf(self.get_image_space_matrix(15)) - g_pool.image_tex.draw( - quad=( - (0, self.image_height), - (self.image_width, self.image_height), - (self.image_width, 0), - (0, 0), - ), - alpha=0.5, - ) - - glLoadMatrixf(self.get_adjusted_pixel_space_matrix(15)) - self.draw_frustum(self.image_width, self.image_height, self.focal_length) - - glLoadMatrixf(self.get_anthropomorphic_matrix()) - model_count = 0 - sphere_color = RGBA(0, 147 / 255.0, 147 / 255.0, 0.2) - initial_sphere_color = RGBA(0, 147 / 255.0, 147 / 255.0, 0.2) - - alternative_sphere_color = RGBA(1, 0.5, 0.5, 0.05) - alternative_initial_sphere_color = RGBA(1, 0.5, 0.5, 0.05) - - for model in sphere_models: - bin_positions = model["bin_positions"] - sphere = model["sphere"] - initial_sphere = model["initial_sphere"] - - if model_count == 0: - # self.draw_sphere(initial_sphere[0],initial_sphere[1], color = sphere_color ) - self.draw_sphere(sphere[0], sphere[1], color=initial_sphere_color) - glutils.draw_points(bin_positions, 3, RGBA(0.6, 0.0, 0.6, 0.5)) - - else: - # self.draw_sphere(initial_sphere[0],initial_sphere[1], color = alternative_sphere_color ) - self.draw_sphere( - sphere[0], sphere[1], color=alternative_initial_sphere_color - ) - - model_count += 1 - - self.draw_circle( - latest_circle[0], - latest_circle[1], - latest_circle[2], - RGBA(0.0, 1.0, 1.0, 0.4), - ) - # self.draw_circle( predicted_circle[0], predicted_circle[1], predicted_circle[2], RGBA(1.0,0.0,0.0,0.4)) - - glutils.draw_points(edges, 2, RGBA(1.0, 0.0, 0.6, 0.5)) - - glLoadMatrixf(self.get_anthropomorphic_matrix()) - self.draw_coordinate_system(4) - - self.trackball.pop() - - self.draw_debug_info(result) - - self.end_update_window() - - return True - - ############ window callbacks ################# - def on_resize(self, window, w, h): - Visualizer.on_resize(self, window, w, h) - self.trackball.set_window_size(w, h) - - def on_window_char(self, window, char): - if char == ord("r"): - self.trackball.distance = [0, 0, -0.1] - self.trackball.pitch = 0 - self.trackball.roll = 0 - - def on_scroll(self, window, x, y): - self.trackball.zoom_to(y) diff --git a/pupil_src/shared_modules/pupil_producers.py b/pupil_src/shared_modules/pupil_producers.py index 3e2800fb9a..c602464d05 100644 --- a/pupil_src/shared_modules/pupil_producers.py +++ b/pupil_src/shared_modules/pupil_producers.py @@ -405,9 +405,12 @@ def detection_progress(self) -> float: for eye_id in (0, 1): total_frames = self.eye_frame_num[eye_id] - current_index = self.eye_frame_idx[eye_id] - progress = (current_index + 1) / total_frames - progress = max(0.0, min(progress, 1.0)) + if total_frames > 0: + current_index = self.eye_frame_idx[eye_id] + progress = (current_index + 1) / total_frames + progress = max(0.0, min(progress, 1.0)) + else: + progress = 1.0 progress_by_eye[eye_id] = progress return min(progress_by_eye) diff --git a/pupil_src/shared_modules/raw_data_exporter.py b/pupil_src/shared_modules/raw_data_exporter.py index a138fa127e..d226e4b812 100644 --- a/pupil_src/shared_modules/raw_data_exporter.py +++ b/pupil_src/shared_modules/raw_data_exporter.py @@ -296,7 +296,6 @@ def dict_export( try: diameter_3d = raw_value["diameter_3d"] model_confidence = raw_value["model_confidence"] - model_id = raw_value["model_id"] sphere_center = raw_value["sphere"]["center"] sphere_radius = raw_value["sphere"]["radius"] circle_3d_center = raw_value["circle_3d"]["center"] @@ -310,7 +309,6 @@ def dict_export( except KeyError: diameter_3d = None model_confidence = None - model_id = None sphere_center = [None, None, None] sphere_radius = None circle_3d_center = [None, None, None] @@ -322,6 +320,9 @@ def dict_export( projected_sphere_axis = [None, None] projected_sphere_angle = None + # pye3d no longer includes this field. Keeping for backwards-compatibility. + model_id = raw_value.get("model_id", None) + return { # 2d data "pupil_timestamp": pupil_timestamp, @@ -420,14 +421,28 @@ def dict_export( gaze_points_3d = raw_value["gaze_point_3d"] # binocular if raw_value.get("eye_centers_3d", None) is not None: - eye_centers0_3d = raw_value["eye_centers_3d"].get(0, [None, None, None]) - eye_centers1_3d = raw_value["eye_centers_3d"].get(1, [None, None, None]) - # - gaze_normals0_3d = raw_value["gaze_normals_3d"].get( - 0, [None, None, None] + eye_centers_3d = raw_value["eye_centers_3d"] + gaze_normals_3d = raw_value["gaze_normals_3d"] + + eye_centers0_3d = ( + eye_centers_3d.get("0", None) + or eye_centers_3d.get(0, None) # backwards compatibility + or [None, None, None] + ) + eye_centers1_3d = ( + eye_centers_3d.get("1", None) + or eye_centers_3d.get(1, None) # backwards compatibility + or [None, None, None] + ) + gaze_normals0_3d = ( + gaze_normals_3d.get("0", None) + or gaze_normals_3d.get(0, None) # backwards compatibility + or [None, None, None] ) - gaze_normals1_3d = raw_value["gaze_normals_3d"].get( - 1, [None, None, None] + gaze_normals1_3d = ( + gaze_normals_3d.get("1", None) + or gaze_normals_3d.get(1, None) # backwards compatibility + or [None, None, None] ) # monocular elif raw_value.get("eye_center_3d", None) is not None: diff --git a/pupil_src/shared_modules/version_utils.py b/pupil_src/shared_modules/version_utils.py index 064d48f9fc..3cfdcae249 100644 --- a/pupil_src/shared_modules/version_utils.py +++ b/pupil_src/shared_modules/version_utils.py @@ -56,6 +56,13 @@ def pupil_version_string() -> str: if version is None: raise ValueError("Version Error") + try: + parts_git_tag = version.split("-") + version_parsed = packaging.version.Version(parts_git_tag[0]) + if version_parsed.is_prerelease: + version = version_parsed.base_version + except packaging.version.InvalidVersion: + pass version = version.replace("v", "") # strip version 'v' # print(version) if "-" in version: diff --git a/pupil_src/shared_modules/video_capture/hmd_streaming.py b/pupil_src/shared_modules/video_capture/hmd_streaming.py index 5c45f9fb9f..c91baceeb9 100644 --- a/pupil_src/shared_modules/video_capture/hmd_streaming.py +++ b/pupil_src/shared_modules/video_capture/hmd_streaming.py @@ -124,20 +124,23 @@ def interpret_buffer(self, buffer, width, height) -> np.ndarray: class HMD_Streaming_Source(Base_Source): name = "HMD Streaming" - def __init__(self, g_pool, topics=("hmd_streaming.world",), *args, **kwargs): + def __init__(self, g_pool, topics=("hmd_streaming.world",), hwm=1, *args, **kwargs): super().__init__(g_pool, *args, **kwargs) self.fps = 30 self.projection_matrix = None self.__topics = topics + self.__hwm = hwm self.frame_sub = zmq_tools.Msg_Receiver( self.g_pool.zmq_ctx, self.g_pool.ipc_sub_url, topics=self.__topics, + hwm=self.__hwm, ) def get_init_dict(self): init_dict = super().get_init_dict() init_dict["topics"] = self.__topics + init_dict["hwm"] = self.__hwm return init_dict def cleanup(self): diff --git a/pupil_src/shared_modules/zmq_tools.py b/pupil_src/shared_modules/zmq_tools.py index c19420175e..ce6ee18b95 100644 --- a/pupil_src/shared_modules/zmq_tools.py +++ b/pupil_src/shared_modules/zmq_tools.py @@ -25,6 +25,9 @@ # import ujson as serializer # uncomment for json serialization assert zmq.__version__ > "15.1" +assert ( + serializer.version[0] == 1 +), "msgpack out of date, please upgrade to version (1, 0, 0)" class ZMQ_handler(logging.Handler): @@ -120,7 +123,7 @@ def recv_remaining_frames(self): yield self.socket.recv() def deserialize_payload(self, payload_serialized, *extra_frames): - payload = serializer.loads(payload_serialized, encoding="utf-8") + payload = serializer.loads(payload_serialized) if extra_frames: payload["__raw_data__"] = extra_frames return payload diff --git a/pupil_src/tests/test_raw_data_exporter.py b/pupil_src/tests/test_raw_data_exporter.py index 64c9c683d0..2f72d03af6 100644 --- a/pupil_src/tests/test_raw_data_exporter.py +++ b/pupil_src/tests/test_raw_data_exporter.py @@ -20,8 +20,12 @@ def _test_exporter(exporter, positions, expected_dict_export, world_index=123): assert len(sample_labels) == len(set(sample_labels)), "Labels must be unique" actual_dict_export = exporter.dict_export(raw_value=positions, world_index=123) - assert set(sample_labels) == set(actual_dict_export.keys()), "Labels must be the keys for the exported dict" - assert actual_dict_export == expected_dict_export, "Actual pupil export must be the same as expeted export" + assert set(sample_labels) == set( + actual_dict_export.keys() + ), "Labels must be the keys for the exported dict" + assert ( + actual_dict_export == expected_dict_export + ), "Actual pupil export must be the same as expeted export" def test_pupil_positions_exporter_capture(): @@ -54,215 +58,227 @@ def test_gaze_positions_exporter_pi(): PUPIL_CAPTURE_PUPIL_TIMESTAMP_0 = 18147.38145 PUPIL_CAPTURE_PUPIL_POSITION_0 = { - 'topic': 'pupil.0', - 'circle_3d': { - 'center': (-2.589605454357244, 5.16258305418917, 106.22798206853192), - 'normal': (0.021375378493324514, 0.3073983655417788, -0.951340810675391), - 'radius': 2.178582911117089 + "topic": "pupil.0", + "circle_3d": { + "center": (-2.589605454357244, 5.16258305418917, 106.22798206853192), + "normal": (0.021375378493324514, 0.3073983655417788, -0.951340810675391), + "radius": 2.178582911117089, }, - 'confidence': 0.9661007130628978, - 'timestamp': 18147.38145, - 'diameter_3d': 4.357165822234178, - 'ellipse': { - 'center': (80.87985171196992, 126.05637442643379), - 'axes': (23.82040062925261, 25.43866813955349), - 'angle': 89.7491053060753 + "confidence": 0.9661007130628978, + "timestamp": 18147.38145, + "diameter_3d": 4.357165822234178, + "ellipse": { + "center": (80.87985171196992, 126.05637442643379), + "axes": (23.82040062925261, 25.43866813955349), + "angle": 89.7491053060753, }, - 'norm_pos': (0.42124922766651, 0.3434563831956573), - 'diameter': 25.43866813955349, - 'sphere': { - 'center': (-2.8461099962771383, 1.4738026676878244, 117.64407179663661), - 'radius': 12.0 + "norm_pos": (0.42124922766651, 0.3434563831956573), + "diameter": 25.43866813955349, + "sphere": { + "center": (-2.8461099962771383, 1.4738026676878244, 117.64407179663661), + "radius": 12.0, }, - 'projected_sphere': { - 'center': (81.00061948941932, 103.76713726422189), - 'axes': (126.48321137440784, 126.48321137440784), - 'angle': 90.0 + "projected_sphere": { + "center": (81.00061948941932, 103.76713726422189), + "axes": (126.48321137440784, 126.48321137440784), + "angle": 90.0, }, - 'model_confidence': 0.8313087355695914, - 'model_id': 14, - 'model_birth_timestamp': 18144.589568, - 'theta': 1.8832541345741016, - 'phi': -1.5483314201277814, - 'method': '3d c++', - 'id': 0 + "model_confidence": 0.8313087355695914, + "model_id": 14, + "model_birth_timestamp": 18144.589568, + "theta": 1.8832541345741016, + "phi": -1.5483314201277814, + "method": "3d c++", + "id": 0, } PUPIL_CAPTURE_PUPIL_EXPORT_DICT_0 = { - 'pupil_timestamp': '18147.38145', - 'world_index': 123, - 'eye_id': 0, - 'confidence': 0.9661007130628978, - 'norm_pos_x': 0.42124922766651, - 'norm_pos_y': 0.3434563831956573, - 'diameter': 25.43866813955349, - 'method': '3d c++', - 'ellipse_center_x': 80.87985171196992, - 'ellipse_center_y': 126.05637442643379, - 'ellipse_axis_a': 23.82040062925261, - 'ellipse_axis_b': 25.43866813955349, - 'ellipse_angle': 89.7491053060753, - 'diameter_3d': 4.357165822234178, - 'model_confidence': 0.8313087355695914, - 'model_id': 14, - 'sphere_center_x': -2.8461099962771383, - 'sphere_center_y': 1.4738026676878244, - 'sphere_center_z': 117.64407179663661, - 'sphere_radius': 12.0, - 'circle_3d_center_x': -2.589605454357244, - 'circle_3d_center_y': 5.16258305418917, - 'circle_3d_center_z': 106.22798206853192, - 'circle_3d_normal_x': 0.021375378493324514, - 'circle_3d_normal_y': 0.3073983655417788, - 'circle_3d_normal_z': -0.951340810675391, - 'circle_3d_radius': 2.178582911117089, - 'theta': 1.8832541345741016, - 'phi': -1.5483314201277814, - 'projected_sphere_center_x': 81.00061948941932, - 'projected_sphere_center_y': 103.76713726422189, - 'projected_sphere_axis_a': 126.48321137440784, - 'projected_sphere_axis_b': 126.48321137440784, - 'projected_sphere_angle': 90.0 + "pupil_timestamp": "18147.38145", + "world_index": 123, + "eye_id": 0, + "confidence": 0.9661007130628978, + "norm_pos_x": 0.42124922766651, + "norm_pos_y": 0.3434563831956573, + "diameter": 25.43866813955349, + "method": "3d c++", + "ellipse_center_x": 80.87985171196992, + "ellipse_center_y": 126.05637442643379, + "ellipse_axis_a": 23.82040062925261, + "ellipse_axis_b": 25.43866813955349, + "ellipse_angle": 89.7491053060753, + "diameter_3d": 4.357165822234178, + "model_confidence": 0.8313087355695914, + "model_id": 14, + "sphere_center_x": -2.8461099962771383, + "sphere_center_y": 1.4738026676878244, + "sphere_center_z": 117.64407179663661, + "sphere_radius": 12.0, + "circle_3d_center_x": -2.589605454357244, + "circle_3d_center_y": 5.16258305418917, + "circle_3d_center_z": 106.22798206853192, + "circle_3d_normal_x": 0.021375378493324514, + "circle_3d_normal_y": 0.3073983655417788, + "circle_3d_normal_z": -0.951340810675391, + "circle_3d_radius": 2.178582911117089, + "theta": 1.8832541345741016, + "phi": -1.5483314201277814, + "projected_sphere_center_x": 81.00061948941932, + "projected_sphere_center_y": 103.76713726422189, + "projected_sphere_axis_a": 126.48321137440784, + "projected_sphere_axis_b": 126.48321137440784, + "projected_sphere_angle": 90.0, } PUPIL_CAPTURE_GAZE_TIMESTAMP_0 = 18147.383181999998 PUPIL_CAPTURE_GAZE_POSITION_0 = { - 'topic': 'gaze.3d.01.', - 'eye_centers_3d': { - 0: (19.646972429677437, 16.653615316725187, -16.741350196951785), - 1: (-41.020316707467195, 10.609042518566177, -34.50487956145177) + "topic": "gaze.3d.01.", + "eye_centers_3d": { + "0": (19.646972429677437, 16.653615316725187, -16.741350196951785), + "1": (-41.020316707467195, 10.609042518566177, -34.50487956145177), }, - 'gaze_normals_3d': { - 0: (-0.02317613920692292, 0.0503770422758375, 0.998461326333173), - 1: (0.008199922593243814, 0.017997633703220717, 0.9998044040963957) + "gaze_normals_3d": { + "0": (-0.02317613920692292, 0.0503770422758375, 0.998461326333173), + "1": (0.008199922593243814, 0.017997633703220717, 0.9998044040963957), }, - 'gaze_point_3d': (-26.67606235654398, 87.20744316999192, 2124.3326293817645), - 'confidence': 0.9661007130628978, - 'timestamp': 18147.383181999998, - 'base_data': [ + "gaze_point_3d": (-26.67606235654398, 87.20744316999192, 2124.3326293817645), + "confidence": 0.9661007130628978, + "timestamp": 18147.383181999998, + "base_data": [ { - 'topic': 'pupil.0', - 'circle_3d': { - 'center': (-2.589605454357244, 5.16258305418917, 106.22798206853192), - 'normal': (0.021375378493324514, 0.3073983655417788, -0.951340810675391), - 'radius': 2.178582911117089 + "topic": "pupil.0", + "circle_3d": { + "center": (-2.589605454357244, 5.16258305418917, 106.22798206853192), + "normal": ( + 0.021375378493324514, + 0.3073983655417788, + -0.951340810675391, + ), + "radius": 2.178582911117089, }, - 'confidence': 0.9661007130628978, - 'timestamp': 18147.38145, - 'diameter_3d': 4.357165822234178, - 'ellipse': { - 'center': (80.87985171196992, 126.05637442643379), - 'axes': (23.82040062925261, 25.43866813955349), - 'angle': 89.7491053060753 + "confidence": 0.9661007130628978, + "timestamp": 18147.38145, + "diameter_3d": 4.357165822234178, + "ellipse": { + "center": (80.87985171196992, 126.05637442643379), + "axes": (23.82040062925261, 25.43866813955349), + "angle": 89.7491053060753, }, - 'norm_pos': (0.42124922766651, 0.3434563831956573), - 'diameter': 25.43866813955349, - 'sphere': { - 'center': (-2.8461099962771383, 1.4738026676878244, 117.64407179663661), - 'radius': 12.0 + "norm_pos": (0.42124922766651, 0.3434563831956573), + "diameter": 25.43866813955349, + "sphere": { + "center": (-2.8461099962771383, 1.4738026676878244, 117.64407179663661), + "radius": 12.0, }, - 'projected_sphere': { - 'center': (81.00061948941932, 103.76713726422189), - 'axes': (126.48321137440784, 126.48321137440784), - 'angle': 90.0 + "projected_sphere": { + "center": (81.00061948941932, 103.76713726422189), + "axes": (126.48321137440784, 126.48321137440784), + "angle": 90.0, }, - 'model_confidence': 0.8313087355695914, - 'model_id': 14, - 'model_birth_timestamp': 18144.589568, - 'theta': 1.8832541345741016, - 'phi': -1.5483314201277814, - 'method': '3d c++', - 'id': 0 + "model_confidence": 0.8313087355695914, + "model_id": 14, + "model_birth_timestamp": 18144.589568, + "theta": 1.8832541345741016, + "phi": -1.5483314201277814, + "method": "3d c++", + "id": 0, }, { - 'topic': 'pupil.1', - 'circle_3d': { - 'center': (-1.8241334685531108, -4.1069075704320746, 101.41643695709374), - 'normal': (-0.008068882229597849, -0.30213665999147843, -0.9532304715171239), - 'radius': 2.0663996742618562 + "topic": "pupil.1", + "circle_3d": { + "center": ( + -1.8241334685531108, + -4.1069075704320746, + 101.41643695709374, + ), + "normal": ( + -0.008068882229597849, + -0.30213665999147843, + -0.9532304715171239, + ), + "radius": 2.0663996742618562, }, - 'confidence': 0.9668871617583898, - 'timestamp': 18147.384914, - 'diameter_3d': 4.1327993485237124, - 'ellipse': { - 'center': (84.84988578600691, 70.96598316624396), - 'axes': (23.769241864453477, 25.268831954531986), - 'angle': 86.07716997149748 + "confidence": 0.9668871617583898, + "timestamp": 18147.384914, + "diameter_3d": 4.1327993485237124, + "ellipse": { + "center": (84.84988578600691, 70.96598316624396), + "axes": (23.769241864453477, 25.268831954531986), + "angle": 86.07716997149748, }, - 'norm_pos': (0.44192648846878596, 0.6303855043424793), - 'diameter': 25.268831954531986, - 'sphere': { - 'center': (-1.7273068817979365, -0.481267650534333, 112.85520261529922), - 'radius': 12.0 + "norm_pos": (0.44192648846878596, 0.6303855043424793), + "diameter": 25.268831954531986, + "sphere": { + "center": (-1.7273068817979365, -0.481267650534333, 112.85520261529922), + "radius": 12.0, }, - 'projected_sphere': { - 'center': (86.51058132990725, 93.35602846464754), - 'axes': (131.85036803950402, 131.85036803950402), - 'angle': 90.0 + "projected_sphere": { + "center": (86.51058132990725, 93.35602846464754), + "axes": (131.85036803950402, 131.85036803950402), + "angle": 90.0, }, - 'model_confidence': 0.600632750497887, - 'model_id': 12, - 'model_birth_timestamp': 18141.421905, - 'theta': 1.2638630532171315, - 'phi': -1.5792609004322697, - 'method': '3d c++', - 'id': 1 - } + "model_confidence": 0.600632750497887, + "model_id": 12, + "model_birth_timestamp": 18141.421905, + "theta": 1.2638630532171315, + "phi": -1.5792609004322697, + "method": "3d c++", + "id": 1, + }, ], - 'norm_pos': (0.5077714574008085, 0.39313176450355647) + "norm_pos": (0.5077714574008085, 0.39313176450355647), } PUPIL_CAPTURE_GAZE_EXPORT_DICT_0 = { - 'gaze_timestamp': '18147.383181999998', - 'world_index': 123, - 'confidence': 0.9661007130628978, - 'norm_pos_x': 0.5077714574008085, - 'norm_pos_y': 0.39313176450355647, - 'base_data': '18147.38145-0 18147.384914-1', - 'gaze_point_3d_x': -26.67606235654398, - 'gaze_point_3d_y': 87.20744316999192, - 'gaze_point_3d_z': 2124.3326293817645, - 'eye_center0_3d_x': 19.646972429677437, - 'eye_center0_3d_y': 16.653615316725187, - 'eye_center0_3d_z': -16.741350196951785, - 'gaze_normal0_x': -0.02317613920692292, - 'gaze_normal0_y': 0.0503770422758375, - 'gaze_normal0_z': 0.998461326333173, - 'eye_center1_3d_x': -41.020316707467195, - 'eye_center1_3d_y': 10.609042518566177, - 'eye_center1_3d_z': -34.50487956145177, - 'gaze_normal1_x': 0.008199922593243814, - 'gaze_normal1_y': 0.017997633703220717, - 'gaze_normal1_z': 0.9998044040963957, + "gaze_timestamp": "18147.383181999998", + "world_index": 123, + "confidence": 0.9661007130628978, + "norm_pos_x": 0.5077714574008085, + "norm_pos_y": 0.39313176450355647, + "base_data": "18147.38145-0 18147.384914-1", + "gaze_point_3d_x": -26.67606235654398, + "gaze_point_3d_y": 87.20744316999192, + "gaze_point_3d_z": 2124.3326293817645, + "eye_center0_3d_x": 19.646972429677437, + "eye_center0_3d_y": 16.653615316725187, + "eye_center0_3d_z": -16.741350196951785, + "gaze_normal0_x": -0.02317613920692292, + "gaze_normal0_y": 0.0503770422758375, + "gaze_normal0_z": 0.998461326333173, + "eye_center1_3d_x": -41.020316707467195, + "eye_center1_3d_y": 10.609042518566177, + "eye_center1_3d_z": -34.50487956145177, + "gaze_normal1_x": 0.008199922593243814, + "gaze_normal1_y": 0.017997633703220717, + "gaze_normal1_z": 0.9998044040963957, } PUPIL_INVISIBLE_GAZE_TIMESTAMP_0 = 14088010559.20657 PUPIL_INVISIBLE_GAZE_POSITION_0 = { - 'topic': 'gaze.pi', - 'norm_pos': (0.49499332203584556, 0.3979458844220197), - 'timestamp': 14088010559.20657, - 'confidence': 1.0 + "topic": "gaze.pi", + "norm_pos": (0.49499332203584556, 0.3979458844220197), + "timestamp": 14088010559.20657, + "confidence": 1.0, } PUPIL_INVISIBLE_GAZE_EXPORT_DICT_0 = { - 'gaze_timestamp': '14088010559.20657', - 'world_index': 123, - 'confidence': 1.0, - 'norm_pos_x': 0.49499332203584556, - 'norm_pos_y': 0.3979458844220197, - 'base_data': None, - 'gaze_point_3d_x': None, - 'gaze_point_3d_y': None, - 'gaze_point_3d_z': None, - 'eye_center0_3d_x': None, - 'eye_center0_3d_y': None, - 'eye_center0_3d_z': None, - 'gaze_normal0_x': None, - 'gaze_normal0_y': None, - 'gaze_normal0_z': None, - 'eye_center1_3d_x': None, - 'eye_center1_3d_y': None, - 'eye_center1_3d_z': None, - 'gaze_normal1_x': None, - 'gaze_normal1_y': None, - 'gaze_normal1_z': None + "gaze_timestamp": "14088010559.20657", + "world_index": 123, + "confidence": 1.0, + "norm_pos_x": 0.49499332203584556, + "norm_pos_y": 0.3979458844220197, + "base_data": None, + "gaze_point_3d_x": None, + "gaze_point_3d_y": None, + "gaze_point_3d_z": None, + "eye_center0_3d_x": None, + "eye_center0_3d_y": None, + "eye_center0_3d_z": None, + "gaze_normal0_x": None, + "gaze_normal0_y": None, + "gaze_normal0_z": None, + "eye_center1_3d_x": None, + "eye_center1_3d_y": None, + "eye_center1_3d_z": None, + "gaze_normal1_x": None, + "gaze_normal1_y": None, + "gaze_normal1_z": None, } diff --git a/requirements.txt b/requirements.txt index b85eb31504..df50e7f215 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,7 @@ ### ### Third-party ### -cython -msgpack==0.5.6 +msgpack>=1.0.0 numexpr packaging>=20.0 psutil @@ -11,8 +10,8 @@ pyopengl pyzmq scikit-learn scipy -glfw -pyre @ git+https://github.com/zeromq/pyre +glfw>=2.0.0 +pyre @ https://github.com/zeromq/pyre/archive/master.zip cysignals ; platform_system != "Windows" @@ -23,7 +22,8 @@ opencv-python==3.* ; platform_system == "Windows" ### Pupil-Labs ### pupil-apriltags==1.0.4 -pupil-detectors==1.1.1 +pupil-detectors==2.0.* +pye3d==0.0.4 # pupil-labs/PyAV 0.4.6 av @ git+https://github.com/pupil-labs/PyAV@v0.4.6 ; platform_system != "Windows"