diff --git a/pupil_src/launchables/player.py b/pupil_src/launchables/player.py index 97069b43ec..c6010a57c3 100644 --- a/pupil_src/launchables/player.py +++ b/pupil_src/launchables/player.py @@ -93,6 +93,7 @@ def player( import player_methods as pm from pupil_recording import PupilRecording from csv_utils import write_key_value_file + from hotkey import Hotkey # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins @@ -113,7 +114,11 @@ def player( from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History - from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection + from pupil_producers import ( + DisabledPupilProducer, + Pupil_From_Recording, + Offline_Pupil_Detection, + ) from gaze_producer.gaze_from_recording import GazeFromRecording from gaze_producer.gaze_from_offline_calibration import ( GazeFromOfflineCalibration, @@ -180,6 +185,7 @@ def interrupt_handler(sig, frame): Raw_Data_Exporter, Annotation_Player, Log_History, + DisabledPupilProducer, Pupil_From_Recording, Offline_Pupil_Detection, GazeFromRecording, @@ -562,7 +568,7 @@ def set_window_size(): label=chr(0xE2C5), getter=lambda: False, setter=do_export, - hotkey="e", + hotkey=Hotkey.EXPORT_START_PLAYER_HOTKEY(), label_font="pupil_icons", ) g_pool.quickbar.extend([g_pool.export_button]) @@ -576,6 +582,7 @@ def set_window_size(): # In priority order (first is default) ("Pupil_From_Recording", {}), ("Offline_Pupil_Detection", {}), + ("DisabledPupilProducer", {}), ] _pupil_producer_plugins = list(reversed(_pupil_producer_plugins)) _gaze_producer_plugins = [ diff --git a/pupil_src/launchables/world.py b/pupil_src/launchables/world.py index 361de0e69b..727a8c11aa 100644 --- a/pupil_src/launchables/world.py +++ b/pupil_src/launchables/world.py @@ -100,12 +100,13 @@ def start_stop_eye(eye_id, make_alive): else: stop_eye_process(eye_id) - def detection_enabled_getter() -> bool: - return g_pool.pupil_detection_enabled + def detection_enabled_getter() -> int: + return int(g_pool.pupil_detection_enabled) - def detection_enabled_setter(is_on: bool): + def detection_enabled_setter(value: int): + is_on = bool(value) g_pool.pupil_detection_enabled = is_on - n = {"subject": "set_pupil_detection_enabled", "value": is_on} + n = {"subject": "pupil_detector.set_enabled", "value": is_on} ipc_pub.notify(n) try: @@ -416,7 +417,8 @@ def on_resize(window, w, h): ) # Needed, to update the window buffer while resizing - consume_events_and_render_buffer() + with gl_utils.current_context(main_window): + consume_events_and_render_buffer() def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) @@ -466,8 +468,8 @@ def get_dt(): g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", 0.8 ) - g_pool.pupil_detection_enabled = session_settings.get( - "pupil_detection_enabled", True + g_pool.pupil_detection_enabled = bool( + session_settings.get("pupil_detection_enabled", True) ) g_pool.active_gaze_mapping_plugin = None g_pool.capture = None @@ -478,7 +480,7 @@ def get_dt(): def handle_notifications(noti): subject = noti["subject"] - if subject == "set_pupil_detection_enabled": + if subject == "pupil_detector.set_enabled": g_pool.pupil_detection_enabled = noti["value"] elif subject == "start_plugin": try: @@ -494,7 +496,7 @@ def handle_notifications(noti): g_pool.plugins.clean() elif subject == "eye_process.started": noti = { - "subject": "set_pupil_detection_enabled", + "subject": "pupil_detector.set_enabled", "value": g_pool.pupil_detection_enabled, } ipc_pub.notify(noti) diff --git a/pupil_src/shared_modules/accuracy_visualizer.py b/pupil_src/shared_modules/accuracy_visualizer.py index 4e5d7b1930..6c1d8d96ed 100644 --- a/pupil_src/shared_modules/accuracy_visualizer.py +++ b/pupil_src/shared_modules/accuracy_visualizer.py @@ -10,29 +10,27 @@ """ import logging -from collections import namedtuple +import traceback import typing as T -import OpenGL.GL as gl import numpy as np +import OpenGL.GL as gl +import scipy.spatial from pyglui import ui -from pyglui.cygl.utils import draw_points_norm, draw_polyline_norm, RGBA -from scipy.spatial import ConvexHull +from pyglui.cygl.utils import RGBA, draw_points_norm, draw_polyline_norm from calibration_choreography import ( ChoreographyAction, ChoreographyMode, ChoreographyNotification, ) -from plugin import Plugin - from gaze_mapping import gazer_classes_by_class_name, registered_gazer_classes from gaze_mapping.notifications import ( - CalibrationSetupNotification, CalibrationResultNotification, + CalibrationSetupNotification, ) from gaze_mapping.utils import closest_matches_monocular - +from plugin import Plugin logger = logging.getLogger(__name__) @@ -60,6 +58,13 @@ def empty() -> "CorrelatedAndCoordinateTransformedResult": camera_space=np.ndarray([]), ) + @property + def is_valid(self) -> bool: + if len(self.norm_space.shape) != 2: + return False + # TODO: Make validity check exhaustive + return True + class CorrelationError(ValueError): pass @@ -80,6 +85,13 @@ def failed() -> "AccuracyPrecisionResult": correlation=CorrelatedAndCoordinateTransformedResult.empty(), ) + @property + def is_valid(self) -> bool: + if not self.correlation.is_valid: + return False + # TODO: Make validity check exhaustive + return True + class ValidationInput: def __init__(self): @@ -356,10 +368,12 @@ def __handle_validation_data_notification(self, note_dict: dict) -> bool: return True def recalculate(self): + NOT_ENOUGH_DATA_COLLECTED_ERR_MSG = ( + "Did not collect enough data to estimate gaze mapping accuracy." + ) + if not self.recent_input.is_complete: - logger.info( - "Did not collect enough data to estimate gaze mapping accuracy." - ) + logger.warning(NOT_ENOUGH_DATA_COLLECTED_ERR_MSG) return results = self.calc_acc_prec_errlines( @@ -373,6 +387,10 @@ def recalculate(self): succession_threshold=self.succession_threshold, ) + if not results.is_valid: + logger.warning(NOT_ENOUGH_DATA_COLLECTED_ERR_MSG) + return + accuracy = results.accuracy.result if np.isnan(accuracy): self.accuracy = None @@ -400,8 +418,13 @@ def recalculate(self): self.error_lines = results.error_lines ref_locations = results.correlation.norm_space[1::2, :] if len(ref_locations) >= 3: - hull = ConvexHull(ref_locations) # requires at least 3 points - self.calibration_area = hull.points[hull.vertices, :] + try: + # requires at least 3 points + hull = scipy.spatial.ConvexHull(ref_locations) + self.calibration_area = hull.points[hull.vertices, :] + except scipy.spatial.qhull.QhullError: + logger.warning("Calibration area could not be calculated") + logger.debug(traceback.format_exc()) @staticmethod def calc_acc_prec_errlines( diff --git a/pupil_src/shared_modules/annotations.py b/pupil_src/shared_modules/annotations.py index b453218f68..1c8ae136f7 100644 --- a/pupil_src/shared_modules/annotations.py +++ b/pupil_src/shared_modules/annotations.py @@ -21,6 +21,8 @@ import player_methods as pm import zmq_tools from plugin import Plugin +from hotkey import Hotkey + logger = logging.getLogger(__name__) @@ -63,12 +65,14 @@ def __init__(self, g_pool, annotation_definitions=None): self._annotation_list_menu = None if annotation_definitions is None: - annotation_definitions = [["My annotation", "E"]] + annotation_definitions = [ + ["My annotation", Hotkey.ANNOTATION_EVENT_DEFAULT_HOTKEY()] + ] self._initial_annotation_definitions = annotation_definitions self._definition_to_buttons = {} self._new_annotation_label = "new annotation label" - self._new_annotation_hotkey = "E" + self._new_annotation_hotkey = Hotkey.ANNOTATION_EVENT_DEFAULT_HOTKEY() def get_init_dict(self): annotation_definitions = list(self._definition_to_buttons.keys()) diff --git a/pupil_src/shared_modules/calibration_choreography/base_plugin.py b/pupil_src/shared_modules/calibration_choreography/base_plugin.py index c756a77eea..c4ff870972 100644 --- a/pupil_src/shared_modules/calibration_choreography/base_plugin.py +++ b/pupil_src/shared_modules/calibration_choreography/base_plugin.py @@ -17,6 +17,7 @@ import audio from pyglui import ui from plugin import Plugin +from hotkey import Hotkey from gaze_mapping.gazer_base import GazerBase from gaze_mapping import default_gazer_class @@ -422,7 +423,7 @@ def validation_setter(should_be_on): "is_active", self, label="C", - hotkey="c", + hotkey=Hotkey.GAZE_CALIBRATION_CAPTURE_HOTKEY(), setter=calibration_setter, on_color=self._THUMBNAIL_COLOR_ON, ) @@ -431,7 +432,7 @@ def validation_setter(should_be_on): "is_active", self, label="T", - hotkey="t", + hotkey=Hotkey.GAZE_VALIDATION_CAPTURE_HOTKEY(), setter=validation_setter, on_color=self._THUMBNAIL_COLOR_ON, ) diff --git a/pupil_src/shared_modules/calibration_choreography/controller/marker_window_controller.py b/pupil_src/shared_modules/calibration_choreography/controller/marker_window_controller.py index 4c658ec535..f5e3243d33 100644 --- a/pupil_src/shared_modules/calibration_choreography/controller/marker_window_controller.py +++ b/pupil_src/shared_modules/calibration_choreography/controller/marker_window_controller.py @@ -22,6 +22,8 @@ from pyglui.cygl.utils import draw_points from pyglui.cygl.utils import RGBA +from gl_utils import draw_circle_filled_func_builder + from .gui_monitor import GUIMonitor from .gui_window import GUIWindow @@ -101,6 +103,8 @@ def __init__(self, marker_scale: float): self.__glfont.set_size(32) self.__glfont.set_color_float((0.2, 0.5, 0.9, 1.0)) self.__glfont.set_align_string(v_align="center") + # Private helper + self.__draw_circle_filled = draw_circle_filled_func_builder(cache_size=4) # Public - Marker Management @@ -316,22 +320,22 @@ def __draw_circle_marker( # TODO: adjust num_points such that circles look smooth; smaller circles need less points # TODO: compare runtimes with `draw_points` - _draw_circle_filled( + self.__draw_circle_filled( screen_point, size=self._MARKER_CIRCLE_SIZE_OUTER * radius, color=RGBA(*self._MARKER_CIRCLE_RGB_OUTER, alpha), ) - _draw_circle_filled( + self.__draw_circle_filled( screen_point, size=self._MARKER_CIRCLE_SIZE_MIDDLE * radius, color=RGBA(*self._MARKER_CIRCLE_RGB_MIDDLE, alpha), ) - _draw_circle_filled( + self.__draw_circle_filled( screen_point, size=self._MARKER_CIRCLE_SIZE_INNER * radius, color=RGBA(*self._MARKER_CIRCLE_RGB_INNER, alpha), ) - _draw_circle_filled( + self.__draw_circle_filled( screen_point, size=self._MARKER_CIRCLE_SIZE_FEEDBACK * radius, color=RGBA(*marker_circle_rgb_feedback, alpha), @@ -528,37 +532,3 @@ def _interp_fn(t, b, c, d, start_sample=15.0, stop_sample=55.0): return 1 - _easeInOutQuad(t - stop_sample, b, c, d - stop_sample) else: return 1.0 - - -@functools.lru_cache(4) # 4 circles needed to draw calibration marker -def _circle_points_around_zero(radius: float, num_points: int) -> np.ndarray: - t = np.linspace(0, 2 * np.pi, num_points, dtype=np.float64) - t.shape = -1, 1 - points = np.hstack([np.cos(t), np.sin(t)]) - points *= radius - return points - - -@functools.lru_cache(4) # 4 circles needed to draw calibration marker -def _circle_points_offset( - offset: T.Tuple[float, float], radius: float, num_points: int, flat: bool = True -) -> np.ndarray: - # NOTE: .copy() to avoid modifying the cached result - points = _circle_points_around_zero(radius, num_points).copy() - points[:, 0] += offset[0] - points[:, 1] += offset[1] - if flat: - points.shape = -1 - return points - - -def _draw_circle_filled( - screen_point: T.Tuple[float, float], size: float, color: RGBA, num_points: int = 50 -): - points = _circle_points_offset( - screen_point, radius=size, num_points=num_points, flat=False - ) - gl.glColor4f(color.r, color.g, color.b, color.a) - gl.glEnableClientState(gl.GL_VERTEX_ARRAY) - gl.glVertexPointer(2, gl.GL_DOUBLE, 0, points) - gl.glDrawArrays(gl.GL_POLYGON, 0, points.shape[0]) diff --git a/pupil_src/shared_modules/camera_intrinsics_estimation.py b/pupil_src/shared_modules/camera_intrinsics_estimation.py index 0e688f1636..399c03adf6 100644 --- a/pupil_src/shared_modules/camera_intrinsics_estimation.py +++ b/pupil_src/shared_modules/camera_intrinsics_estimation.py @@ -34,6 +34,8 @@ from plugin import Plugin +from hotkey import Hotkey + # logging import logging @@ -153,7 +155,11 @@ def get_monitors_idx_list(): ) self.button = ui.Thumb( - "collect_new", self, setter=self.advance, label="I", hotkey="i" + "collect_new", + self, + setter=self.advance, + label="I", + hotkey=Hotkey.CAMERA_INTRINSIC_ESTIMATOR_COLLECT_NEW_CAPTURE_HOTKEY(), ) self.button.on_color[:] = (0.3, 0.2, 1.0, 0.9) self.g_pool.quickbar.insert(0, self.button) diff --git a/pupil_src/shared_modules/display_recent_gaze.py b/pupil_src/shared_modules/display_recent_gaze.py index ff54caaa33..6e6ef0ace2 100644 --- a/pupil_src/shared_modules/display_recent_gaze.py +++ b/pupil_src/shared_modules/display_recent_gaze.py @@ -11,6 +11,8 @@ from plugin import System_Plugin_Base from pyglui.cygl.utils import draw_points_norm, RGBA +from gl_utils import draw_circle_filled_func_builder +from methods import denormalize class Display_Recent_Gaze(System_Plugin_Base): @@ -23,16 +25,24 @@ def __init__(self, g_pool): super().__init__(g_pool) self.order = 0.8 self.pupil_display_list = [] + self._draw_circle_filled = draw_circle_filled_func_builder() def recent_events(self, events): for pt in events.get("gaze", []): - self.pupil_display_list.append((pt["norm_pos"], pt["confidence"] * 0.8)) + recent_frame_size = self.g_pool.capture.frame_size + point = denormalize(pt["norm_pos"], recent_frame_size, flip_y=True) + self.pupil_display_list.append((point, pt["confidence"] * 0.8)) + self.pupil_display_list[:-3] = [] def gl_display(self): for pt, a in self.pupil_display_list: # This could be faster if there would be a method to also add multiple colors per point - draw_points_norm([pt], size=35, color=RGBA(1.0, 0.2, 0.4, a)) + self._draw_circle_filled( + tuple(pt), + size=35 / 2, + color=RGBA(1.0, 0.2, 0.4, a), + ) def get_init_dict(self): return {} diff --git a/pupil_src/shared_modules/file_methods.py b/pupil_src/shared_modules/file_methods.py index 3afcf892c8..e11a4bca45 100644 --- a/pupil_src/shared_modules/file_methods.py +++ b/pupil_src/shared_modules/file_methods.py @@ -42,7 +42,9 @@ def __init__(self, file_path, *args, **kwargs): super().__init__(*args, **kwargs) self.file_path = os.path.expanduser(file_path) try: - self.update(**load_object(self.file_path, allow_legacy=False)) + if os.path.getsize(file_path) > 0: + # Only try to load object if file is not empty + self.update(**load_object(self.file_path, allow_legacy=False)) except IOError: logger.debug( f"Session settings file '{self.file_path}' not found." diff --git a/pupil_src/shared_modules/fixation_detector.py b/pupil_src/shared_modules/fixation_detector.py index 4853f89915..8dccf572b2 100644 --- a/pupil_src/shared_modules/fixation_detector.py +++ b/pupil_src/shared_modules/fixation_detector.py @@ -51,6 +51,7 @@ from methods import denormalize from plugin import Plugin from pupil_recording import PupilRecording, RecordingInfo +from hotkey import Hotkey logger = logging.getLogger(__name__) @@ -415,7 +416,7 @@ def jump_prev_fixation(_): setter=jump_next_fixation, getter=lambda: False, label=chr(0xE044), - hotkey="f", + hotkey=Hotkey.FIXATION_NEXT_PLAYER_HOTKEY(), label_font="pupil_icons", ) self.next_fix_button.status_text = "Next Fixation" @@ -426,7 +427,7 @@ def jump_prev_fixation(_): setter=jump_prev_fixation, getter=lambda: False, label=chr(0xE045), - hotkey="F", + hotkey=Hotkey.FIXATION_PREV_PLAYER_HOTKEY(), label_font="pupil_icons", ) self.prev_fix_button.status_text = "Previous Fixation" diff --git a/pupil_src/shared_modules/gl_utils/__init__.py b/pupil_src/shared_modules/gl_utils/__init__.py index 9b197532b0..1edea23251 100644 --- a/pupil_src/shared_modules/gl_utils/__init__.py +++ b/pupil_src/shared_modules/gl_utils/__init__.py @@ -8,6 +8,7 @@ See COPYING and COPYING.LESSER for license details. ---------------------------------------------------------------------------~(*) """ +from .draw import draw_circle_filled_func_builder from .trackball import Trackball from .utils import ( _Rectangle, @@ -15,6 +16,7 @@ basic_gl_setup, clear_gl_screen, Coord_System, + current_context, cvmat_to_glmat, get_content_scale, get_framebuffer_scale, diff --git a/pupil_src/shared_modules/gl_utils/draw.py b/pupil_src/shared_modules/gl_utils/draw.py new file mode 100644 index 0000000000..8a751518c6 --- /dev/null +++ b/pupil_src/shared_modules/gl_utils/draw.py @@ -0,0 +1,82 @@ +""" +(*)~--------------------------------------------------------------------------- +Pupil - eye tracking platform +Copyright (C) 2012-2021 Pupil Labs + +Distributed under the terms of the GNU +Lesser General Public License (LGPL v3.0). +See COPYING and COPYING.LESSER for license details. +---------------------------------------------------------------------------~(*) +""" +import functools +import typing as T + +import numpy as np +import OpenGL.GL as gl +from pyglui.cygl.utils import RGBA + + +def draw_circle_filled_func_builder(cache_size: int = 0): + _circle_points_around_zero_f = _circle_points_around_zero + _circle_points_offset_f = _circle_points_offset + + if cache_size > 0: + _circle_points_around_zero_f = functools.lru_cache(cache_size)( + _circle_points_around_zero_f + ) + _circle_points_offset_f = functools.lru_cache(cache_size)( + _circle_points_offset_f + ) + + return functools.partial( + _draw_circle_filled, + _circle_points_around_zero_f=_circle_points_around_zero_f, + _circle_points_offset_f=_circle_points_offset_f, + ) + + +def _draw_circle_filled( + screen_point: T.Tuple[float, float], + size: float, + color: RGBA, + num_points: int = 50, + *, + _circle_points_around_zero_f, + _circle_points_offset_f, +): + points = _circle_points_offset_f( + screen_point, + radius=size, + num_points=num_points, + flat=False, + _circle_points_around_zero_f=_circle_points_around_zero_f, + ) + gl.glColor4f(color.r, color.g, color.b, color.a) + gl.glEnableClientState(gl.GL_VERTEX_ARRAY) + gl.glVertexPointer(2, gl.GL_DOUBLE, 0, points) + gl.glDrawArrays(gl.GL_POLYGON, 0, points.shape[0]) + + +def _circle_points_offset( + offset: T.Tuple[float, float], + radius: float, + num_points: int, + flat: bool = True, + *, + _circle_points_around_zero_f, +) -> np.ndarray: + # NOTE: .copy() to avoid modifying the cached result + points = _circle_points_around_zero_f(radius, num_points).copy() + points[:, 0] += offset[0] + points[:, 1] += offset[1] + if flat: + points.shape = -1 + return points + + +def _circle_points_around_zero(radius: float, num_points: int) -> np.ndarray: + t = np.linspace(0, 2 * np.pi, num_points, dtype=np.float64) + t.shape = -1, 1 + points = np.hstack([np.cos(t), np.sin(t)]) + points *= radius + return points diff --git a/pupil_src/shared_modules/gl_utils/utils.py b/pupil_src/shared_modules/gl_utils/utils.py index 956b05089b..a0690919d7 100644 --- a/pupil_src/shared_modules/gl_utils/utils.py +++ b/pupil_src/shared_modules/gl_utils/utils.py @@ -331,6 +331,16 @@ def get_window_title_bar_rect(window) -> _Rectangle: ) +@contextlib.contextmanager +def current_context(window): + prev_context = glfw.get_current_context() + glfw.make_context_current(window) + try: + yield + finally: + glfw.make_context_current(prev_context) + + _GLFWErrorReportingDict = T.Dict[T.Union[None, int], str] diff --git a/pupil_src/shared_modules/hololens_relay.py b/pupil_src/shared_modules/hololens_relay.py index 94613aa087..e5ca3af75e 100644 --- a/pupil_src/shared_modules/hololens_relay.py +++ b/pupil_src/shared_modules/hololens_relay.py @@ -393,7 +393,7 @@ def on_recv(self, socket, ipc_pub): calib_method = "HMD_Calibration_3D" ipc_pub.notify({"subject": "start_plugin", "name": calib_method}) - ipc_pub.notify({"subject": "set_pupil_detection_enabled", "value": True}) + ipc_pub.notify({"subject": "pupil_detector.set_enabled", "value": True}) ipc_pub.notify( { "subject": "eye_process.should_start.{}".format(0), diff --git a/pupil_src/shared_modules/hotkey.py b/pupil_src/shared_modules/hotkey.py new file mode 100644 index 0000000000..5446ff5710 --- /dev/null +++ b/pupil_src/shared_modules/hotkey.py @@ -0,0 +1,191 @@ +""" +(*)~--------------------------------------------------------------------------- +Pupil - eye tracking platform +Copyright (C) 2012-2021 Pupil Labs + +Distributed under the terms of the GNU +Lesser General Public License (LGPL v3.0). +See COPYING and COPYING.LESSER for license details. +---------------------------------------------------------------------------~(*) +""" + + +class Hotkey: + """""" + + @staticmethod + def ANNOTATION_EVENT_DEFAULT_HOTKEY(): + """Add annotation (default keyboard shortcut) + + Capture Order: 40 + Player Order: 50 + """ + return "x" + + @staticmethod + def CAMERA_INTRINSIC_ESTIMATOR_COLLECT_NEW_CAPTURE_HOTKEY(): + """Camera intrinsic estimation: Take snapshot of circle pattern + + Capture Order: 50 + """ + return "i" + + @staticmethod + def EXPORT_START_PLAYER_HOTKEY(): + """Start export + + Player Order: 30 + """ + return "e" + + @staticmethod + def FIXATION_NEXT_PLAYER_HOTKEY(): + """Fixation: Show next + + Player Order: 60 + """ + return "f" + + @staticmethod + def FIXATION_PREV_PLAYER_HOTKEY(): + """Fixation: Show previous + + Player Order: 61 + """ + return "F" + + @staticmethod + def GAZE_CALIBRATION_CAPTURE_HOTKEY(): + """Start and stop calibration + + Capture Order: 20 + """ + return "c" + + @staticmethod + def GAZE_VALIDATION_CAPTURE_HOTKEY(): + """Start and stop validation + + Capture Order: 21 + """ + return "t" + + @staticmethod + def RECORDER_RUNNING_TOGGLE_CAPTURE_HOTKEY(): + """Start and stop recording + + Capture Order: 10 + """ + return "r" + + @staticmethod + def SURFACE_TRACKER_ADD_SURFACE_CAPTURE_AND_PLAYER_HOTKEY(): + """Surface tracker: Add new surface + + Capture Order: 30 + Player Order: 40 + """ + return "a" + + @staticmethod + def SEEK_BAR_MOVE_BACKWARDS_PLAYER_HOTKEY(): + # This is only implicitly used by pyglui.ui.Seek_Bar + """Step to previous frame\\* / Decrease playback speed\\*\\* + + Printable: + Player Order: 20 + """ + return 263 + + @staticmethod + def SEEK_BAR_MOVE_FORWARDS_PLAYER_HOTKEY(): + # This is only implicitly used by pyglui.ui.Seek_Bar + """Step to next frame\\* / Increase playback speed\\*\\* + + Printable: + Player Order: 21 + """ + return 262 + + @staticmethod + def SEEK_BAR_PLAY_PAUSE_PLAYER_HOTKEY(): + # This is only implicitly used by pyglui.ui.Seek_Bar + """Play and pause video + + Printable: + Player Order: 10 + """ + return 32 + + +def generate_markdown_hotkey_docs() -> str: + import pandas as pd + + def generate_row(hotkey_id, hotkey_method): + hotkey_value = hotkey_method.__get__(Hotkey)() + hotkey_docsring = hotkey_method.__get__(Hotkey).__doc__ + doc_lines = [l.strip() for l in hotkey_docsring.split("\n") if len(l.strip())] + + if len(doc_lines) > 0: + hotkey_descr = doc_lines[0] + else: + hotkey_descr = "" + + if len(doc_lines) > 1: + hotkey_meta = dict( + tuple(map(str.strip, l.split(":"))) for l in doc_lines[1:] + ) + else: + hotkey_meta = {} + + hotkey_printable = hotkey_meta.get("Printable") + hotkey_order_in_capture = hotkey_meta.get("Capture Order", None) + hotkey_order_in_player = hotkey_meta.get("Player Order", None) + + return { + "_ID": hotkey_id, + "_Order_In_Capture": hotkey_order_in_capture, + "_Order_In_Player": hotkey_order_in_player, + "Keyboard Shortcut": f"`{hotkey_printable or hotkey_value}`", + "Description": hotkey_descr, + } + + hotkeys_df = pd.DataFrame( + [ + generate_row(hotkey_id, hotkey_method) + for hotkey_id, hotkey_method in vars(Hotkey).items() + if hotkey_id.endswith("_HOTKEY") + ] + ) + hotkeys_df = hotkeys_df.set_index(["Keyboard Shortcut"]) + + # Only show columns that don't start with an underscore + visible_columns = [c for c in hotkeys_df.columns if not c.startswith("_")] + + capture_df = hotkeys_df[hotkeys_df["_Order_In_Capture"].notnull()] + capture_df = capture_df.sort_values(by=["_Order_In_Capture"]) + + player_df = hotkeys_df[hotkeys_df["_Order_In_Player"].notnull()] + player_df = player_df.sort_values(by=["_Order_In_Player"]) + + capture_title_md = "# Pupil Capture" + capture_table_md = capture_df[visible_columns].to_markdown() + + player_title_md = "# Pupil Player" + player_table_md = player_df[visible_columns].to_markdown() + + player_footnote = "\\* While paused\n\\* During playback" + + return "\n" + "\n\n".join( + [ + capture_title_md, + capture_table_md, + player_title_md, + player_table_md, + player_footnote, + ] + ) + + +if __name__ == "__main__": + print(generate_markdown_hotkey_docs()) diff --git a/pupil_src/shared_modules/network_time_sync.py b/pupil_src/shared_modules/network_time_sync.py index 1fdd53c7ed..0e71e031e7 100644 --- a/pupil_src/shared_modules/network_time_sync.py +++ b/pupil_src/shared_modules/network_time_sync.py @@ -9,11 +9,12 @@ ---------------------------------------------------------------------------~(*) """ +import functools from time import sleep from uvc import get_time_monotonic import socket +import socketserver import threading -import asyncore import struct from random import random @@ -36,49 +37,45 @@ """ -class Time_Echo(asyncore.dispatcher_with_send): +class Time_Echo(socketserver.BaseRequestHandler): """ Subclass do not use directly! reply to request with timestamp """ - def __init__(self, sock, time_fn): + def __init__(self, *args, time_fn, **kwargs): self.time_fn = time_fn - asyncore.dispatcher_with_send.__init__(self, sock) + super().__init__(*args, **kwargs) - def handle_read(self): - # expecting `sync` message - data = self.recv(4) - if data: - self.send(struct.pack(" str: + return self.server_address[0] + + @property + def port(self) -> int: + return self.server_address[1] def __del__(self): logger.debug("Server closed") @@ -92,20 +89,15 @@ class Clock_Sync_Master(threading.Thread): def __init__(self, time_fn): threading.Thread.__init__(self) - self.socket_map = {} - self.server = Time_Echo_Server(time_fn, self.socket_map) + self.server = Time_Echo_Server(time_fn=time_fn) self.start() def run(self): - asyncore.loop(use_poll=True, timeout=1) + self.server.serve_forever() def stop(self): - # we dont use server.close() as this raises a bad file decritoor exception in loop - self.server.connected = False - self.server.accepting = False - self.server.del_channel() + self.server.shutdown() self.join() - self.server.socket.close() logger.debug("Server Thread closed") def terminate(self): @@ -211,20 +203,19 @@ def run(self): def _get_offset(self): try: - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.settimeout(1.0) - server_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - server_socket.connect((self.host, self.port)) - times = [] - for request in range(60): - t0 = self.get_time() - server_socket.send(b"sync") - message = server_socket.recv(8) - t2 = self.get_time() - t1 = struct.unpack(" str: def pupil_data_source_selection_label(cls) -> str: return cls.plugin_menu_label() + @staticmethod + def available_pupil_producer_plugins(g_pool) -> list: + def is_plugin_included(p, g_pool) -> bool: + # Skip plugins that are not pupil producers + if not issubclass(p, Pupil_Producer_Base): + return False + # Skip pupil producer stub + if p is DisabledPupilProducer: + return False + # Skip pupil producers that are not available within g_pool context + if not p.is_available_within_context(g_pool): + return False + return True + + return [ + p for p in g_pool.plugin_by_name.values() if is_plugin_included(p, g_pool) + ] + @classmethod def pupil_data_source_selection_order(cls) -> float: return float("inf") @@ -76,17 +94,7 @@ def __init__(self, g_pool): def init_ui(self): self.add_menu() - pupil_producer_plugins = [ - p - for p in self.g_pool.plugin_by_name.values() - if issubclass(p, Pupil_Producer_Base) - ] - # Skip pupil producers that are not available within g_pool context - pupil_producer_plugins = [ - p - for p in pupil_producer_plugins - if p.is_available_within_context(self.g_pool) - ] + pupil_producer_plugins = self.available_pupil_producer_plugins(self.g_pool) pupil_producer_plugins.sort(key=lambda p: p.pupil_data_source_selection_label()) pupil_producer_plugins.sort(key=lambda p: p.pupil_data_source_selection_order()) pupil_producer_labels = [ @@ -291,6 +299,53 @@ def _legend_font(self, scale): self.glfont.pop_state() +class DisabledPupilProducer(Pupil_Producer_Base): + """ + This is a stub implementation of a pupil producer, + intended to be used when no (other) pupil producer is available. + """ + + @classmethod + def is_available_within_context(cls, g_pool) -> bool: + if g_pool.app == "player": + recording = PupilRecording(rec_dir=g_pool.rec_dir) + meta_info = recording.meta_info + if ( + meta_info.recording_software_name + == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_INVISIBLE + ): + # Enable in Player only if Pupil Invisible recording + return True + return False + + @classmethod + def plugin_menu_label(cls) -> str: + raise RuntimeError() # This method should never be called + return "Disabled Pupil Producer" + + @classmethod + def pupil_data_source_selection_order(cls) -> float: + raise RuntimeError() # This method should never be called + return 0.1 + + def __init__(self, g_pool): + super().__init__(g_pool) + # Create empty pupil_positions for all plugins that depend on it + pupil_data = pm.PupilDataBisector(data=fm.PLData([], [], [])) + g_pool.pupil_positions = pupil_data + self._pupil_changed_announcer.announce_existing() + logger.debug("pupil positions changed") + + def init_ui(self): + pass + + def deinit_ui(self): + pass + + def _refresh_timelines(self): + pass + + class Pupil_From_Recording(Pupil_Producer_Base): @classmethod def is_available_within_context(cls, g_pool) -> bool: @@ -303,6 +358,12 @@ def is_available_within_context(cls, g_pool) -> bool: ): # Disable pupil from recording in Player if Pupil Mobile recording return False + if ( + meta_info.recording_software_name + == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_INVISIBLE + ): + # Disable pupil from recording in Player if Pupil Invisible recording + return False return super().is_available_within_context(g_pool) @classmethod diff --git a/pupil_src/shared_modules/pupil_recording/info/recording_info_utils.py b/pupil_src/shared_modules/pupil_recording/info/recording_info_utils.py index 1b8c78c641..a953b77702 100644 --- a/pupil_src/shared_modules/pupil_recording/info/recording_info_utils.py +++ b/pupil_src/shared_modules/pupil_recording/info/recording_info_utils.py @@ -79,17 +79,34 @@ def f(value): def read_info_csv_file(rec_dir: str) -> dict: + """Read `info.csv` file from recording.""" file_path = os.path.join(rec_dir, "info.csv") with open(file_path, "r") as file: return csv_utils.read_key_value_file(file) def read_info_json_file(rec_dir: str) -> dict: + """Read `info.json` file from recording.""" file_path = os.path.join(rec_dir, "info.json") with open(file_path, "r") as file: return json.load(file) +def read_info_invisible_json_file(rec_dir: str) -> dict: + """Read `info.invisible.json` file from recording.""" + file_path = os.path.join(rec_dir, "info.invisible.json") + with open(file_path, "r") as file: + return json.load(file) + + +def read_pupil_invisible_info_file(rec_dir: str) -> dict: + """Read info file from Pupil Invisible recording.""" + try: + return read_info_json_file(rec_dir) + except FileNotFoundError: + return read_info_invisible_json_file(rec_dir) + + def parse_duration_string(duration_string: str) -> int: """Returns number of seconds from string 'HH:MM:SS'.""" H, M, S = [int(part) for part in duration_string.split(":")] diff --git a/pupil_src/shared_modules/pupil_recording/update/invisible.py b/pupil_src/shared_modules/pupil_recording/update/invisible.py index 4926fdf54c..b8f84d4871 100644 --- a/pupil_src/shared_modules/pupil_recording/update/invisible.py +++ b/pupil_src/shared_modules/pupil_recording/update/invisible.py @@ -137,14 +137,18 @@ def _pi_path_core_path_pairs(recording: PupilRecording): def _rewrite_timestamps(recording: PupilRecording): - start_time = recording.meta_info.start_time_synced_ns + + # Use start time from info file (instead of recording.meta_info.start_time_synced_ns) + # to have a more precise value and avoid having a negative first timestamp when rewriting + info_json = utils.read_pupil_invisible_info_file(recording.rec_dir) + start_time_synced_ns = int(info_json["start_time"]) def conversion(timestamps: np.array): # Subtract start_time from all times in the recording, so timestamps # start at 0. This is to increase precision when converting # timestamps to float32, e.g. for OpenGL! SECONDS_PER_NANOSECOND = 1e-9 - return (timestamps - start_time) * SECONDS_PER_NANOSECOND + return (timestamps - start_time_synced_ns) * SECONDS_PER_NANOSECOND update_utils._rewrite_times(recording, dtype=" bool: + producers = Pupil_Producer_Base.available_pupil_producer_plugins(self.g_pool) + return len(producers) > 0 + def init_ui(self): self.add_menu() self.menu.label = "Raw Data Exporter" @@ -129,11 +142,13 @@ def init_ui(self): "Select your export frame range using the trim marks in the seek bar. This will affect all exporting plugins." ) ) - self.menu.append( - ui.Switch( - "should_export_pupil_positions", self, label="Export Pupil Positions" - ) + + pupil_positions_switch = ui.Switch( + "should_export_pupil_positions", self, label="Export Pupil Positions" ) + pupil_positions_switch.read_only = not self._is_pupil_producer_avaiable + self.menu.append(pupil_positions_switch) + self.menu.append( ui.Switch( "should_export_field_info", diff --git a/pupil_src/shared_modules/recorder.py b/pupil_src/shared_modules/recorder.py index e1a131c5b8..698fc94f4b 100644 --- a/pupil_src/shared_modules/recorder.py +++ b/pupil_src/shared_modules/recorder.py @@ -38,6 +38,8 @@ # from scipy.interpolate import UnivariateSpline from plugin import System_Plugin_Base +from hotkey import Hotkey + logger = logging.getLogger(__name__) @@ -185,7 +187,11 @@ def init_ui(self): ) ) self.button = ui.Thumb( - "running", self, setter=self.toggle, label="R", hotkey="r" + "running", + self, + setter=self.toggle, + label="R", + hotkey=Hotkey.RECORDER_RUNNING_TOGGLE_CAPTURE_HOTKEY(), ) self.button.on_color[:] = (1, 0.0, 0.0, 0.8) self.g_pool.quickbar.insert(2, self.button) diff --git a/pupil_src/shared_modules/surface_tracker/background_tasks.py b/pupil_src/shared_modules/surface_tracker/background_tasks.py index 1573db71c5..e1bd860432 100644 --- a/pupil_src/shared_modules/surface_tracker/background_tasks.py +++ b/pupil_src/shared_modules/surface_tracker/background_tasks.py @@ -19,6 +19,10 @@ import background_helper import player_methods +import file_methods + +from .surface_marker import Surface_Marker + logger = logging.getLogger(__name__) @@ -221,6 +225,7 @@ def get_export_proxy( gaze_positions, fixations, camera_model, + marker_cache_path, mp_context, ): exporter = Exporter( @@ -231,6 +236,7 @@ def get_export_proxy( gaze_positions, fixations, camera_model, + marker_cache_path, ) proxy = background_helper.IPC_Logging_Task_Proxy( "Offline Surface Tracker Exporter", @@ -250,6 +256,7 @@ def __init__( gaze_positions, fixations, camera_model, + marker_cache_path, ): self.export_range = export_range self.metrics_dir = os.path.join(export_dir, "surfaces") @@ -260,6 +267,7 @@ def __init__( self.camera_model = camera_model self.gaze_on_surfaces = None self.fixations_on_surfaces = None + self.marker_cache_path = marker_cache_path def save_surface_statisics_to_file(self): logger.info("exporting metrics to {}".format(self.metrics_dir)) @@ -298,6 +306,17 @@ def save_surface_statisics_to_file(self): "Saved surface gaze and fixation data for '{}'".format(surface.name) ) + # Cleanup surface related data to release memory + self.surfaces = None + self.fixations = None + self.gaze_positions = None + self.gaze_on_surfaces = None + self.fixations_on_surfaces = None + + # Perform marker export *after* surface data is released + # to avoid holding everything in memory all at once. + self._export_marker_detections() + logger.info("Done exporting reference surface data.") return @@ -338,6 +357,46 @@ def _map_gaze_and_fixations(self): return gaze_on_surface, fixations_on_surface + def _export_marker_detections(self): + + # Load the temporary marker cache created by the offline surface tracker + marker_cache = file_methods.Persistent_Dict(self.marker_cache_path) + marker_cache = marker_cache["marker_cache"] + + try: + file_path = os.path.join(self.metrics_dir, "marker_detections.csv") + with open(file_path, "w", encoding="utf-8", newline="") as csv_file: + csv_writer = csv.writer(csv_file, delimiter=",") + csv_writer.writerow( + ( + "world_index", + "marker_uid", + "corner_0_x", + "corner_0_y", + "corner_1_x", + "corner_1_y", + "corner_2_x", + "corner_2_y", + "corner_3_x", + "corner_3_y", + ) + ) + for idx, serialized_markers in enumerate(marker_cache): + for m in map(Surface_Marker.deserialize, serialized_markers): + flat_corners = [x for c in m.verts_px for x in c[0]] + assert len(flat_corners) == 8 # sanity check + csv_writer.writerow( + ( + idx, + m.uid, + *flat_corners, + ) + ) + finally: + # Delete the temporary marker cache created by the offline surface tracker + os.remove(self.marker_cache_path) + self.marker_cache_path = None + def _export_surface_visibility(self): with open( os.path.join(self.metrics_dir, "surface_visibility.csv"), @@ -481,8 +540,10 @@ def _export_surface_positions(self, surface, surface_name): "num_detected_markers", "dist_img_to_surf_trans", "surf_to_dist_img_trans", + "num_definition_markers", ) ) + surface_definition_marker_count = len(surface.registered_marker_uids) for idx, (ts, ref_surf_data) in enumerate( zip(self.world_timestamps, surface.location_cache) ): @@ -501,6 +562,7 @@ def _export_surface_positions(self, surface, surface_name): ref_surf_data.num_detected_markers, ref_surf_data.dist_img_to_surf_trans, ref_surf_data.surf_to_dist_img_trans, + surface_definition_marker_count, ) ) diff --git a/pupil_src/shared_modules/surface_tracker/surface.py b/pupil_src/shared_modules/surface_tracker/surface.py index e1d1120359..0b0b3fbcc7 100644 --- a/pupil_src/shared_modules/surface_tracker/surface.py +++ b/pupil_src/shared_modules/surface_tracker/surface.py @@ -139,6 +139,10 @@ def property_dict(x: Surface) -> dict: def defined(self): return self.build_up_status >= 1.0 + @property + def registered_marker_uids(self) -> typing.Set[Surface_Marker_UID]: + return set(self._registered_markers_dist.keys()) + @property def registered_markers_dist(self) -> Surface_Marker_UID_To_Aggregate_Mapping: return self._registered_markers_dist @@ -291,11 +295,8 @@ def locate( registered_markers_undist.keys() ) - # If the surface is defined by 2+ markers, we require 2+ markers to be detected. - # If the surface is defined by 1 marker, we require 1 marker to be detected. - if not visible_registered_marker_ids or len( - visible_registered_marker_ids - ) < min(2, len(registered_markers_undist)): + # If no surface marker is detected, return + if not visible_registered_marker_ids: return Surface_Location(detected=False) visible_verts_dist = np.array( diff --git a/pupil_src/shared_modules/surface_tracker/surface_tracker.py b/pupil_src/shared_modules/surface_tracker/surface_tracker.py index 1ede943df4..e93dd96712 100644 --- a/pupil_src/shared_modules/surface_tracker/surface_tracker.py +++ b/pupil_src/shared_modules/surface_tracker/surface_tracker.py @@ -26,6 +26,7 @@ MarkerType, ApriltagFamily, ) +from hotkey import Hotkey logger = logging.getLogger(__name__) @@ -139,7 +140,7 @@ def init_ui(self): setter=self.on_add_surface_click, getter=lambda: False, label="A", - hotkey="a", + hotkey=Hotkey.SURFACE_TRACKER_ADD_SURFACE_CAPTURE_AND_PLAYER_HOTKEY(), ) self.g_pool.quickbar.append(self.add_button) self._update_ui() diff --git a/pupil_src/shared_modules/surface_tracker/surface_tracker_offline.py b/pupil_src/shared_modules/surface_tracker/surface_tracker_offline.py index 86cdbc4550..9094085155 100644 --- a/pupil_src/shared_modules/surface_tracker/surface_tracker_offline.py +++ b/pupil_src/shared_modules/surface_tracker/surface_tracker_offline.py @@ -14,6 +14,7 @@ import multiprocessing import os import platform +import tempfile import time import typing as T @@ -559,6 +560,16 @@ def on_notify(self, notification): break elif notification["subject"] == "should_export": + # Create new marker cache temporary file + # Backgroud exporter is responsible of removing the temporary file when finished + file_handle, marker_cache_path = tempfile.mkstemp() + os.close(file_handle) # https://bugs.python.org/issue42830 + + # Save marker cache into the new temporary file + temp_marker_cache = file_methods.Persistent_Dict(marker_cache_path) + temp_marker_cache["marker_cache"] = self.marker_cache + temp_marker_cache.save() + proxy = background_tasks.get_export_proxy( notification["export_dir"], notification["range"], @@ -567,6 +578,7 @@ def on_notify(self, notification): self.g_pool.gaze_positions, self.g_pool.fixations, self.camera_model, + marker_cache_path, mp_context, ) self.export_proxies.add(proxy) diff --git a/requirements.txt b/requirements.txt index 12ea29d8af..a226763c86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ opencv-python==3.* ; platform_system == "Windows" ### pupil-apriltags==1.0.4 pupil-detectors==2.0.* -pye3d==0.0.6 +pye3d==0.0.7 # pupil-labs/PyAV 0.4.6 av @ git+https://github.com/pupil-labs/PyAV@v0.4.6 ; platform_system != "Windows"