Merge pull request #1802 from pupil-labs/restructured-gaze-calibratio…

…n-and-mapping

Restructured gaze calibration and mapping

.travis/run_tests.sh

@@ -1,6 +1,7 @@
 python -m pip install -U pip
 
 pip install -U git+https://github.com/pupil-labs/pyndsi
+pip install scikit-learn
 
 pip install pytest==5.2.2
 pytest

docs/dependencies-macos.md

@@ -78,6 +78,7 @@ pip install psutil
 pip install pyaudio
 pip install pyopengl
 pip install pyzmq
+pip install scikit-learn
 pip install scipy
 pip install git+https://github.com/zeromq/pyre
 

docs/dependencies-ubuntu17.md

@@ -159,6 +159,7 @@ pip install psutil
 pip install pyaudio
 pip install pyopengl
 pip install pyzmq
+pip install scikit-learn
 pip install scipy
 pip install git+https://github.com/zeromq/pyre
 

docs/dependencies-ubuntu18.md

@@ -64,6 +64,7 @@ pip install psutil
 pip install pyaudio
 pip install pyopengl
 pip install pyzmq
+pip install scikit-learn
 pip install scipy
 pip install git+https://github.com/zeromq/pyre
 

docs/dependencies-windows.md

@@ -61,6 +61,7 @@ pip install psutil
 pip install pyaudio
 pip install pyopengl
 pip install pyzmq
+pip install scikit-learn
 pip install scipy
 pip install win_inet_pton
 pip install git+https://github.com/zeromq/pyre

pupil_src/launchables/eye.py

@@ -71,7 +71,6 @@ def eye(
     Streams Pupil coordinates.
 
     Reacts to notifications:
-        ``set_detection_mapping_mode``: Sets detection method
         ``eye_process.should_stop``: Stops the eye process
         ``recording.started``: Starts recording eye video
         ``recording.stopped``: Stops recording eye video
@@ -147,8 +146,7 @@ def eye(
         from roi import Roi
 
         from background_helper import IPC_Logging_Task_Proxy
-        from pupil_detector_plugins import available_detector_plugins
-        from pupil_detector_plugins.manager import PupilDetectorManager
+        from pupil_detector_plugins import available_detector_plugins, EVENT_KEY
 
         IPC_Logging_Task_Proxy.push_url = ipc_push_url
 
@@ -197,13 +195,8 @@ def get_timestamp():
         g_pool.get_timestamp = get_timestamp
         g_pool.get_now = get_time_monotonic
 
-        default_detector_cls, available_detectors = available_detector_plugins()
-        plugins = (
-            manager_classes
-            + source_classes
-            + available_detectors
-            + [PupilDetectorManager, Roi]
-        )
+        default_2d, default_3d, available_detectors = available_detector_plugins()
+        plugins = manager_classes + source_classes + available_detectors + [Roi]
         g_pool.plugin_by_name = {p.__name__: p for p in plugins}
 
         preferred_names = [
@@ -225,11 +218,12 @@ def get_timestamp():
             # TODO: extend with plugins
             (default_capture_name, default_capture_settings),
             ("UVC_Manager", {}),
+            # Detectors needs to be loaded first to set `g_pool.pupil_detector`
+            (default_2d.__name__, {}),
+            (default_3d.__name__, {}),
             ("NDSI_Manager", {}),
             ("HMD_Streaming_Manager", {}),
             ("File_Manager", {}),
-            # Detector needs to be loaded first to set `g_pool.pupil_detector`
-            (default_detector_cls.__name__, {}),
             ("PupilDetectorManager", {}),
             ("Roi", {}),
         ]
@@ -598,8 +592,7 @@ def window_should_update():
                             {"subject": "recording.should_stop", "remote_notify": "all"}
                         )
 
-                result = event.get("pupil_detection_result", None)
-                if result is not None:
+                for result in event.get(EVENT_KEY, ()):
                     pupil_socket.send(result)
 
             cpu_graph.update()

pupil_src/launchables/player.py

@@ -323,8 +323,7 @@ def set_scale(new_scale):
         )
 
         # populated by producers
-        g_pool.pupil_positions = pm.Bisector()
-        g_pool.pupil_positions_by_id = (pm.Bisector(), pm.Bisector())
+        g_pool.pupil_positions = pm.PupilDataBisector()
         g_pool.gaze_positions = pm.Bisector()
         g_pool.fixations = pm.Affiliator()
         g_pool.eye_movements = pm.Affiliator()

pupil_src/launchables/service.py

@@ -29,15 +29,15 @@ def service(
     """Maps pupil to gaze data, can run various plug-ins.
 
     Reacts to notifications:
-       ``set_detection_mapping_mode``: Sets detection method
+       ``set_pupil_detection_enabled``: Sets detection enabled
        ``start_plugin``: Starts given plugin with the given arguments
        ``eye_process.started``: Sets the detection method eye process
        ``service_process.should_stop``: Stops the service process
 
     Emits notifications:
         ``eye_process.should_start``
         ``eye_process.should_stop``
-        ``set_detection_mapping_mode``
+        ``set_pupil_detection_enabled``
         ``service_process.started``
         ``service_process.stopped``
         ``launcher_process.should_stop``
@@ -95,7 +95,11 @@ def stop_eye_process(eye_id):
 
         # Plug-ins
         from plugin import Plugin, Plugin_List, import_runtime_plugins
-        from calibration_routines import calibration_plugins, gaze_mapping_plugins
+        from calibration_choreography import (
+            available_calibration_choreography_plugins,
+            patch_loaded_plugins_with_choreography_plugin,
+        )
+        from gaze_mapping import registered_gazer_classes
         from pupil_remote import Pupil_Remote
         from pupil_groups import Pupil_Groups
         from frame_publisher import Frame_Publisher
@@ -142,6 +146,8 @@ def get_timestamp():
 
         g_pool.get_timestamp = get_timestamp
 
+        available_choreography_plugins = available_calibration_choreography_plugins()
+
         # manage plugins
         runtime_plugins = import_runtime_plugins(
             os.path.join(g_pool.user_dir, "plugins")
@@ -155,16 +161,17 @@ def get_timestamp():
         ] + runtime_plugins
         plugin_by_index = (
             runtime_plugins
-            + calibration_plugins
-            + gaze_mapping_plugins
+            + available_choreography_plugins
+            + registered_gazer_classes()
             + user_launchable_plugins
         )
         name_by_index = [pupil_datum.__name__ for pupil_datum in plugin_by_index]
         plugin_by_name = dict(zip(name_by_index, plugin_by_index))
         default_plugins = [
             ("Service_UI", {}),
             ("Dummy_Gaze_Mapper", {}),
-            ("HMD_Calibration", {}),
+            # Calibration choreography plugin is added bellow by calling
+            # patch_world_session_settings_with_choreography_plugin
             ("Pupil_Remote", {}),
             ("Blink_Detection", {}),
             ("Fixation_Detector", {}),
@@ -189,10 +196,10 @@ def get_dt():
         g_pool.min_calibration_confidence = session_settings.get(
             "min_calibration_confidence", 0.8
         )
-        g_pool.detection_mapping_mode = session_settings.get(
-            "detection_mapping_mode", "2d"
+        g_pool.pupil_detection_enabled = session_settings.get(
+            "pupil_detection_enabled", True
         )
-        g_pool.active_calibration_plugin = None
+
         g_pool.active_gaze_mapping_plugin = None
 
         audio.audio_mode = session_settings.get("audio_mode", audio.default_audio_mode)
@@ -201,10 +208,16 @@ def get_dt():
         logger.warning("Process started.")
         g_pool.service_should_run = True
 
-        # plugins that are loaded based on user settings from previous session
-        g_pool.plugins = Plugin_List(
-            g_pool, session_settings.get("loaded_plugins", default_plugins)
+        loaded_plugins = session_settings.get("loaded_plugins", default_plugins)
+
+        # Resolve the active calibration choreography plugin
+        loaded_plugins = patch_loaded_plugins_with_choreography_plugin(
+            loaded_plugins, app=g_pool.app
         )
+        session_settings["loaded_plugins"] = loaded_plugins
+
+        # plugins that are loaded based on user settings from previous session
+        g_pool.plugins = Plugin_List(g_pool, loaded_plugins)
 
         # NOTE: The Pupil_Remote plugin fails to load when the port is already in use
         # and will set this variable to false. Then we should not even start the eye
@@ -218,23 +231,14 @@ def get_dt():
 
         def handle_notifications(n):
             subject = n["subject"]
-            if subject == "set_detection_mapping_mode":
-                if n["mode"] == "2d":
-                    if (
-                        "Vector_Gaze_Mapper"
-                        in g_pool.active_gaze_mapping_plugin.class_name
-                    ):
-                        logger.warning(
-                            "The gaze mapper is not supported in 2d mode. Please recalibrate."
-                        )
-                        g_pool.plugins.add(plugin_by_name["Dummy_Gaze_Mapper"])
-                g_pool.detection_mapping_mode = n["mode"]
+            if subject == "set_pupil_detection_enabled":
+                g_pool.pupil_detection_enabled = n["mode"]
             elif subject == "start_plugin":
                 g_pool.plugins.add(plugin_by_name[n["name"]], args=n.get("args", {}))
             elif subject == "eye_process.started":
                 n = {
-                    "subject": "set_detection_mapping_mode",
-                    "mode": g_pool.detection_mapping_mode,
+                    "subject": "set_pupil_detection_enabled",
+                    "value": g_pool.pupil_detection_enabled,
                 }
                 ipc_pub.notify(n)
             elif subject == "service_process.should_stop":
@@ -294,7 +298,7 @@ def handle_notifications(n):
         session_settings[
             "min_calibration_confidence"
         ] = g_pool.min_calibration_confidence
-        session_settings["detection_mapping_mode"] = g_pool.detection_mapping_mode
+        session_settings["pupil_detection_enabled"] = g_pool.pupil_detection_enabled
         session_settings["audio_mode"] = audio.audio_mode
         session_settings.close()