Move scene pointer logic to diff file, add pointer on/off logic

examples/20_rayclick.py

@@ -19,32 +19,42 @@
 assert isinstance(mesh, trimesh.Trimesh)
 mesh.apply_scale(0.05)
 
-vertices = mesh.vertices
-faces = mesh.faces
-print(f"Loaded mesh with {vertices.shape} vertices, {faces.shape} faces")
-
 mesh_handle = server.add_mesh_trimesh(
     name="/mesh",
     mesh=mesh,
     wxyz=tf.SO3.exp(onp.array([onp.pi / 2, 0.0, 0.0])).wxyz,
     position=(0.0, 0.0, 0.0),
 )
 
+button_handle = server.add_gui_checkbox("Enable Rayclicks", False)
+@button_handle.on_update
+def _(_) -> None:
+    # is there a better name for this?
+    server.scene_pointer_enabled = button_handle.value
+
+# Note: Scene clicks don't interrupt the scenenodeclicks.
+@mesh_handle.on_click
+def _(_):
+    print("Mesh clicked")
+
 hit_pos_handle = None
-def on_rayclick(origin: typing.Tuple, direction: typing.Tuple) -> None:
+
+@server.on_scene_pointer
+def on_rayclick(message: viser.ScenePointerEvent) -> None:
     global hit_pos_handle
 
     # check for intersection with the mesh
     mesh_tf = tf.SO3(mesh_handle.wxyz).inverse().as_matrix()
-    origin = (mesh_tf @ onp.array(origin)).reshape(1, 3)
-    direction = (mesh_tf @ onp.array(direction)).reshape(1, 3)
+    origin = (mesh_tf @ onp.array(message.ray_origin)).reshape(1, 3)
+    direction = (mesh_tf @ onp.array(message.ray_direction)).reshape(1, 3)
     intersector = trimesh.ray.ray_triangle.RayMeshIntersector(mesh)
     hit_pos, _, _ = intersector.intersects_location(origin, direction)
 
     # if no hit, remove the hit vis from the scene.
     if len(hit_pos) == 0:
-        hit_pos_handle.remove()
-        hit_pos_handle = None
+        if hit_pos_handle is not None:
+            hit_pos_handle.remove()
+            hit_pos_handle = None
         return
 
     # get the first hit position
@@ -60,7 +70,5 @@ def on_rayclick(origin: typing.Tuple, direction: typing.Tuple) -> None:
         mesh=hit_pos_mesh
     )
 
-server.add_rayclick_cb(on_rayclick)
-
 while True:
     time.sleep(10.0)

src/viser/__init__.py

@@ -10,8 +10,9 @@
 from ._gui_handles import GuiTabGroupHandle as GuiTabGroupHandle
 from ._gui_handles import GuiTabHandle as GuiTabHandle
 from ._icons_enum import Icon as Icon
+from ._scene_handles import ScenePointerEvent as ScenePointerEvent
 from ._scene_handles import CameraFrustumHandle as CameraFrustumHandle
-from ._scene_handles import ClickEvent as ClickEvent
+from ._scene_handles import SceneNodePointerEvent as SceneNodePointerEvent
 from ._scene_handles import FrameHandle as FrameHandle
 from ._scene_handles import GlbHandle as GlbHandle
 from ._scene_handles import Gui3dContainerHandle as Gui3dContainerHandle

src/viser/_message_api.py

@@ -14,7 +14,7 @@
 import queue
 import threading
 import time
-from typing import TYPE_CHECKING, Dict, Optional, Tuple, TypeVar, Union, cast
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, TypeVar, Union, cast, Callable
 
 import imageio.v3 as iio
 import numpy as onp
@@ -26,8 +26,9 @@
 
 from . import _messages, infra, theme
 from ._scene_handles import (
+    ScenePointerEvent,
     CameraFrustumHandle,
-    ClickEvent,
+    SceneNodePointerEvent,
     FrameHandle,
     GlbHandle,
     Gui3dContainerHandle,
@@ -138,7 +139,10 @@ def __init__(self, handler: infra.MessageHandler) -> None:
             str, TransformControlsHandle
         ] = {}
         self._handle_from_node_name: Dict[str, SceneNodeHandle] = {}
-        self._handle_rayclick: List[Callable[Tuple, Tuple]] = []
+
+        # Callbacks for scene pointer events -- by default don't enable them.
+        self._scene_pointer_cb: List[Callable[[ScenePointerEvent], None]] = []
+        self._scene_pointer_enabled = False
 
         handler.register_handler(
             _messages.TransformControlsUpdateMessage,
@@ -149,8 +153,8 @@ def __init__(self, handler: infra.MessageHandler) -> None:
             self._handle_click_updates,
         )
         handler.register_handler(
-            _messages.RayClickMessage,
-            self._handle_rayclick_updates,
+            _messages.ScenePointerMessage,
+            self._handle_scene_pointer_updates,
             )
 
         self._atomic_lock = threading.Lock()
@@ -630,23 +634,41 @@ def _handle_click_updates(
         if handle is None or handle._impl.click_cb is None:
             return
         for cb in handle._impl.click_cb:
-            event = ClickEvent(client_id=client_id, target=handle)
+            event = SceneNodePointerEvent(client_id=client_id, target=handle)
             cb(event)  # type: ignore
 
-    def _handle_rayclick_updates(
-        self, client_id: ClientId, message: _messages.ClickMessage
+    @property
+    def scene_pointer_enabled(self) -> bool:
+        """Whether scene pointer events are enabled."""
+        return self._scene_pointer_enabled
+
+    @scene_pointer_enabled.setter
+    def scene_pointer_enabled(self, enable: bool) -> None:
+        """Enable or disable scene pointer events."""
+        self._scene_pointer_enabled = enable
+        self._queue(_messages.EnableScenePointerMessage(
+            enabled=enable
+            ))
+
+    def _handle_scene_pointer_updates(
+        self, client_id: ClientId, message: _messages.ScenePointerMessage
     ):
         """Callback for handling click messages."""
-        for cb in self._handle_rayclick:
-            cb(message.origin, message.direction)
-
-    def add_rayclick_cb(
-        self,
-        cb: Callable[[Tuple[float, float, float], Tuple[float, float, float]], None],
-    ) -> None:
-        """Add a callback for rayclick events. 
-        Callback takes in the origin and direction of the ray."""
-        self._handle_rayclick.append(cb)
+        for cb in self._scene_pointer_cb:
+            event = ScenePointerEvent(
+                client_id=client_id,
+                pointer_type=message.pointer_type,
+                ray_origin=message.ray_origin,
+                ray_direction=message.ray_direction,
+            )
+            cb(event)
+
+    def on_scene_pointer(
+        self, func: Callable[[ScenePointerEvent], None],
+    ) -> Callable[[ScenePointerEvent], None]:
+        """Add a callback for scene pointer events. """
+        self._scene_pointer_cb.append(func)
+        return func
 
     def add_3d_gui_container(
         self,

src/viser/_messages.py

@@ -51,13 +51,20 @@ class ViewerCameraMessage(Message):
 
 
 @dataclasses.dataclass
-class RayClickMessage(Message):
-    """Message for raycast clicks.
+class ScenePointerMessage(Message):
+    """Message for a raycast-like pointer in the scene. 
     origin is the viewing camera position, in world coordinates.
     direction is the vector if a ray is projected from the camera through the clicked pixel,
     """
-    origin: Tuple[float, float, float]
-    direction: Tuple[float, float, float]
+    pointer_type: Literal["click"]  # Later we can add `double_click`, `move`, `down`, `up`, etc
+    ray_origin: Tuple[float, float, float]
+    ray_direction: Tuple[float, float, float]
+
+
+@dataclasses.dataclass
+class EnableScenePointerMessage(Message):
+    """Message to enable/disable scene pointer"""
+    enabled: bool
 
 
 @dataclasses.dataclass

src/viser/_scene_handles.py

@@ -12,6 +12,7 @@
     Dict,
     Generic,
     List,
+    Literal,
     Optional,
     Tuple,
     Type,
@@ -28,6 +29,14 @@
     from ._message_api import ClientId, MessageApi
 
 
+@dataclasses.dataclass(frozen=True)
+class ScenePointerEvent:
+    client_id: ClientId
+    pointer_type: Literal["click"]  # Later we can add `double_click`, `move`, `down`, `up`, etc
+    ray_origin: Tuple[float, float, float]
+    ray_direction: Tuple[float, float, float]
+
+
 TSceneNodeHandle = TypeVar("TSceneNodeHandle", bound="SceneNodeHandle")
 TSupportsVisibility = TypeVar("TSupportsVisibility", bound="_SupportsVisibility")
 
@@ -44,7 +53,7 @@ class _SceneNodeHandleState:
     )
     visible: bool = True
     # TODO: we should remove SceneNodeHandle as an argument here.
-    click_cb: Optional[List[Callable[[ClickEvent[SceneNodeHandle]], None]]] = None
+    click_cb: Optional[List[Callable[[SceneNodePointerEvent[SceneNodeHandle]], None]]] = None
 
 
 @dataclasses.dataclass
@@ -108,16 +117,16 @@ def remove(self) -> None:
 
 
 @dataclasses.dataclass(frozen=True)
-class ClickEvent(Generic[TSceneNodeHandle]):
+class SceneNodePointerEvent(Generic[TSceneNodeHandle]):
     client_id: ClientId
     target: TSceneNodeHandle
 
 
 @dataclasses.dataclass
 class _SupportsClick(SceneNodeHandle):
     def on_click(
-        self: TSceneNodeHandle, func: Callable[[ClickEvent[TSceneNodeHandle]], None]
-    ) -> Callable[[ClickEvent[TSceneNodeHandle]], None]:
+        self: TSceneNodeHandle, func: Callable[[SceneNodePointerEvent[TSceneNodeHandle]], None]
+    ) -> Callable[[SceneNodePointerEvent[TSceneNodeHandle]], None]:
         """Attach a callback for when a scene node is clicked.
 
         TODO:

src/viser/client/src/App.tsx

@@ -15,6 +15,7 @@ import {
 import { BlendFunction, KernelSize } from "postprocessing";
 
 import { SynchronizedCameraControls } from "./CameraControls";
+import { ScenePointerControls } from "./ScenePointerControls";
 import { Box, MantineProvider, MediaQuery } from "@mantine/core";
 import React from "react";
 import { SceneNodeThreeObject, UseSceneTree } from "./SceneTree";
@@ -67,6 +68,7 @@ export type ViewerContextContents = {
     "ready" | "triggered" | "pause" | "in_progress"
   >;
   getRenderRequest: React.MutableRefObject<null | GetRenderRequestMessage>;
+  useScenePointer: React.MutableRefObject<boolean>;
 };
 export const ViewerContext = React.createContext<null | ViewerContextContents>(
   null,
@@ -108,6 +110,7 @@ function ViewerRoot() {
     messageQueueRef: React.useRef([]),
     getRenderRequestState: React.useRef("ready"),
     getRenderRequest: React.useRef(null),
+    useScenePointer: React.useRef(false),
   };
 
   return (
@@ -180,6 +183,7 @@ function ViewerCanvas({ children }: { children: React.ReactNode }) {
       <AdaptiveEvents />
       <SceneContextSetter />
       <SynchronizedCameraControls />
+      <ScenePointerControls />
       <Selection>
         <SceneNodeThreeObject name="" />
         <EffectComposer enabled={true} autoClear={false}>

src/viser/client/src/CameraControls.tsx

@@ -108,59 +108,6 @@ export function SynchronizedCameraControls() {
     };
   }, [camera]);
 
-  // Note: This fires for double-click, which might not be available for mobile.
-  // Maybe we should also support mobile via long-presses? Worth investigating.
-  // Also, instead of double-click, we could consider an alt-click or meta-click.
-  const sendClickThrottled = makeThrottledMessageSender(
-    viewer.websocketRef,
-    20,
-  );
-  React.useEffect(() => {
-    const onMouseDouble = (e: MouseEvent) => {
-      // check that the mouse event happened inside the canvasRef.
-      if (e.target !== viewer.canvasRef.current!) return;
-
-      // clientX/Y are relative to the viewport, offsetX/Y are relative to the canvasRef.
-      // clientX==offsetX if there is no titlebar, but clientX>offsetX if there is a titlebar.
-      const mouseVector = new THREE.Vector2();
-      console.log(e);
-      console.log(viewer.canvasRef.current!.clientWidth, viewer.canvasRef.current!.clientHeight);
-      mouseVector.x = 2 * (e.offsetX / viewer.canvasRef.current!.clientWidth) - 1;
-      mouseVector.y = 1 - 2 * (e.offsetY / viewer.canvasRef.current!.clientHeight);
-
-      const mouse_in_scene = !(
-        mouseVector.x > 1 ||
-        mouseVector.x < -1 ||
-        mouseVector.y > 1 ||
-        mouseVector.y < -1
-      );
-      if (!mouse_in_scene) { return; } 
-
-      const raycaster = new THREE.Raycaster();
-      raycaster.setFromCamera(mouseVector, camera);
-
-      console.log(e.offsetX, e.offsetY);
-
-      sendClickThrottled({
-        type: 'RayClickMessage',
-        origin: [
-          raycaster.ray.origin.x, 
-          -raycaster.ray.origin.z, 
-          raycaster.ray.origin.y
-        ],
-        direction: [
-          raycaster.ray.direction.x, 
-          -raycaster.ray.direction.z, 
-          raycaster.ray.direction.y
-        ],
-      });
-    };
-    window.addEventListener('dblclick', onMouseDouble, false);
-    return () => {
-      window.removeEventListener('dblclick', onMouseDouble, false);
-    };
-  }, [camera, sendClickThrottled]);
-
   // Keyboard controls.
   React.useEffect(() => {
     const KEYCODE = {