RayClicks integration

examples/20_rayclick.py

@@ -0,0 +1,74 @@
+"""Rayclicks
+
+Visualize a mesh. To get the demo data, see `./assets/download_dragon_mesh.sh`.
+"""
+
+import time
+import typing
+from pathlib import Path
+
+import trimesh
+import numpy as onp
+
+import viser
+import viser.transforms as tf
+
+server = viser.ViserServer()
+
+mesh = trimesh.load_mesh(Path(__file__).parent / "assets/dragon.obj")
+assert isinstance(mesh, trimesh.Trimesh)
+mesh.apply_scale(0.05)
+
+mesh_handle = server.add_mesh_trimesh(
+    name="/mesh",
+    mesh=mesh,
+    wxyz=tf.SO3.exp(onp.array([onp.pi / 2, 0.0, 0.0])).wxyz,
+    position=(0.0, 0.0, 0.0),
+)
+
+button_handle = server.add_gui_checkbox("Enable Rayclicks", False)
+@button_handle.on_update
+def _(_) -> None:
+    # is there a better name for this?
+    server.scene_pointer_enabled = button_handle.value
+
+# Note: Scene clicks don't interrupt the scenenodeclicks.
+@mesh_handle.on_click
+def _(_):
+    print("Mesh clicked")
+
+hit_pos_handle = None
+
+@server.on_scene_pointer
+def on_rayclick(message: viser.ScenePointerEvent) -> None:
+    global hit_pos_handle
+
+    # check for intersection with the mesh
+    mesh_tf = tf.SO3(mesh_handle.wxyz).inverse().as_matrix()
+    origin = (mesh_tf @ onp.array(message.ray_origin)).reshape(1, 3)
+    direction = (mesh_tf @ onp.array(message.ray_direction)).reshape(1, 3)
+    intersector = trimesh.ray.ray_triangle.RayMeshIntersector(mesh)
+    hit_pos, _, _ = intersector.intersects_location(origin, direction)
+
+    # if no hit, remove the hit vis from the scene.
+    if len(hit_pos) == 0:
+        if hit_pos_handle is not None:
+            hit_pos_handle.remove()
+            hit_pos_handle = None
+        return
+
+    # get the first hit position
+    hit_pos = sorted(hit_pos, key=lambda x: onp.linalg.norm(x - origin))[0]
+
+    # put the hit position back into the world frame
+    hit_pos = (tf.SO3(mesh_handle.wxyz).as_matrix() @ hit_pos.T).T
+    hit_pos_mesh = trimesh.creation.icosphere(radius=0.1)
+    hit_pos_mesh.vertices += hit_pos
+    hit_pos_mesh.visual.vertex_colors = (1.0, 0.0, 0.0, 1.0)
+    hit_pos_handle = server.add_mesh_trimesh(
+        name="/hit_pos",
+        mesh=hit_pos_mesh
+    )
+
+while True:
+    time.sleep(10.0)

src/viser/__init__.py

@@ -10,8 +10,9 @@
 from ._gui_handles import GuiTabGroupHandle as GuiTabGroupHandle
 from ._gui_handles import GuiTabHandle as GuiTabHandle
 from ._icons_enum import Icon as Icon
+from ._scene_handles import ScenePointerEvent as ScenePointerEvent
 from ._scene_handles import CameraFrustumHandle as CameraFrustumHandle
-from ._scene_handles import ClickEvent as ClickEvent
+from ._scene_handles import SceneNodePointerEvent as SceneNodePointerEvent
 from ._scene_handles import FrameHandle as FrameHandle
 from ._scene_handles import GlbHandle as GlbHandle
 from ._scene_handles import Gui3dContainerHandle as Gui3dContainerHandle

src/viser/_message_api.py

@@ -14,7 +14,7 @@
 import queue
 import threading
 import time
-from typing import TYPE_CHECKING, Dict, Optional, Tuple, TypeVar, Union, cast
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, TypeVar, Union, cast, Callable
 
 import imageio.v3 as iio
 import numpy as onp
@@ -26,8 +26,9 @@
 
 from . import _messages, infra, theme
 from ._scene_handles import (
+    ScenePointerEvent,
     CameraFrustumHandle,
-    ClickEvent,
+    SceneNodePointerEvent,
     FrameHandle,
     GlbHandle,
     Gui3dContainerHandle,
@@ -139,6 +140,10 @@ def __init__(self, handler: infra.MessageHandler) -> None:
         ] = {}
         self._handle_from_node_name: Dict[str, SceneNodeHandle] = {}
 
+        # Callbacks for scene pointer events -- by default don't enable them.
+        self._scene_pointer_cb: List[Callable[[ScenePointerEvent], None]] = []
+        self._scene_pointer_enabled = False
+
         handler.register_handler(
             _messages.TransformControlsUpdateMessage,
             self._handle_transform_controls_updates,
@@ -147,7 +152,11 @@ def __init__(self, handler: infra.MessageHandler) -> None:
             _messages.SceneNodeClickedMessage,
             self._handle_click_updates,
         )
-
+        handler.register_handler(
+            _messages.ScenePointerMessage,
+            self._handle_scene_pointer_updates,
+            )
+
         self._atomic_lock = threading.Lock()
         self._queued_messages: queue.Queue = queue.Queue()
         self._locked_thread_id = -1
@@ -625,8 +634,41 @@ def _handle_click_updates(
         if handle is None or handle._impl.click_cb is None:
             return
         for cb in handle._impl.click_cb:
-            event = ClickEvent(client_id=client_id, target=handle)
+            event = SceneNodePointerEvent(client_id=client_id, target=handle)
             cb(event)  # type: ignore
+
+    @property
+    def scene_pointer_enabled(self) -> bool:
+        """Whether scene pointer events are enabled."""
+        return self._scene_pointer_enabled
+
+    @scene_pointer_enabled.setter
+    def scene_pointer_enabled(self, enable: bool) -> None:
+        """Enable or disable scene pointer events."""
+        self._scene_pointer_enabled = enable
+        self._queue(_messages.EnableScenePointerMessage(
+            enabled=enable
+            ))
+
+    def _handle_scene_pointer_updates(
+        self, client_id: ClientId, message: _messages.ScenePointerMessage
+    ):
+        """Callback for handling click messages."""
+        for cb in self._scene_pointer_cb:
+            event = ScenePointerEvent(
+                client_id=client_id,
+                pointer_type=message.pointer_type,
+                ray_origin=message.ray_origin,
+                ray_direction=message.ray_direction,
+            )
+            cb(event)
+
+    def on_scene_pointer(
+        self, func: Callable[[ScenePointerEvent], None],
+    ) -> Callable[[ScenePointerEvent], None]:
+        """Add a callback for scene pointer events. """
+        self._scene_pointer_cb.append(func)
+        return func
 
     def add_3d_gui_container(
         self,

src/viser/_messages.py

@@ -50,6 +50,23 @@ class ViewerCameraMessage(Message):
     up_direction: Tuple[float, float, float]
 
 
+@dataclasses.dataclass
+class ScenePointerMessage(Message):
+    """Message for a raycast-like pointer in the scene. 
+    origin is the viewing camera position, in world coordinates.
+    direction is the vector if a ray is projected from the camera through the clicked pixel,
+    """
+    pointer_type: Literal["click"]  # Later we can add `double_click`, `move`, `down`, `up`, etc
+    ray_origin: Tuple[float, float, float]
+    ray_direction: Tuple[float, float, float]
+
+
+@dataclasses.dataclass
+class EnableScenePointerMessage(Message):
+    """Message to enable/disable scene pointer"""
+    enabled: bool
+
+
 @dataclasses.dataclass
 class CameraFrustumMessage(Message):
     """Variant of CameraMessage used for visualizing camera frustums.

src/viser/_scene_handles.py

@@ -12,6 +12,7 @@
     Dict,
     Generic,
     List,
+    Literal,
     Optional,
     Tuple,
     Type,
@@ -28,6 +29,14 @@
     from ._message_api import ClientId, MessageApi
 
 
+@dataclasses.dataclass(frozen=True)
+class ScenePointerEvent:
+    client_id: ClientId
+    pointer_type: Literal["click"]  # Later we can add `double_click`, `move`, `down`, `up`, etc
+    ray_origin: Tuple[float, float, float]
+    ray_direction: Tuple[float, float, float]
+
+
 TSceneNodeHandle = TypeVar("TSceneNodeHandle", bound="SceneNodeHandle")
 TSupportsVisibility = TypeVar("TSupportsVisibility", bound="_SupportsVisibility")
 
@@ -44,7 +53,7 @@ class _SceneNodeHandleState:
     )
     visible: bool = True
     # TODO: we should remove SceneNodeHandle as an argument here.
-    click_cb: Optional[List[Callable[[ClickEvent[SceneNodeHandle]], None]]] = None
+    click_cb: Optional[List[Callable[[SceneNodePointerEvent[SceneNodeHandle]], None]]] = None
 
 
 @dataclasses.dataclass
@@ -108,16 +117,16 @@ def remove(self) -> None:
 
 
 @dataclasses.dataclass(frozen=True)
-class ClickEvent(Generic[TSceneNodeHandle]):
+class SceneNodePointerEvent(Generic[TSceneNodeHandle]):
     client_id: ClientId
     target: TSceneNodeHandle
 
 
 @dataclasses.dataclass
 class _SupportsClick(SceneNodeHandle):
     def on_click(
-        self: TSceneNodeHandle, func: Callable[[ClickEvent[TSceneNodeHandle]], None]
-    ) -> Callable[[ClickEvent[TSceneNodeHandle]], None]:
+        self: TSceneNodeHandle, func: Callable[[SceneNodePointerEvent[TSceneNodeHandle]], None]
+    ) -> Callable[[SceneNodePointerEvent[TSceneNodeHandle]], None]:
         """Attach a callback for when a scene node is clicked.
 
         TODO:

src/viser/client/src/App.tsx

@@ -15,6 +15,7 @@ import {
 import { BlendFunction, KernelSize } from "postprocessing";
 
 import { SynchronizedCameraControls } from "./CameraControls";
+import { ScenePointerControls } from "./ScenePointerControls";
 import { Box, MantineProvider, MediaQuery } from "@mantine/core";
 import React from "react";
 import { SceneNodeThreeObject, UseSceneTree } from "./SceneTree";
@@ -67,6 +68,7 @@ export type ViewerContextContents = {
     "ready" | "triggered" | "pause" | "in_progress"
   >;
   getRenderRequest: React.MutableRefObject<null | GetRenderRequestMessage>;
+  useScenePointer: React.MutableRefObject<boolean>;
 };
 export const ViewerContext = React.createContext<null | ViewerContextContents>(
   null,
@@ -108,6 +110,7 @@ function ViewerRoot() {
     messageQueueRef: React.useRef([]),
     getRenderRequestState: React.useRef("ready"),
     getRenderRequest: React.useRef(null),
+    useScenePointer: React.useRef(false),
   };
 
   return (
@@ -180,6 +183,7 @@ function ViewerCanvas({ children }: { children: React.ReactNode }) {
       <AdaptiveEvents />
       <SceneContextSetter />
       <SynchronizedCameraControls />
+      <ScenePointerControls />
       <Selection>
         <SceneNodeThreeObject name="" />
         <EffectComposer enabled={true} autoClear={false}>

src/viser/client/src/ScenePointerControls.tsx

@@ -0,0 +1,67 @@
+import { ViewerContext } from "./App";
+import { makeThrottledMessageSender } from "./WebsocketFunctions";
+import { useThree } from "@react-three/fiber";
+import React, { useContext } from "react";
+import { PerspectiveCamera } from "three";
+import * as THREE from "three";
+
+export function ScenePointerControls() {
+  const viewer = useContext(ViewerContext)!;
+  const camera = useThree((state) => state.camera as PerspectiveCamera);
+
+  const sendClickThrottled = makeThrottledMessageSender(
+    viewer.websocketRef,
+    20,
+  );
+  React.useEffect(() => {
+    const onMouseClick = (e: MouseEvent) => {
+      console.log("click event");
+
+      // Don't send click events if the scene pointer events are disabled.
+      if (!viewer.useScenePointer.current!) return;
+
+      // check that the mouse event happened inside the canvasRef.
+      if (e.target !== viewer.canvasRef.current!) return;
+
+      // clientX/Y are relative to the viewport, offsetX/Y are relative to the canvasRef.
+      // clientX==offsetX if there is no titlebar, but clientX>offsetX if there is a titlebar.
+      const mouseVector = new THREE.Vector2();
+      mouseVector.x = 2 * (e.offsetX / viewer.canvasRef.current!.clientWidth) - 1;
+      mouseVector.y = 1 - 2 * (e.offsetY / viewer.canvasRef.current!.clientHeight);
+
+      const mouse_in_scene = !(
+        mouseVector.x > 1 ||
+        mouseVector.x < -1 ||
+        mouseVector.y > 1 ||
+        mouseVector.y < -1
+      );
+      if (!mouse_in_scene) { return; } 
+
+      const raycaster = new THREE.Raycaster();
+      raycaster.setFromCamera(mouseVector, camera);
+
+      console.log("sending scenepointer", e.offsetX, e.offsetY);
+
+      sendClickThrottled({
+        type: 'ScenePointerMessage',
+        pointer_type: 'click',
+        ray_origin: [
+          raycaster.ray.origin.x, 
+          -raycaster.ray.origin.z, 
+          raycaster.ray.origin.y
+        ],
+        ray_direction: [
+          raycaster.ray.direction.x, 
+          -raycaster.ray.direction.z, 
+          raycaster.ray.direction.y
+        ],
+      });
+    };
+    window.addEventListener('click', onMouseClick, false);
+    return () => {
+      window.removeEventListener('click', onMouseClick, false);
+    };
+  }, [camera, sendClickThrottled]);
+
+  return null;
+}

src/viser/client/src/WebsocketInterface.tsx

@@ -93,6 +93,13 @@ function useMessageHandler() {
         setTheme(message);
         return;
       }
+
+      // Enable/disable whether scene pointer events are sent.
+      case "EnableScenePointerMessage": {
+        viewer.useScenePointer.current = message.enabled;
+        return;
+      }
+
       // Add a coordinate frame.
       case "FrameMessage": {
         addSceneNodeMakeParents(