change uint8 to uint32 in websocketmessages for splatS

examples/01_image.py

@@ -11,6 +11,7 @@
 
 import imageio.v3 as iio
 import numpy as onp
+
 import viser
 
 

examples/02_gui.py

@@ -5,6 +5,7 @@
 import time
 
 import numpy as onp
+
 import viser
 
 

examples/03_gui_callbacks.py

@@ -6,9 +6,10 @@
 import time
 
 import numpy as onp
-import viser
 from typing_extensions import assert_never
 
+import viser
+
 
 def main() -> None:
     server = viser.ViserServer()

examples/05_camera_commands.py

@@ -7,6 +7,7 @@
 import time
 
 import numpy as onp
+
 import viser
 import viser.transforms as tf
 

examples/06_mesh.py

@@ -8,6 +8,7 @@
 
 import numpy as onp
 import trimesh
+
 import viser
 import viser.transforms as tf
 

examples/07_record3d_visualizer.py

@@ -8,10 +8,11 @@
 
 import numpy as onp
 import tyro
+from tqdm.auto import tqdm
+
 import viser
 import viser.extras
 import viser.transforms as tf
-from tqdm.auto import tqdm
 
 
 def main(

examples/08_smpl_visualizer.py

@@ -15,6 +15,7 @@
 import numpy as np
 import numpy as onp
 import tyro
+
 import viser
 import viser.transforms as tf
 

examples/09_urdf_visualizer.py

@@ -14,8 +14,9 @@
 
 import numpy as onp
 import tyro
-import viser
 from robot_descriptions.loaders.yourdfpy import load_robot_description
+
+import viser
 from viser.extras import ViserUrdf
 
 # A subset of robots available in the robot_descriptions package.

examples/10_realsense.py

@@ -11,9 +11,10 @@
 import numpy as np
 import numpy.typing as npt
 import pyrealsense2 as rs  # type: ignore
-import viser
 from tqdm.auto import tqdm
 
+import viser
+
 
 @contextlib.contextmanager
 def realsense_pipeline(fps: int = 30):

examples/11_colmap_visualizer.py

@@ -10,9 +10,10 @@
 import imageio.v3 as iio
 import numpy as onp
 import tyro
+from tqdm.auto import tqdm
+
 import viser
 import viser.transforms as tf
-from tqdm.auto import tqdm
 from viser.extras.colmap import (
     read_cameras_binary,
     read_images_binary,

examples/12_click_meshes.py

@@ -6,6 +6,7 @@
 import time
 
 import matplotlib
+
 import viser
 
 

examples/15_gui_in_scene.py

@@ -9,6 +9,7 @@
 from typing import Optional
 
 import numpy as onp
+
 import viser
 import viser.transforms as tf
 

examples/17_background_composite.py

@@ -9,6 +9,7 @@
 import numpy as onp
 import trimesh
 import trimesh.creation
+
 import viser
 
 server = viser.ViserServer()

examples/18_splines.py

@@ -6,6 +6,7 @@
 import time
 
 import numpy as onp
+
 import viser
 
 

examples/19_get_renders.py

@@ -6,6 +6,7 @@
 
 import imageio.v3 as iio
 import numpy as onp
+
 import viser
 
 

examples/20_scene_pointer.py

@@ -16,6 +16,7 @@
 import trimesh
 import trimesh.creation
 import trimesh.ray
+
 import viser
 import viser.transforms as tf
 from viser.theme import TitlebarConfig

examples/22_games.py

@@ -7,9 +7,10 @@
 
 import numpy as onp
 import trimesh.creation
+from typing_extensions import assert_never
+
 import viser
 import viser.transforms as tf
-from typing_extensions import assert_never
 
 
 def main() -> None:

examples/23_plotly.py

@@ -7,9 +7,10 @@
 import numpy as onp
 import plotly.express as px
 import plotly.graph_objects as go
-import viser
 from PIL import Image
 
+import viser
+
 
 def create_sinusoidal_wave(t: float) -> go.Figure:
     """Create a sinusoidal wave plot, starting at time t."""

examples/23_smpl_visualizer_skinned.py

@@ -19,6 +19,7 @@
 import numpy as np
 import numpy as onp
 import tyro
+
 import viser
 import viser.transforms as tf
 

examples/experimental/gaussian_splats.py

@@ -9,8 +9,9 @@
 import numpy as onp
 import numpy.typing as onpt
 import tyro
-import viser
 from plyfile import PlyData
+
+import viser
 from viser import transforms as tf
 
 
@@ -82,29 +83,33 @@ def load_ply_file(ply_file_path: Path, center: bool = False) -> SplatFile:
     positions = onp.stack([v["x"], v["y"], v["z"]], axis=-1)
     scales = onp.exp(onp.stack([v["scale_0"], v["scale_1"], v["scale_2"]], axis=-1))
     wxyzs = onp.stack([v["rot_0"], v["rot_1"], v["rot_2"], v["rot_3"]], axis=1)
-    colors = 0.5 + SH_C0 * onp.stack([v["f_dc_0"], v["f_dc_1"], v["f_dc_2"]], axis=1) 
+    colors = 0.5 + SH_C0 * onp.stack([v["f_dc_0"], v["f_dc_1"], v["f_dc_2"]], axis=1)
     # print(v["f_dc_0"].shape) # prints (numGaussians)
     # print(colors.shape) # prints (numGaussians, 3)
     opacities = 1.0 / (1.0 + onp.exp(-v["opacity"][:, None]))
-    
+
     # Load all zero order SH coefficients
     dc_terms = onp.stack([v["f_dc_0"], v["f_dc_1"], v["f_dc_2"]], axis=1)
 
     # Load higher order SH coefficients (f_rest_0, ... f_rest_44), which are either level 1 or higher
     # Note: .ply file supports maximum SH degree of 3, R = f_rest_0, ... f_rest_14; G = f_rest_15, ... f_rest_29
     rest_terms = []
     i = 0
-    #while f"f_rest_{i}" in v:
+    # while f"f_rest_{i}" in v:
     while i < 15:
-        rest_terms.append(v[f"f_rest_{i}"]) # has shape (numGaussians, )
+        rest_terms.append(v[f"f_rest_{i}"])  # has shape (numGaussians, )
         rest_terms.append(v[f"f_rest_{15 + i}"])
         rest_terms.append(v[f"f_rest_{30 + i}"])
         i += 1
     # while f"f_rest_{i}" in v:
     #     rest_terms.append(v[f"f_rest_{i}"])
     #     i += 1
-    if len(rest_terms) > 0: # if we do have higher than zero order SH, we will process them and add them here.
-        sh_coeffs = onp.stack([v["f_dc_0"], v["f_dc_1"], v["f_dc_2"]] + rest_terms, axis=1)
+    if (
+        len(rest_terms) > 0
+    ):  # if we do have higher than zero order SH, we will process them and add them here.
+        sh_coeffs = onp.stack(
+            [v["f_dc_0"], v["f_dc_1"], v["f_dc_2"]] + rest_terms, axis=1
+        )
     sh_degree = int(onp.sqrt(sh_coeffs.shape[1] // 3) - 1)
 
     Rs = tf.SO3(wxyzs).as_matrix()
@@ -125,15 +130,15 @@ def load_ply_file(ply_file_path: Path, center: bool = False) -> SplatFile:
     print(
         f"PLY file with {num_gaussians=} loaded in {time.time() - start_time} seconds"
     )
-    print(onp.stack([v["f_dc_0"], v["f_dc_1"], v["f_dc_2"]], axis=1)[0 , :])
-    print(sh_coeffs[0, :]) # first gaussian, all 48 coefficients
+    print(onp.stack([v["f_dc_0"], v["f_dc_1"], v["f_dc_2"]], axis=1)[0, :])
+    print(sh_coeffs[0, :])  # first gaussian, all 48 coefficients
     # print(sh_coeffs[0, :]) # next 3 SH coefficients that are the 1st order
 
     return {
         "centers": positions[0:1, :],
         "rgbs": colors[0:1, :],
         "opacities": opacities[0:1, :],
-        "covariances": 10000*covariances[0:1, :],
+        "covariances": 10000 * covariances[0:1, :],
         "sh_degree": sh_degree,
         "sh_coeffs": sh_coeffs[0:1, :],
     }
@@ -165,6 +170,7 @@ def _(event: viser.GuiEvent) -> None:
             raise SystemExit("Please provide a filepath to a .splat or .ply file.")
 
         server.scene.add_transform_controls(f"/{i}")
+
         gs_handle = server.scene._add_gaussian_splats(
             f"/{i}/gaussian_splats",
             centers=splat_data["centers"],
@@ -188,7 +194,3 @@ def _(_, gs_handle=gs_handle, remove_button=remove_button) -> None:
 
 if __name__ == "__main__":
     tyro.cli(main)
-
-
-
-print("yapyap")

src/viser/_messages.py

@@ -761,14 +761,15 @@ class GaussianSplatsMessage(Message):
     - cov5 (f16), cov6 (f16)
     - rgba (int32)
     Where cov1-6 are the upper triangular elements of the covariance matrix."""
-    
+
     sh_buffer: onpt.NDArray[onp.uint32]
     """The spherical harmonic buffer contains:
-    - 
-    -
-    -
+    For every 24 uint32s, read each coefficient as a float16:
+    - r: every 0 mod3 float16
+    - g: every 1 mod3 float16
+    - b: every 2 mod3 float 16
+    goes from 0th order until 3rd order
     """
-
 
 
 @dataclasses.dataclass

src/viser/client/src/MessageHandler.tsx

@@ -983,6 +983,7 @@ function useMessageHandler() {
         return;
       }
       case "GaussianSplatsMessage": {
+
         addSceneNodeMakeParents(
           new SceneNode<THREE.Group>(message.name, (ref) => {
             return (

src/viser/client/src/Splatting/spherical_harmonics_testing.ipynb

@@ -29,7 +29,6 @@
     "#  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE\n",
     "#  POSSIBILITY OF SUCH DAMAGE.\n",
     "\n",
-    "import torch\n",
     "\n",
     "C0 = 0.28209479177387814\n",
     "C1 = 0.4886025119029199\n",
@@ -141,6 +140,7 @@
    ],
    "source": [
     "import numpy as np\n",
+    "\n",
     "deg = 3\n",
     "sh = np.random.rand(3, 16)\n",
     "dirs = np.array([1, 0, 0])\n",

src/viser/client/src/WebsocketMessages.tsx

@@ -850,8 +850,8 @@ export interface CubicBezierSplineMessage {
 export interface GaussianSplatsMessage {
   type: "GaussianSplatsMessage";
   name: string;
-  buffer: Uint8Array;
-  sh_buffer: Uint8Array;
+  buffer: Uint32Array;
+  sh_buffer: Uint32Array;
 }
 /** Message from server->client requesting a render of the current viewport.
  *