diff --git a/examples/01_image.py b/examples/01_image.py index 05568b67..86a2a686 100644 --- a/examples/01_image.py +++ b/examples/01_image.py @@ -11,6 +11,7 @@ import imageio.v3 as iio import numpy as onp + import viser diff --git a/examples/02_gui.py b/examples/02_gui.py index e1322835..e48e63fd 100644 --- a/examples/02_gui.py +++ b/examples/02_gui.py @@ -5,6 +5,7 @@ import time import numpy as onp + import viser diff --git a/examples/03_gui_callbacks.py b/examples/03_gui_callbacks.py index a7d2133a..a1df139e 100644 --- a/examples/03_gui_callbacks.py +++ b/examples/03_gui_callbacks.py @@ -6,9 +6,10 @@ import time import numpy as onp -import viser from typing_extensions import assert_never +import viser + def main() -> None: server = viser.ViserServer() diff --git a/examples/05_camera_commands.py b/examples/05_camera_commands.py index 7130b7aa..4a51aeaa 100644 --- a/examples/05_camera_commands.py +++ b/examples/05_camera_commands.py @@ -7,6 +7,7 @@ import time import numpy as onp + import viser import viser.transforms as tf diff --git a/examples/06_mesh.py b/examples/06_mesh.py index eeeaf783..3d30f09e 100644 --- a/examples/06_mesh.py +++ b/examples/06_mesh.py @@ -8,6 +8,7 @@ import numpy as onp import trimesh + import viser import viser.transforms as tf diff --git a/examples/07_record3d_visualizer.py b/examples/07_record3d_visualizer.py index 2d869257..e8b926bb 100644 --- a/examples/07_record3d_visualizer.py +++ b/examples/07_record3d_visualizer.py @@ -8,10 +8,11 @@ import numpy as onp import tyro +from tqdm.auto import tqdm + import viser import viser.extras import viser.transforms as tf -from tqdm.auto import tqdm def main( diff --git a/examples/08_smpl_visualizer.py b/examples/08_smpl_visualizer.py index 3ea097c0..7d89f6b7 100644 --- a/examples/08_smpl_visualizer.py +++ b/examples/08_smpl_visualizer.py @@ -15,6 +15,7 @@ import numpy as np import numpy as onp import tyro + import viser import viser.transforms as tf diff --git a/examples/09_urdf_visualizer.py b/examples/09_urdf_visualizer.py index fe448539..3aebee5a 100644 --- a/examples/09_urdf_visualizer.py +++ b/examples/09_urdf_visualizer.py @@ -14,8 +14,9 @@ import numpy as onp import tyro -import viser from robot_descriptions.loaders.yourdfpy import load_robot_description + +import viser from viser.extras import ViserUrdf # A subset of robots available in the robot_descriptions package. diff --git a/examples/10_realsense.py b/examples/10_realsense.py index 8cc08f34..a16e6c2d 100644 --- a/examples/10_realsense.py +++ b/examples/10_realsense.py @@ -11,9 +11,10 @@ import numpy as np import numpy.typing as npt import pyrealsense2 as rs # type: ignore -import viser from tqdm.auto import tqdm +import viser + @contextlib.contextmanager def realsense_pipeline(fps: int = 30): diff --git a/examples/11_colmap_visualizer.py b/examples/11_colmap_visualizer.py index 493131b6..c9db0130 100644 --- a/examples/11_colmap_visualizer.py +++ b/examples/11_colmap_visualizer.py @@ -10,9 +10,10 @@ import imageio.v3 as iio import numpy as onp import tyro +from tqdm.auto import tqdm + import viser import viser.transforms as tf -from tqdm.auto import tqdm from viser.extras.colmap import ( read_cameras_binary, read_images_binary, diff --git a/examples/12_click_meshes.py b/examples/12_click_meshes.py index ddd70584..8fdec08f 100644 --- a/examples/12_click_meshes.py +++ b/examples/12_click_meshes.py @@ -6,6 +6,7 @@ import time import matplotlib + import viser diff --git a/examples/15_gui_in_scene.py b/examples/15_gui_in_scene.py index 9947d839..8a5638c5 100644 --- a/examples/15_gui_in_scene.py +++ b/examples/15_gui_in_scene.py @@ -9,6 +9,7 @@ from typing import Optional import numpy as onp + import viser import viser.transforms as tf diff --git a/examples/17_background_composite.py b/examples/17_background_composite.py index 3904d442..6098f02f 100644 --- a/examples/17_background_composite.py +++ b/examples/17_background_composite.py @@ -9,6 +9,7 @@ import numpy as onp import trimesh import trimesh.creation + import viser server = viser.ViserServer() diff --git a/examples/18_splines.py b/examples/18_splines.py index 3c939c9d..a6d42ebc 100644 --- a/examples/18_splines.py +++ b/examples/18_splines.py @@ -6,6 +6,7 @@ import time import numpy as onp + import viser diff --git a/examples/19_get_renders.py b/examples/19_get_renders.py index f235730b..2de8a065 100644 --- a/examples/19_get_renders.py +++ b/examples/19_get_renders.py @@ -6,6 +6,7 @@ import imageio.v3 as iio import numpy as onp + import viser diff --git a/examples/20_scene_pointer.py b/examples/20_scene_pointer.py index 1069c302..944976f5 100644 --- a/examples/20_scene_pointer.py +++ b/examples/20_scene_pointer.py @@ -16,6 +16,7 @@ import trimesh import trimesh.creation import trimesh.ray + import viser import viser.transforms as tf from viser.theme import TitlebarConfig diff --git a/examples/22_games.py b/examples/22_games.py index 6a45f5a6..1ff2f33a 100644 --- a/examples/22_games.py +++ b/examples/22_games.py @@ -7,9 +7,10 @@ import numpy as onp import trimesh.creation +from typing_extensions import assert_never + import viser import viser.transforms as tf -from typing_extensions import assert_never def main() -> None: diff --git a/examples/23_plotly.py b/examples/23_plotly.py index 48cfd36b..7f7231a7 100644 --- a/examples/23_plotly.py +++ b/examples/23_plotly.py @@ -7,9 +7,10 @@ import numpy as onp import plotly.express as px import plotly.graph_objects as go -import viser from PIL import Image +import viser + def create_sinusoidal_wave(t: float) -> go.Figure: """Create a sinusoidal wave plot, starting at time t.""" diff --git a/examples/23_smpl_visualizer_skinned.py b/examples/23_smpl_visualizer_skinned.py index 2fb364c4..02df8506 100644 --- a/examples/23_smpl_visualizer_skinned.py +++ b/examples/23_smpl_visualizer_skinned.py @@ -19,6 +19,7 @@ import numpy as np import numpy as onp import tyro + import viser import viser.transforms as tf diff --git a/examples/experimental/gaussian_splats.py b/examples/experimental/gaussian_splats.py index bfa6a9c5..26a0427c 100644 --- a/examples/experimental/gaussian_splats.py +++ b/examples/experimental/gaussian_splats.py @@ -9,8 +9,9 @@ import numpy as onp import numpy.typing as onpt import tyro -import viser from plyfile import PlyData + +import viser from viser import transforms as tf @@ -82,11 +83,11 @@ def load_ply_file(ply_file_path: Path, center: bool = False) -> SplatFile: positions = onp.stack([v["x"], v["y"], v["z"]], axis=-1) scales = onp.exp(onp.stack([v["scale_0"], v["scale_1"], v["scale_2"]], axis=-1)) wxyzs = onp.stack([v["rot_0"], v["rot_1"], v["rot_2"], v["rot_3"]], axis=1) - colors = 0.5 + SH_C0 * onp.stack([v["f_dc_0"], v["f_dc_1"], v["f_dc_2"]], axis=1) + colors = 0.5 + SH_C0 * onp.stack([v["f_dc_0"], v["f_dc_1"], v["f_dc_2"]], axis=1) # print(v["f_dc_0"].shape) # prints (numGaussians) # print(colors.shape) # prints (numGaussians, 3) opacities = 1.0 / (1.0 + onp.exp(-v["opacity"][:, None])) - + # Load all zero order SH coefficients dc_terms = onp.stack([v["f_dc_0"], v["f_dc_1"], v["f_dc_2"]], axis=1) @@ -94,17 +95,21 @@ def load_ply_file(ply_file_path: Path, center: bool = False) -> SplatFile: # Note: .ply file supports maximum SH degree of 3, R = f_rest_0, ... f_rest_14; G = f_rest_15, ... f_rest_29 rest_terms = [] i = 0 - #while f"f_rest_{i}" in v: + # while f"f_rest_{i}" in v: while i < 15: - rest_terms.append(v[f"f_rest_{i}"]) # has shape (numGaussians, ) + rest_terms.append(v[f"f_rest_{i}"]) # has shape (numGaussians, ) rest_terms.append(v[f"f_rest_{15 + i}"]) rest_terms.append(v[f"f_rest_{30 + i}"]) i += 1 # while f"f_rest_{i}" in v: # rest_terms.append(v[f"f_rest_{i}"]) # i += 1 - if len(rest_terms) > 0: # if we do have higher than zero order SH, we will process them and add them here. - sh_coeffs = onp.stack([v["f_dc_0"], v["f_dc_1"], v["f_dc_2"]] + rest_terms, axis=1) + if ( + len(rest_terms) > 0 + ): # if we do have higher than zero order SH, we will process them and add them here. + sh_coeffs = onp.stack( + [v["f_dc_0"], v["f_dc_1"], v["f_dc_2"]] + rest_terms, axis=1 + ) sh_degree = int(onp.sqrt(sh_coeffs.shape[1] // 3) - 1) Rs = tf.SO3(wxyzs).as_matrix() @@ -125,15 +130,15 @@ def load_ply_file(ply_file_path: Path, center: bool = False) -> SplatFile: print( f"PLY file with {num_gaussians=} loaded in {time.time() - start_time} seconds" ) - print(onp.stack([v["f_dc_0"], v["f_dc_1"], v["f_dc_2"]], axis=1)[0 , :]) - print(sh_coeffs[0, :]) # first gaussian, all 48 coefficients + print(onp.stack([v["f_dc_0"], v["f_dc_1"], v["f_dc_2"]], axis=1)[0, :]) + print(sh_coeffs[0, :]) # first gaussian, all 48 coefficients # print(sh_coeffs[0, :]) # next 3 SH coefficients that are the 1st order return { "centers": positions[0:1, :], "rgbs": colors[0:1, :], "opacities": opacities[0:1, :], - "covariances": 10000*covariances[0:1, :], + "covariances": 10000 * covariances[0:1, :], "sh_degree": sh_degree, "sh_coeffs": sh_coeffs[0:1, :], } @@ -165,6 +170,7 @@ def _(event: viser.GuiEvent) -> None: raise SystemExit("Please provide a filepath to a .splat or .ply file.") server.scene.add_transform_controls(f"/{i}") + gs_handle = server.scene._add_gaussian_splats( f"/{i}/gaussian_splats", centers=splat_data["centers"], @@ -188,7 +194,3 @@ def _(_, gs_handle=gs_handle, remove_button=remove_button) -> None: if __name__ == "__main__": tyro.cli(main) - - - -print("yapyap") \ No newline at end of file diff --git a/src/viser/_messages.py b/src/viser/_messages.py index 43ac1eec..0961a1fe 100644 --- a/src/viser/_messages.py +++ b/src/viser/_messages.py @@ -761,14 +761,15 @@ class GaussianSplatsMessage(Message): - cov5 (f16), cov6 (f16) - rgba (int32) Where cov1-6 are the upper triangular elements of the covariance matrix.""" - + sh_buffer: onpt.NDArray[onp.uint32] """The spherical harmonic buffer contains: - - - - - - + For every 24 uint32s, read each coefficient as a float16: + - r: every 0 mod3 float16 + - g: every 1 mod3 float16 + - b: every 2 mod3 float 16 + goes from 0th order until 3rd order """ - @dataclasses.dataclass diff --git a/src/viser/client/src/MessageHandler.tsx b/src/viser/client/src/MessageHandler.tsx index 67878c2f..a8fea8c8 100644 --- a/src/viser/client/src/MessageHandler.tsx +++ b/src/viser/client/src/MessageHandler.tsx @@ -983,6 +983,7 @@ function useMessageHandler() { return; } case "GaussianSplatsMessage": { + addSceneNodeMakeParents( new SceneNode(message.name, (ref) => { return ( diff --git a/src/viser/client/src/Splatting/spherical_harmonics_testing.ipynb b/src/viser/client/src/Splatting/spherical_harmonics_testing.ipynb index 29c5acf7..ff1a2313 100644 --- a/src/viser/client/src/Splatting/spherical_harmonics_testing.ipynb +++ b/src/viser/client/src/Splatting/spherical_harmonics_testing.ipynb @@ -29,7 +29,6 @@ "# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE\n", "# POSSIBILITY OF SUCH DAMAGE.\n", "\n", - "import torch\n", "\n", "C0 = 0.28209479177387814\n", "C1 = 0.4886025119029199\n", @@ -141,6 +140,7 @@ ], "source": [ "import numpy as np\n", + "\n", "deg = 3\n", "sh = np.random.rand(3, 16)\n", "dirs = np.array([1, 0, 0])\n", diff --git a/src/viser/client/src/WebsocketMessages.tsx b/src/viser/client/src/WebsocketMessages.tsx index 264fcbf0..74c44aa3 100644 --- a/src/viser/client/src/WebsocketMessages.tsx +++ b/src/viser/client/src/WebsocketMessages.tsx @@ -850,8 +850,8 @@ export interface CubicBezierSplineMessage { export interface GaussianSplatsMessage { type: "GaussianSplatsMessage"; name: string; - buffer: Uint8Array; - sh_buffer: Uint8Array; + buffer: Uint32Array; + sh_buffer: Uint32Array; } /** Message from server->client requesting a render of the current viewport. *