From 5cadd539d11b8901db87f76e7017ebdf7b35f2b7 Mon Sep 17 00:00:00 2001 From: Jonas Kulhanek Date: Fri, 29 Dec 2023 13:19:49 +0100 Subject: [PATCH 1/4] Prepare CameraTrajectoryPanel --- src/viser/client/src/App.tsx | 8 +- src/viser/client/src/CameraControls.tsx | 3 + .../ControlPanel/CameraTrajectoryPanel.tsx | 2056 +++++++++++++++++ .../client/src/ControlPanel/ControlPanel.tsx | 2 + .../client/src/ControlPanel/GuiState.tsx | 2 + .../src/ControlPanel/MultiSlider.styles.tsx | 243 ++ .../client/src/ControlPanel/MultiSlider.tsx | 774 +++++++ src/viser/client/src/ControlPanel/curve.tsx | 115 + src/viser/client/src/ThreeAssets.tsx | 16 +- 9 files changed, 3210 insertions(+), 9 deletions(-) create mode 100644 src/viser/client/src/ControlPanel/CameraTrajectoryPanel.tsx create mode 100644 src/viser/client/src/ControlPanel/MultiSlider.styles.tsx create mode 100644 src/viser/client/src/ControlPanel/MultiSlider.tsx create mode 100644 src/viser/client/src/ControlPanel/curve.tsx diff --git a/src/viser/client/src/App.tsx b/src/viser/client/src/App.tsx index 02de0b335..03ffccb1f 100644 --- a/src/viser/client/src/App.tsx +++ b/src/viser/client/src/App.tsx @@ -50,6 +50,7 @@ import { GetRenderRequestMessage, Message } from "./WebsocketMessages"; import { makeThrottledMessageSender } from "./WebsocketFunctions"; import { useDisclosure } from "@mantine/hooks"; import { computeR_threeworld_world } from "./WorldTransformUtils"; +import { stat } from "fs"; export type ViewerContextContents = { // Zustand hooks. @@ -187,7 +188,7 @@ function ViewerContents() { - {viewer.useGui((state) => state.theme.show_logo) ? ( + {viewer.useGui((state) => state.theme.show_logo && !state.isRenderMode) ? ( ) : null} @@ -204,6 +205,7 @@ function ViewerCanvas({ children }: { children: React.ReactNode }) { viewer.websocketRef, 20, ); + const isRenderMode = viewer.useGui((state) => state.isRenderMode); return ( - + {(!isRenderMode) && ( + + )} state.camera as PerspectiveCamera); + const isRenderMode = viewer.useGui((state) => state.isRenderMode); const sendCameraThrottled = makeThrottledMessageSender( viewer.websocketRef, @@ -43,6 +44,7 @@ export function SynchronizedCameraControls() { // Callback for sending cameras. const sendCamera = React.useCallback(() => { + console.log("Sending camera"); const three_camera = camera; const camera_control = viewer.cameraControlRef.current; @@ -198,6 +200,7 @@ export function SynchronizedCameraControls() { return ( void): void { + const inputElemenet = document.createElement('input'); + inputElemenet.style.display = 'none'; + inputElemenet.type = 'file'; + + inputElemenet.addEventListener('change', () => { + if (inputElemenet.files) { + onFilePicked(inputElemenet.files[0]); + } + }); + + const teardown = () => { + document.body.removeEventListener('focus', teardown, true); + setTimeout(() => { + document.body.removeChild(inputElemenet); + }, 1000); + } + document.body.addEventListener('focus', teardown, true); + + document.body.appendChild(inputElemenet); + inputElemenet.click(); +} + + +export interface CameraTrajectoryPanelProps { + visible: boolean +} + +export interface Camera { + time: number + name: string + fov: number + position: [number, number, number] + wxyz: [number, number, number, number] +} + +function getCameraHash({ fov, position, wxyz }: { fov: number, position: [number, number, number], wxyz: [number, number, number, number] }) { + const data = [fov, ...position, ...wxyz]; + const seed = 0; + let h1 = 0xdeadbeef ^ seed, h2 = 0x41c6ce57 ^ seed; + for(let i = 0, ch; i < data.length; i++) { + h1 = Math.imul(h1 ^ data[i], 2654435761); + h2 = Math.imul(h2 ^ data[i], 1597334677); + } + h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507); + h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909); + h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507); + h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909); + return 4294967296 * (2097151 & h2) + (h1 >>> 0); +} + +function mapNumberToAlphabet(number: number) : string { + let out = "" + const n = 10 + 'z'.charCodeAt(0) - 'a'.charCodeAt(0) + 1; + while (number > 0) { + const current = number % n; + out += (current < 10) ? (current).toString() : String.fromCharCode('a'.charCodeAt(0) + current - 10); + number = Math.floor(number / n); + } + return out; +} + +function rescaleCameras(cameras: Camera[]) : Camera[] { + if (cameras.length === 0) return []; + if (cameras.length == 1) return [{...cameras[0], time: 0}]; + const min = Math.min(...cameras.map(x => x.time)); + const max = Math.max(...cameras.map(x => x.time)); + return cameras.map(x => ({...x, time: (x.time - min) / (max - min)})); +} + + +function getPoseFromCamera(viewer: ViewerContextContents) { + const three_camera = viewer.cameraRef.current!; + const R_threecam_cam = new THREE.Quaternion().setFromEuler( + new THREE.Euler(Math.PI, 0.0, 0.0), + ); + const R_world_threeworld = getR_threeworld_world(viewer).invert(); + const R_world_camera = R_world_threeworld.clone() + .multiply(three_camera.quaternion) + .multiply(R_threecam_cam); + + return { + wxyz: [ + R_world_camera.w, + R_world_camera.x, + R_world_camera.y, + R_world_camera.z, + ] as [number, number, number, number], + position: three_camera.position + .clone() + .applyQuaternion(R_world_threeworld) + .toArray(), + aspect: three_camera.aspect, + fov: (three_camera.fov * Math.PI) / 180.0, + }; +} + + +export default function CameraTrajectoryPanel(props: CameraTrajectoryPanelProps) { + if (!props.visible) { + return null; + } + + const viewer = React.useContext(ViewerContext)!; + const R_threeworld_world = getR_threeworld_world(viewer); + const removeSceneNode = viewer.useSceneTree((state) => state.removeSceneNode); + const addSceneNode = viewer.useSceneTree((state) => state.addSceneNode); + const nodeFromName = viewer.useSceneTree((state) => state.nodeFromName); + const isRenderMode = viewer.useGui((state) => state.isRenderMode); + + const baseTreeName = "CameraTrajectory" + React.useEffect(() => { + if (!(baseTreeName in nodeFromName)) { + addSceneNode( + new SceneNode(baseTreeName, (ref) => ( + + )) as SceneNode, + ); + addSceneNode( + new SceneNode( + `${baseTreeName}/PlayerCamera`, + (ref) => ( + + ), + ) as SceneNode); + const attr = viewer.nodeAttributesFromName.current; + if (attr[`${baseTreeName}/PlayerCamera`] === undefined) attr[`${baseTreeName}/PlayerCamera`] = {}; + attr[`${baseTreeName}/PlayerCamera`]!.visibility = false; + return () => { + removeSceneNode(`${baseTreeName}/PlayerCamera`); + removeSceneNode(baseTreeName); + } + } + }, []); + + const [isCycle, setIsCycle] = React.useState(false); + const [isPlaying, setIsPlaying] = React.useState(false); + const [fps, setFps] = React.useState(30); + const [smoothness, setSmoothness] = React.useState(0.5); + const [cameras, setCameras] = React.useState([]); + const [fov, setFov] = React.useState(1.); + const [renderWidth, setRenderWidth] = React.useState(1920); + const [renderHeight, setRenderHeight] = React.useState(1080); + const [playerTime, setPlayerTime] = React.useState(0.); + const aspect = renderWidth / renderHeight; + + const curveObject = React.useMemo(() => cameras.length > 1 ? get_curve_object_from_cameras( + cameras.map(({fov, wxyz, position, time}: Camera) => ({ + time, + fov, + position: new THREE.Vector3(...position), + quaternion: new THREE.Quaternion(wxyz[1], wxyz[2], wxyz[3], wxyz[0]), + })), isCycle, smoothness) : null, [cameras, isCycle, smoothness]); + + // Update cameras and trajectory + React.useEffect(() => { + // Update trajectory + if (!(baseTreeName in nodeFromName)) return; + const children = nodeFromName[baseTreeName]!.children; + const enabledCameras = new Set(cameras.map(x => `${baseTreeName}/Camera.${x.name}`)); + children.filter((c) => !enabledCameras.has(c)).forEach((c) => { + removeSceneNode(c); + const attr = viewer.nodeAttributesFromName.current; + if (attr[c] !== undefined) delete attr[c]; + }); + cameras.forEach((camera, index) => { + const nodeName = `${baseTreeName}/Camera.${camera.name}`; + if (!(nodeName in nodeFromName)) { + const node: SceneNode = new SceneNode( + `${baseTreeName}/Camera.${camera.name}`, + (ref) => ( + + ), + ); + addSceneNode(node); + } + const attr = viewer.nodeAttributesFromName.current; + if (attr[nodeName] === undefined) attr[nodeName] = {}; + attr[nodeName]!.wxyz = camera.wxyz; + attr[nodeName]!.position = camera.position; + }); + }, [cameras, aspect, smoothness]); + + + // Render camera path + React.useEffect(() => { + if (!(baseTreeName in nodeFromName)) return; + const nodeName = `${baseTreeName}/Trajectory`; + if (curveObject !== null) { + const num_points = fps * seconds; + const points = curveObject.curve_positions.getPoints(num_points); + const resolution = new THREE.Vector2( window.innerWidth, window.innerHeight ); + const geometry = new MeshLineGeometry(); + geometry.setPoints(points); + const material = new MeshLineMaterial({ + lineWidth: 0.03, + color: 0xff5024, + resolution, + }); + addSceneNode(new SceneNode(nodeName, (ref) => { + return + }, () => { + geometry.dispose(); + material.dispose(); + }) as SceneNode); + + } else if (nodeName in nodeFromName) { + removeSceneNode(nodeName); + } + }, [curveObject, fps, isRenderMode]); + + React.useEffect(() => { + // set the camera + if (curveObject !== null) { + if (isRenderMode) { + const point = getKeyframePoint(playerTime); + const position = curveObject.curve_positions.getPoint(point).applyQuaternion(R_threeworld_world); + const lookat = curveObject.curve_lookats.getPoint(point).applyQuaternion(R_threeworld_world); + const up = curveObject.curve_ups.getPoint(point).multiplyScalar(-1).applyQuaternion(R_threeworld_world); + const fov = curveObject.curve_fovs.getPoint(point).z; + + const cameraControls = viewer.cameraControlRef.current!; + const threeCamera = viewer.cameraRef.current!; + + threeCamera.up.set(...up.toArray()); + cameraControls.updateCameraUp(); + cameraControls.setLookAt(...position.toArray(), ...lookat.toArray(), false); + const target = position.clone().add(lookat); + // NOTE: lookat is being ignore when calling setLookAt + cameraControls.setTarget(...target.toArray(), false); + threeCamera.setFocalLength( + (0.5 * threeCamera.getFilmHeight()) / Math.tan(fov / 2.0), + ); + cameraControls.update(1.); + } else { + const point = getKeyframePoint(playerTime); + const position = curveObject.curve_positions.getPoint(point); + const lookat = curveObject.curve_lookats.getPoint(point); + const up = curveObject.curve_ups.getPoint(point); + + const mat = get_transform_matrix(position, lookat, up); + const quaternion = new THREE.Quaternion().setFromRotationMatrix(mat); + + addSceneNode( + new SceneNode( + `${baseTreeName}/PlayerCamera`, + (ref) => ( + + ), + ) as SceneNode); + const attr = viewer.nodeAttributesFromName.current; + if (attr[`${baseTreeName}/PlayerCamera`] === undefined) attr[`${baseTreeName}/PlayerCamera`] = {}; + attr[`${baseTreeName}/PlayerCamera`]!.visibility = true; + attr[`${baseTreeName}/PlayerCamera`]!.wxyz = [quaternion.w, quaternion.x, quaternion.y, quaternion.z]; + attr[`${baseTreeName}/PlayerCamera`]!.position = position.toArray(); + } + } else { + const attr = viewer.nodeAttributesFromName.current; + if (attr[`${baseTreeName}/PlayerCamera`] !== undefined) + attr[`${baseTreeName}/PlayerCamera`]!.visibility = false; + } + }, [curveObject, fps, isRenderMode, playerTime]); + + const [seconds, setSeconds] = React.useState(4); + + React.useEffect(() => { + if (isPlaying && cameras.length > 1) { + const interval = setInterval(() => { + setPlayerTime((prev) => { + let out = Math.min(1., prev + 1 / (fps * seconds)) + if (out >= 1) { + setIsPlaying(false); + out = 0; + } + return out; + }); + }, 1000 / fps); + return () => clearInterval(interval); + } + }, [isPlaying, seconds, fps]); + + + const addCamera = () => { + const { position, wxyz } = getPoseFromCamera(viewer); + const hash = getCameraHash({ fov, position, wxyz }); + const name = `${mapNumberToAlphabet(hash).slice(0, 6)}`; + + if (cameras.length >= 2) { + const mult = 1 - 1/cameras.length; + setCameras([...cameras.map(x => ({...x, time: x.time * mult})), { + time: 1, + name, + position, + wxyz, + fov, + }]); + } else { + setCameras([...cameras, { + time: cameras.length === 0 ? 0 : 1, + name, + position, + wxyz, + fov, + }]); + } + } + + + const displayRenderTime = false; + + + const getKeyframePoint = (progress: number) => { + const times = []; + const ratio = (cameras.length - 1) / cameras.length; + cameras.forEach((camera) => { + const time = camera.time; + times.push(isCycle ? time * ratio : time); + }); + + if (isCycle) { + times.push(1.0); + } + + let new_point = 0.0; + if (progress <= times[0]) { + new_point = 0.0; + } else if (progress >= times[times.length - 1]) { + new_point = 1.0; + } else { + let i = 0; + while ( + i < times.length - 1 && + !(progress >= times[i] && progress < times[i + 1]) + ) { + i += 1; + } + const percentage = (progress - times[i]) / (times[i + 1] - times[i]); + new_point = (i + percentage) / (times.length - 1); + } + return new_point; + }; + + + const loadCameraPath = (camera_path_object: any) => { + const new_camera_list = []; + + setRenderHeight(camera_path_object.render_height); + setRenderWidth(camera_path_object.render_width); + if (camera_path_object.camera_type !== "perspective") { + // TODO: handle this better! + throw new Error("Unsupported camera type: " + camera_path_object.camera_type); + } + + setFps(camera_path_object.fps); + setSeconds(camera_path_object.seconds); + + setSmoothness(camera_path_object.smoothness_value); + setIsCycle(camera_path_object.is_cycle); + + for (let i = 0; i < camera_path_object.keyframes.length; i += 1) { + const keyframe = camera_path_object.keyframes[i]; + + // properties + const properties = new Map(JSON.parse(keyframe.properties)); + const mat = new THREE.Matrix4(); + mat.fromArray(JSON.parse(keyframe.matrix)); + const position = new THREE.Vector3(); + const quaternion = new THREE.Quaternion(); + const scale = new THREE.Vector3(); + mat.decompose(position, quaternion, scale); + + // aspect = keyframe.aspect; + const camera: Camera = { + position: position.toArray(), + wxyz: [quaternion.w, quaternion.x, quaternion.y, quaternion.z], + name: properties.get("NAME") as string, + time: properties.get("TIME") as number, + fov: keyframe.fov, + }; + new_camera_list.push(camera); + } + setCameras(new_camera_list); + } + + + const getCameraPath = () => { + const num_points = Math.round(fps * seconds); + const camera_path = []; + + for (let i = 0; i < num_points; i += 1) { + const pt = getKeyframePoint(i / num_points); + + const position = curveObject!.curve_positions.getPoint(pt); + const lookat = curveObject!.curve_lookats.getPoint(pt); + const up = curveObject!.curve_ups.getPoint(pt); + const fov = curveObject!.curve_fovs.getPoint(pt).z; + + const mat = get_transform_matrix(position, lookat, up); + + if (displayRenderTime) { + const renderTime = curveObject!.curve_render_times.getPoint(pt).z; + camera_path.push({ + camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix + fov, + aspect: aspect, + render_time: Math.max(Math.min(renderTime, 1.0), 0.0), // clamp time values to [0, 1] + }); + } else { + camera_path.push({ + camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix + fov, + aspect: aspect, + }); + } + } + + const keyframes = []; + for (let i = 0; i < cameras.length; i += 1) { + const camera = cameras[i]; + + const up = new THREE.Vector3(0, 1, 0); // y is up in local space + const lookat = new THREE.Vector3(0, 0, 1); // z is forward in local space + const wxyz = camera.wxyz + const quaternion = new THREE.Quaternion(wxyz[1], wxyz[2], wxyz[3], wxyz[0]); + + up.applyQuaternion(quaternion); + lookat.applyQuaternion(quaternion); + + const matrix = get_transform_matrix(new THREE.Vector3(...camera.position), lookat, up); + keyframes.push({ + matrix: JSON.stringify(matrix.toArray()), + fov: camera.fov, + aspect: aspect, + properties: JSON.stringify([ + ["FOV", camera.fov], + ["NAME", camera.name], + ["TIME", camera.time], + ]), + }); + } + + // const myData + const camera_path_object = { + format: "nerfstudio-viewer", + keyframes, + camera_type: "perspective", + render_height: renderHeight, + render_width: renderWidth, + camera_path, + fps, + seconds, + smoothness_value: smoothness, + is_cycle: isCycle, + crop: null, + }; + return camera_path_object; + } + + + const exportCameraPath = () => { + const cameraPath = getCameraPath(); + + // create file in browser + const json = JSON.stringify(cameraPath, null, 2); + const blob = new Blob([json], { type: 'application/json' }); + const href = URL.createObjectURL(blob); + + // create "a" HTLM element with href to file + const link = document.createElement('a'); + link.href = href; + + const filename = 'camera_path.json'; + link.download = filename; + document.body.appendChild(link); + link.click(); + // clean up "a" element & remove ObjectURL + document.body.removeChild(link); + URL.revokeObjectURL(href); + } + + const uploadCameraPath = () => { + pickFile((file) => { + const fr = new FileReader(); + fr.onload = (res) => { + const camera_path_object = JSON.parse(res.target!.result! as string); + loadCameraPath(camera_path_object); + }; + fr.readAsText(file); + }); + } + + const marks = []; + for (let i = 0; i <= 1; i += 0.25) { + marks.push({ value: i, label: `${(seconds * i).toFixed(1).toString()}s` }); + } + + // const table = useMantineReactTable({ + // data: cameras, + // enableBottomToolbar: false, + // enableTopToolbar: false, + // enableSorting: false, + // enableColumnActions: false, + // enableDensityToggle: false, + // enableRowSelection: false, + // enableRowOrdering: true, + // enableHiding: false, + // enableColumnFilters: false, + // enablePagination: false, + // mantineTableProps: { + // verticalSpacing: 2, + // }, + // enableRowActions: true, + // mantinePaperProps: { shadow: undefined }, + // mantineTableContainerProps: { sx: { maxHeight: "30em" } }, + // mantinePaginationProps: { + // showRowsPerPage: false, + // }, + // displayColumnDefOptions: { + // 'mrt-row-drag': { + // header: "", + // minSize: 0, + // size: 10, + // maxSize: 10, + // }, + // 'mrt-row-actions': { + // header: "", + // minSize: 0, + // size: 10, + // maxSize: 10, + // }, + // }, + // renderRowActions: (row) => { + // return { + // const cameraId = row.row.original.name; + // setCameras(rescaleCameras(cameras.filter(x => x.name !== cameraId))); + // }}> + // + // + // }, + // // mantineTableBodyRowProps={({ row }) => ({ + // // onPointerOver: () => { + // // setLabelVisibility(row.getValue("name"), true); + // // }, + // // onPointerOut: () => { + // // setLabelVisibility(row.getValue("name"), false);/ + // // }, + // // ...(row.subRows === undefined || row.subRows.length === 0 + // // ? {} + // // : { + // // onClick: () => { + // // row.toggleExpanded(); + // // }, + // // sx: { + // // cursor: "pointer", + // // }, + // // }), + // // })} + // mantineRowDragHandleProps: ({table}) => ({ + // onDragEnd: () => { + // const { draggingRow, hoveredRow } = table.getState(); + // if (hoveredRow && draggingRow) { + // const clone = [...cameras]; + // clone.splice(hoveredRow.index, 0, clone.splice(draggingRow.index, 1)[0]); + // } + // }, + // }), + // columns: [{ + // header: "Time", + // minSize: 10, + // size: 30, + // accessorFn(originalRow) { + // return (originalRow.time * seconds).toFixed(2).toString() + "s"; + // }, + // }, { + // size: 30, + // minSize: 10, + // header: "Camera", + // accessorKey: "name", + // }], + // initialState: { + // density: "xs", + // } + // }); + + return (<> + + + + + + } + step={1} + size="xs" + onChange={(newValue) => newValue !== "" && setFps(newValue)} + styles={{ + input: { + minHeight: "1.625rem", + height: "1.625rem", + }, + }} + stepHoldDelay={500} + stepHoldInterval={(t) => Math.max(1000 / t ** 2, 25)} + /> + } + step={1.0} + size="xs" + onChange={(newValue) => newValue !== "" && setFov(newValue)} + styles={{ + input: { + minHeight: "1.625rem", + height: "1.625rem", + }, + }} + stepHoldDelay={500} + stepHoldInterval={(t) => Math.max(1000 / t ** 2, 25)} + /> + + + } + step={100} + size="xs" + onChange={(newValue) => newValue !== "" && setRenderWidth(newValue)} + styles={{ + input: { + minHeight: "1.625rem", + height: "1.625rem", + }, + }} + stepHoldDelay={500} + stepHoldInterval={(t) => Math.max(1000 / t ** 2, 25)} + /> + } + step={100} + size="xs" + onChange={(newValue) => newValue !== "" && setRenderHeight(newValue)} + styles={{ + input: { + minHeight: "1.625rem", + height: "1.625rem", + }, + }} + stepHoldDelay={500} + stepHoldInterval={(t) => Math.max(1000 / t ** 2, 25)} + /> + + } + step={1.00} + size="xs" + onChange={(newValue) => newValue !== "" && setSeconds(newValue)} + styles={{ + input: { + minHeight: "1.625rem", + height: "1.625rem", + }, + }} + stepHoldDelay={500} + stepHoldInterval={(t) => Math.max(1000 / t ** 2, 25)} + /> + Smoothness + + + Timeline + `${(seconds*x).toFixed(2)}s`} + value={cameras.map(x=>x.time)} + onChange={(value) => { + setCameras(cameras.map((camera, index) => ({ + ...camera, + time: value[index], + }))); + }} + onChangeEnd={(value) => { + setCameras(cameras.map((camera, index) => ({ + ...camera, + time: value[index], + }))); + }} + marks={marks} /> + + + + + + + + + + + {cameras.map((camera, index) => { + return ( + + + + + + ) + })} + +
TimeCamera
+ + { + setCameras(rescaleCameras([...cameras.slice(0, index), ...cameras.slice(index + 1)])); + }}> + + + { + const clone = [...cameras]; + const tmp = cameras[index]; + clone[index] = { + ...cameras[index - 1], + time: cameras[index].time + }; + clone[index - 1] = { + ...tmp, + time: cameras[index - 1].time + } + setCameras(clone); + }}> + + + { + const clone = [...cameras]; + const tmp = cameras[index]; + clone[index] = { + ...cameras[index + 1], + time: cameras[index].time + }; + clone[index + 1] = { + ...tmp, + time: cameras[index + 1].time + } + setCameras(clone); + }}> + + + + {(seconds * camera.time).toFixed(2).toString()}s{camera.name}
+ + + Player + + + setPlayerTime(Math.max(0, ...cameras.map(x => x.time).filter(x => x < playerTime)))}> + + + {isPlaying ? setIsPlaying(false)}> + + : setIsPlaying(true)}> + + } + setPlayerTime(Math.min(1, ...cameras.map(x => x.time).filter(x => x > playerTime)))}> + + + + { + viewer.useGui.setState({ isRenderMode: event.currentTarget.checked }); + }} + size="sm" + /> + ) +} + +CameraTrajectoryPanel.defaultProps = { + visible: true, +}; + + + +//function set_camera_position(camera, matrix) { +// const mat = new THREE.Matrix4(); +// mat.fromArray(matrix.elements); +// mat.decompose(camera.position, camera.quaternion, camera.scale); +//} +// +//function CameraList(props) { +// const throttled_time_message_sender = props.throttled_time_message_sender; +// const sceneTree = props.sceneTree; +// const cameras = props.cameras; +// const camera_main = props.camera_main; +// const transform_controls = props.transform_controls; +// const setCameras = props.setCameras; +// const swapCameras = props.swapCameras; +// const fovLabel = props.fovLabel; +// const setFovLabel = props.setFovLabel; +// const cameraProperties = props.cameraProperties; +// const setCameraProperties = props.setCameraProperties; +// const isAnimated = props.isAnimated; +// // eslint-disable-next-line no-unused-vars +// const slider_value = props.slider_value; +// const set_slider_value = props.set_slider_value; +// +// const [expanded, setExpanded] = React.useState(null); +// +// const camera_type = useSelector((state) => state.renderingState.camera_type); +// +// const handleChange = +// (cameraUUID: string) => +// (event: React.SyntheticEvent, isExpanded: boolean) => { +// setExpanded(isExpanded ? cameraUUID : false); +// }; +// +// const set_transform_controls = (index) => { +// // camera helper object so grab the camera inside +// const camera = sceneTree.find_object_no_create([ +// 'Camera Path', +// 'Cameras', +// index.toString(), +// 'Camera', +// ]); +// if (camera !== null) { +// const viewer_buttons = document.getElementsByClassName( +// 'ViewerWindow-buttons', +// )[0]; +// if (camera === transform_controls.object) { +// // double click to remove controls from object +// transform_controls.detach(); +// viewer_buttons.style.display = 'none'; +// } else { +// transform_controls.detach(); +// transform_controls.attach(camera); +// viewer_buttons.style.display = 'block'; +// } +// } +// }; +// +// const reset_slider_render_on_change = () => { +// // set slider and render camera back to 0 +// const slider_min = 0; +// const camera_render = sceneTree.find_object_no_create([ +// 'Cameras', +// 'Render Camera', +// ]); +// const camera_render_helper = sceneTree.find_object_no_create([ +// 'Cameras', +// 'Render Camera', +// 'Helper', +// ]); +// if (cameras.length >= 1) { +// let first_camera = sceneTree.find_object_no_create([ +// 'Camera Path', +// 'Cameras', +// 0, +// 'Camera', +// ]); +// if (first_camera.type !== 'PerspectiveCamera' && cameras.length > 1) { +// first_camera = sceneTree.find_object_no_create([ +// 'Camera Path', +// 'Cameras', +// 1, +// 'Camera', +// ]); +// } +// set_camera_position(camera_render, first_camera.matrix); +// camera_render_helper.set_visibility(true); +// camera_render.fov = first_camera.fov; +// camera_render.renderTime = first_camera.renderTime; +// } +// set_slider_value(slider_min); +// }; +// +// const delete_camera = (index: number) => { +// const camera_render_helper = sceneTree.find_object_no_create([ +// 'Cameras', +// 'Render Camera', +// 'Helper', +// ]); +// console.log('TODO: deleting camera: ', index); +// sceneTree.delete(['Camera Path', 'Cameras', index.toString(), 'Camera']); +// sceneTree.delete([ +// 'Camera Path', +// 'Cameras', +// index.toString(), +// 'Camera Helper', +// ]); +// +// setCameras([...cameras.slice(0, index), ...cameras.slice(index + 1)]); +// // detach and hide transform controls +// transform_controls.detach(); +// const viewer_buttons = document.getElementsByClassName( +// 'ViewerWindow-buttons', +// )[0]; +// viewer_buttons.style.display = 'none'; +// if (cameras.length < 1) { +// camera_render_helper.set_visibility(false); +// } +// reset_slider_render_on_change(); +// }; +// +// const cameraList = cameras.map((camera, index) => { +// return ( +// +// } +// aria-controls="panel1bh-content" +// id="panel1bh-header" +// > +// +// +// +// +// +// +// +// +// +// +// +// +// {isAnimated('FOV') && camera_type !== 'equirectangular' && ( +// HI +// )} +// {!isAnimated('FOV') && !isAnimated('RenderTime') && ( +//

+// Animated camera properties will show up here! +//

+// )} +// +// +// ); +// }); +// return
{cameraList}
; +//} +// +//export default function CameraPanel(props) { +// // unpack relevant information +// const sceneTree = props.sceneTree; +// const camera_main = sceneTree.find_object_no_create([ +// 'Cameras', +// 'Main Camera', +// ]); +// const camera_render = sceneTree.find_object_no_create([ +// 'Cameras', +// 'Render Camera', +// ]); +// const camera_render_helper = sceneTree.find_object_no_create([ +// 'Cameras', +// 'Render Camera', +// 'Helper', +// ]); +// const transform_controls = sceneTree.find_object_no_create([ +// 'Transform Controls', +// ]); +// +// // redux store state +// const DEFAULT_FOV = 50; +// const DEFAULT_RENDER_TIME = 0.0; +// +// interface Camera { +// time: number +// name: string +// }; +// +// // react state +// const [cameras, setCameras] = React.useState([]); +// // Mapping of camera id to each camera's properties +// const [cameraProperties, setCameraProperties] = React.useState(new Map()); +// const [slider_value, set_slider_value] = React.useState(0); +// const [smoothness_value, set_smoothness_value] = React.useState(0.5); +// const [is_playing, setIsPlaying] = React.useState(false); +// const [is_cycle, setIsCycle] = React.useState(false); +// const [seconds, setSeconds] = React.useState(4); +// const [fps, setFps] = React.useState(24); +// const [render_modal_open, setRenderModalOpen] = React.useState(false); +// const [load_path_modal_open, setLoadPathModalOpen] = React.useState(false); +// const [animate, setAnimate] = React.useState(new Set()); +// const [globalFov, setGlobalFov] = React.useState(DEFAULT_FOV); +// const [globalRenderTime, setGlobalRenderTime] = +// React.useState(DEFAULT_RENDER_TIME); +// +// const scene_state = sceneTree.get_scene_state(); +// +// // Template for sharing state between Vanilla JS Three.js and React components +// // eslint-disable-next-line no-unused-vars +// const [mouseInScene, setMouseInScene] = React.useState(false); +// React.useEffect(() => { +// scene_state.addCallback( +// (value: boolean) => setMouseInScene(value), +// 'mouse_in_scene', +// ); +// // eslint-disable-next-line react-hooks/exhaustive-deps +// }, []); +// +// // ui state +// const [fovLabel, setFovLabel] = React.useState(FOV_LABELS.FOV); +// +// // nonlinear render option +// const slider_min = 0; +// const slider_max = 1; +// +// // animation constants +// const total_num_steps = seconds * fps; +// const step_size = slider_max / total_num_steps; +// +// const reset_slider_render_on_add = (new_camera_list) => { +// // set slider and render camera back to 0 +// if (new_camera_list.length >= 1) { +// set_camera_position(camera_render, new_camera_list[0].matrix); +// setFieldOfView(new_camera_list[0].fov); +// set_slider_value(slider_min); +// } +// }; +// +// const add_camera = () => { +// const camera_main_copy = camera_main.clone(); +// camera_main_copy.aspect = 1.0; +// camera_main_copy.fov = globalFov; +// camera_main_copy.renderTime = globalRenderTime; +// const new_camera_properties = new Map(); +// camera_main_copy.properties = new_camera_properties; +// new_camera_properties.set('FOV', globalFov); +// new_camera_properties.set('NAME', `Camera ${cameras.length}`); +// // TIME VALUES ARE 0-1 +// if (cameras.length === 0) { +// new_camera_properties.set('TIME', 0.0); +// } else { +// new_camera_properties.set('TIME', 1.0); +// } +// +// const ratio = (cameras.length - 1) / cameras.length; +// +// const new_properties = new Map(cameraProperties); +// new_properties.forEach((properties) => { +// properties.set('TIME', properties.get('TIME') * ratio); +// }); +// new_properties.set(camera_main_copy.uuid, new_camera_properties); +// setCameraProperties(new_properties); +// +// const new_camera_list = cameras.concat(camera_main_copy); +// setCameras(new_camera_list); +// reset_slider_render_on_add(new_camera_list); +// }; +// +// const setCameraProperty = (property, value, index) => { +// const activeCamera = cameras[index]; +// const activeProperties = new Map(activeCamera.properties); +// activeProperties.set(property, value); +// const newProperties = new Map(cameraProperties); +// newProperties.set(activeCamera.uuid, activeProperties); +// activeCamera.properties = activeProperties; +// setCameraProperties(newProperties); +// }; +// +// const swapCameras = (index: number, new_index: number) => { +// if ( +// Math.min(index, new_index) < 0 || +// Math.max(index, new_index) >= cameras.length +// ) +// return; +// +// const swapCameraTime = cameras[index].time; +// cameras[index].time = cameras[new_index].time; +// cameras[new_index].time = swapCameraTime; +// +// const new_cameras = [ +// ...cameras.slice(0, index), +// ...cameras.slice(index + 1), +// ]; +// setCameras([ +// ...new_cameras.slice(0, new_index), +// cameras[index], +// ...new_cameras.slice(new_index), +// ]); +// +// // reset_slider_render_on_change(); +// }; +// +// // force a rerender if the cameras are dragged around +// let update_cameras_interval = null; +// // eslint-disable-next-line no-unused-vars +// transform_controls.addEventListener('mouseDown', (event) => { +// // prevent multiple loops +// if (update_cameras_interval === null) { +// // hardcoded for 100 ms per update +// update_cameras_interval = setInterval(() => {}, 100); +// } +// }); +// // eslint-disable-next-line no-unused-vars +// transform_controls.addEventListener('mouseUp', (event) => { +// if (update_cameras_interval !== null) { +// clearInterval(update_cameras_interval); +// update_cameras_interval = null; +// setCameras(cameras); +// } +// }); +// +// // draw cameras and curve to the scene +// useEffect(() => { +// // draw the cameras +// +// const labels = Array.from(document.getElementsByClassName('label')); +// labels.forEach((label) => { +// label.remove(); +// }); +// +// sceneTree.delete(['Camera Path', 'Cameras']); // delete old cameras, which is important +// if (cameras.length < 1) { +// dispatch({ +// type: 'write', +// path: 'renderingState/camera_choice', +// data: 'Main Camera', +// }); +// camera_render_helper.set_visibility(false); +// } else { +// camera_render_helper.set_visibility(true); +// } +// for (let i = 0; i < cameras.length; i += 1) { +// const camera = cameras[i]; +// // camera.aspect = render_width / render_height; +// const camera_helper = new CameraHelper(camera, 0x393e46); +// +// const labelDiv = document.createElement('div'); +// labelDiv.className = 'label'; +// labelDiv.textContent = camera.name; +// labelDiv.style.color = 'black'; +// labelDiv.style.backgroundColor = 'rgba(255, 255, 255, 0.61)'; +// labelDiv.style.backdropFilter = 'blur(5px)'; +// labelDiv.style.padding = '6px'; +// labelDiv.style.borderRadius = '6px'; +// labelDiv.style.visibility = 'visible'; +// const camera_label = new CSS2DObject(labelDiv); +// camera_label.name = 'CAMERA_LABEL'; +// camera_label.position.set(0, -0.1, -0.1); +// camera_helper.add(camera_label); +// camera_label.layers.set(0); +// +// // camera +// sceneTree.set_object_from_path( +// ['Camera Path', 'Cameras', i.toString(), 'Camera'], +// camera, +// ); +// // camera helper +// sceneTree.set_object_from_path( +// ['Camera Path', 'Cameras', i.toString(), 'Camera Helper'], +// camera_helper, +// ); +// } +// // eslint-disable-next-line react-hooks/exhaustive-deps +// }, [cameras, cameraProperties, render_width, render_height]); +// +// // update the camera curve +// const curve_object = get_curve_object_from_cameras( +// cameras, +// is_cycle, +// smoothness_value, +// ); +// +// const getKeyframePoint = (progress: Number) => { +// const times = []; +// const ratio = (cameras.length - 1) / cameras.length; +// cameras.forEach((camera) => { +// const time = camera.properties.get('TIME'); +// times.push(is_cycle ? time * ratio : time); +// }); +// +// if (is_cycle) { +// times.push(1.0); +// } +// +// let new_point = 0.0; +// if (progress <= times[0]) { +// new_point = 0.0; +// } else if (progress >= times[times.length - 1]) { +// new_point = 1.0; +// } else { +// let i = 0; +// while ( +// i < times.length - 1 && +// !(progress >= times[i] && progress < times[i + 1]) +// ) { +// i += 1; +// } +// const percentage = (progress - times[i]) / (times[i + 1] - times[i]); +// new_point = (i + percentage) / (times.length - 1); +// } +// return new_point; +// }; +// +// if (cameras.length > 1) { +// const num_points = fps * seconds; +// const points = curve_object.curve_positions.getPoints(num_points); +// const geometry = new THREE.BufferGeometry().setFromPoints(points); +// const spline = new MeshLine(); +// spline.setGeometry(geometry); +// const material = new MeshLineMaterial({ lineWidth: 0.01, color: 0xff5024 }); +// const spline_mesh = new THREE.Mesh(spline.geometry, material); +// sceneTree.set_object_from_path(['Camera Path', 'Curve'], spline_mesh); +// +// // set the camera +// +// const point = getKeyframePoint(slider_value); +// let position = null; +// let lookat = null; +// let up = null; +// let fov = null; +// position = curve_object.curve_positions.getPoint(point); +// lookat = curve_object.curve_lookats.getPoint(point); +// up = curve_object.curve_ups.getPoint(point); +// fov = curve_object.curve_fovs.getPoint(point).z; +// +// const mat = get_transform_matrix(position, lookat, up); +// set_camera_position(camera_render, mat); +// setFieldOfView(fov); +// } else { +// sceneTree.delete(['Camera Path', 'Curve']); +// } +// +// const values = []; +// cameras.forEach((camera) => { +// const time = camera.properties.get('TIME'); +// const ratio = (cameras.length - 1) / cameras.length; +// values.push(is_cycle ? time * ratio : time); +// }); +// +// if (is_cycle && cameras.length !== 0) { +// values.push(1.0); +// } +// +// const handleKeyframeSlider = ( +// newValue: number | number[], +// activeThumb: number, +// ) => { +// if (activeThumb === cameras.length) return; +// const ratio = (cameras.length - 1) / cameras.length; +// const val = newValue[activeThumb]; +// setCameraProperty( +// 'TIME', +// is_cycle ? Math.min(val / ratio, 1.0) : val, +// activeThumb, +// ); +// }; +// +// // when the slider changes, update the main camera position +// useEffect(() => { +// if (cameras.length > 1) { +// const point = getKeyframePoint(slider_value); +// let position = null; +// let lookat = null; +// let up = null; +// let fov = null; +// let render_time = null; +// position = curve_object.curve_positions.getPoint(point); +// lookat = curve_object.curve_lookats.getPoint(point); +// up = curve_object.curve_ups.getPoint(point); +// fov = curve_object.curve_fovs.getPoint(point).z; +// render_time = curve_object.curve_render_times.getPoint(point).z; +// render_time = Math.max(Math.min(render_time, 1.0), 0.0); // clamp time values to [0, 1] +// const mat = get_transform_matrix(position, lookat, up); +// set_camera_position(camera_render, mat); +// setFieldOfView(fov); +// setGlobalFov(fov); +// setGlobalRenderTime(render_time); +// } +// // eslint-disable-next-line react-hooks/exhaustive-deps +// }, [slider_value, render_height, render_width]); +// +// // call this function whenever slider state changes +// useEffect(() => { +// if (is_playing && cameras.length > 1) { +// const interval = setInterval(() => { +// set_slider_value((prev) => prev + step_size); +// }, 1000 / fps); +// return () => clearInterval(interval); +// } +// return () => {}; +// // eslint-disable-next-line react-hooks/exhaustive-deps +// }, [is_playing]); +// +// // make sure to pause if the slider reaches the end +// useEffect(() => { +// if (slider_value >= slider_max) { +// set_slider_value(slider_max); +// setIsPlaying(false); +// } +// }, [slider_value]); +// +// const get_camera_path = () => { +// // NOTE: currently assuming these are ints +// const num_points = fps * seconds; +// const camera_path = []; +// +// for (let i = 0; i < num_points; i += 1) { +// const pt = getKeyframePoint(i / num_points); +// +// const position = curve_object.curve_positions.getPoint(pt); +// const lookat = curve_object.curve_lookats.getPoint(pt); +// const up = curve_object.curve_ups.getPoint(pt); +// const fov = curve_object.curve_fovs.getPoint(pt).z; +// +// const mat = get_transform_matrix(position, lookat, up); +// +// if (display_render_time) { +// const renderTime = curve_object.curve_render_times.getPoint(pt).z; +// camera_path.push({ +// camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix +// fov, +// aspect: camera_render.aspect, +// render_time: Math.max(Math.min(renderTime, 1.0), 0.0), // clamp time values to [0, 1] +// }); +// } else { +// camera_path.push({ +// camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix +// fov, +// aspect: camera_render.aspect, +// }); +// } +// } +// +// const keyframes = []; +// for (let i = 0; i < cameras.length; i += 1) { +// const camera = cameras[i]; +// keyframes.push({ +// matrix: JSON.stringify(camera.matrix.toArray()), +// fov: camera.fov, +// aspect: camera_render.aspect, +// properties: JSON.stringify(Array.from(camera.properties.entries())), +// }); +// } +// +// // const myData +// const camera_path_object = { +// keyframes, +// camera_type, +// render_height, +// render_width, +// camera_path, +// fps, +// seconds, +// smoothness_value, +// is_cycle, +// crop: null, +// }; +// return camera_path_object; +// }; +// +// const export_camera_path = () => { +// // export the camera path +// // inspired by: +// // https://stackoverflow.com/questions/55613438/reactwrite-to-json-file-or-export-download-no-server +// +// sendWebsocketMessage(viser_websocket, { type: 'SaveCheckpointMessage' }); +// +// const camera_path_object = get_camera_path(); +// console.log(camera_render.toJSON()); +// +// // create file in browser +// const json = JSON.stringify(camera_path_object, null, 2); +// const blob = new Blob([json], { type: 'application/json' }); +// const href = URL.createObjectURL(blob); +// +// // create "a" HTLM element with href to file +// const link = document.createElement('a'); +// link.href = href; +// +// const filename = 'camera_path.json'; +// link.download = filename; +// document.body.appendChild(link); +// link.click(); +// // clean up "a" element & remove ObjectURL +// document.body.removeChild(link); +// URL.revokeObjectURL(href); +// }; +// +// const load_camera_path = (camera_path_object) => { +// const new_camera_list = []; +// const new_properties = new Map(cameraProperties); +// +// setRenderHeight(camera_path_object.render_height); +// setRenderWidth(camera_path_object.render_width); +// setCameraType(camera_path_object.camera_type); +// +// setFps(camera_path_object.fps); +// setSeconds(camera_path_object.seconds); +// +// set_smoothness_value(camera_path_object.smoothness_value); +// setIsCycle(camera_path_object.is_cycle); +// +// for (let i = 0; i < camera_path_object.keyframes.length; i += 1) { +// const keyframe = camera_path_object.keyframes[i]; +// const camera = new THREE.PerspectiveCamera( +// keyframe.fov, +// keyframe.aspect, +// 0.1, +// 1000, +// ); +// +// // properties +// camera.properties = new Map(JSON.parse(keyframe.properties)); +// new_properties.set(camera.uuid, camera.properties); +// +// const mat = new THREE.Matrix4(); +// mat.fromArray(JSON.parse(keyframe.matrix)); +// set_camera_position(camera, mat); +// new_camera_list.push(camera); +// } +// +// setCameraProperties(new_properties); +// setCameras(new_camera_list); +// reset_slider_render_on_add(new_camera_list); +// +// if ('crop' in camera_path_object && camera_path_object.crop !== null) { +// const bg_color = camera_path_object.crop.crop_bg_color; +// sendWebsocketMessage(viser_websocket, { +// type: 'CropParamsMessage', +// crop_enabled: true, +// crop_bg_color: [bg_color.r, bg_color.g, bg_color.b], +// crop_center: camera_path_object.crop.crop_center, +// crop_scale: camera_path_object.crop.crop_scale, +// }); +// } +// }; +// +// const uploadCameraPath = (e) => { +// const fileUpload = e.target.files[0]; +// +// const fr = new FileReader(); +// fr.onload = (res) => { +// const camera_path_object = JSON.parse(res.target.result); +// load_camera_path(camera_path_object); +// }; +// +// fr.readAsText(fileUpload); +// }; +// +// const open_render_modal = () => { +// setRenderModalOpen(true); +// +// const camera_path_object = get_camera_path(); +// +// sendWebsocketMessage(viser_websocket, { +// type: 'CameraPathPayloadMessage', +// camera_path_filename: export_path, +// camera_path: camera_path_object, +// }); +// sendWebsocketMessage(viser_websocket, { type: 'SaveCheckpointMessage' }); +// }; +// +// const open_load_path_modal = () => { +// sendWebsocketMessage(viser_websocket, { type: 'CameraPathOptionsRequest' }); +// setLoadPathModalOpen(true); +// }; +// +// const isAnimated = (property) => animate.has(property); +// +// const toggleAnimate = (property) => { +// const new_animate = new Set(animate); +// if (animate.has(property)) { +// new_animate.delete(property); +// setAnimate(new_animate); +// } else { +// new_animate.add(property); +// setAnimate(new_animate); +// } +// }; +// +// const setAllCameraFOV = (val) => { +// if (fovLabel === FOV_LABELS.FOV) { +// for (let i = 0; i < cameras.length; i += 1) { +// cameras[i].fov = val; +// } +// } else { +// for (let i = 0; i < cameras.length; i += 1) { +// cameras[i].setFocalLength(val / cameras[i].aspect); +// } +// } +// }; +// +// const setAllCameraRenderTime = (val) => { +// for (let i = 0; i < cameras.length; i += 1) { +// cameras[i].renderTime = val; +// } +// }; +// +// return ( +//
+//
+//
+// +// +//
+//
+// +//
+//
+// +// +//
+//
+// +// +// +// +//
+// {display_render_time && ( +//
+// +// +// +// +//
+// )} +// {camera_type !== 'equirectangular' && ( +//
+// +// +// +// +//
+// )} +//
+//
+// +//
+//
+// +// {!is_cycle ? ( +// +// ) : ( +// +// )} +// +//
+//
+// +// +// +//
+//
+//
+// +//

Smoothness

+// +// { +// set_smoothness_value(value); +// }} +// /> +// +// +//
+//
+// +// Camera Keyframes +// +// { +// if (cameras.length === 0) { +// return ''; +// } +// if (i === cameras.length && is_cycle) { +// return `${cameras[0].properties.get('NAME')} @ ${parseFloat( +// (value * seconds).toFixed(2), +// )}s`; +// } +// return `${cameras[i].properties.get('NAME')} @ ${parseFloat( +// (value * seconds).toFixed(2), +// )}s`; +// }} +// marks={marks} +// min={slider_min} +// max={slider_max} +// disabled={cameras.length < 2} +// track={false} +// onChange={handleKeyframeSlider} +// sx={{ +// '& .MuiSlider-thumb': { +// borderRadius: '6px', +// width: `${24.0 / Math.max(Math.sqrt(cameras.length), 2)}px`, +// }, +// }} +// disableSwap +// /> +// +// Playback +// +// { +// set_slider_value(value); +// }} +// /> +//
+//
+// +// +// {/* eslint-disable-next-line no-nested-ternary */} +// {!is_playing && slider_max === slider_value ? ( +// +// ) : !is_playing ? ( +// +// ) : ( +// +// )} +// +// +//
+//
+// +//
+//
+// ); +//} \ No newline at end of file diff --git a/src/viser/client/src/ControlPanel/ControlPanel.tsx b/src/viser/client/src/ControlPanel/ControlPanel.tsx index e2223be51..04b527733 100644 --- a/src/viser/client/src/ControlPanel/ControlPanel.tsx +++ b/src/viser/client/src/ControlPanel/ControlPanel.tsx @@ -21,6 +21,7 @@ import BottomPanel from "./BottomPanel"; import FloatingPanel from "./FloatingPanel"; import { ThemeConfigurationMessage } from "../WebsocketMessages"; import SidebarPanel from "./SidebarPanel"; +import CameraTrajectoryPanel from "./CameraTrajectoryPanel"; // Must match constant in Python. const ROOT_CONTAINER_ID = "root"; @@ -77,6 +78,7 @@ export default function ControlPanel(props: { const panelContents = ( <> + diff --git a/src/viser/client/src/ControlPanel/GuiState.tsx b/src/viser/client/src/ControlPanel/GuiState.tsx index 2efe5c73b..402e84a6d 100644 --- a/src/viser/client/src/ControlPanel/GuiState.tsx +++ b/src/viser/client/src/ControlPanel/GuiState.tsx @@ -43,6 +43,7 @@ interface GuiState { guiAttributeFromId: { [id: string]: { visible?: boolean; disabled?: boolean } | undefined; }; + isRenderMode: boolean; } interface GuiActions { @@ -77,6 +78,7 @@ const cleanGuiState: GuiState = { guiConfigFromId: {}, guiValueFromId: {}, guiAttributeFromId: {}, + isRenderMode: false, }; export function computeRelativeLuminance(color: string) { diff --git a/src/viser/client/src/ControlPanel/MultiSlider.styles.tsx b/src/viser/client/src/ControlPanel/MultiSlider.styles.tsx new file mode 100644 index 000000000..779925ee6 --- /dev/null +++ b/src/viser/client/src/ControlPanel/MultiSlider.styles.tsx @@ -0,0 +1,243 @@ +import { createStyles, rem } from '@mantine/styles'; +import { MantineColor, getSize, MantineNumberSize } from '@mantine/styles'; + + +export const sizes = { + xs: rem(4), + sm: rem(6), + md: rem(8), + lg: rem(10), + xl: rem(12), +}; + +export const useSliderRootStyles = createStyles((theme) => ({ + root: { + ...theme.fn.fontStyles(), + WebkitTapHighlightColor: 'transparent', + outline: 0, + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + touchAction: 'none', + position: 'relative', + }, +})); + + +interface ThumbStyles { + color: MantineColor; + disabled: boolean; + thumbSize: number | string; +} + +export const useThumbStyles = createStyles((theme, { color, disabled, thumbSize }: ThumbStyles, { size }) => ({ + label: { + position: 'absolute', + top: rem(-36), + backgroundColor: theme.colorScheme === 'dark' ? theme.colors.dark[4] : theme.colors.gray[9], + fontSize: theme.fontSizes.xs, + color: theme.white, + padding: `calc(${theme.spacing.xs} / 2)`, + borderRadius: theme.radius.sm, + whiteSpace: 'nowrap', + pointerEvents: 'none', + userSelect: 'none', + touchAction: 'none', + }, + + thumb: { + ...theme.fn.focusStyles(), + boxSizing: 'border-box', + position: 'absolute', + display: disabled ? 'none' : 'flex', + height: thumbSize ? rem(thumbSize) : `calc(${getSize({ sizes, size })} * 2)`, + width: thumbSize ? rem(thumbSize) : `calc(${getSize({ sizes, size })} * 2)`, + backgroundColor: + theme.colorScheme === 'dark' + ? theme.fn.themeColor(color, theme.fn.primaryShade()) + : theme.white, + border: `${rem(4)} solid ${ + theme.colorScheme === 'dark' + ? theme.white + : theme.fn.themeColor(color, theme.fn.primaryShade()) + }`, + color: + theme.colorScheme === 'dark' + ? theme.white + : theme.fn.themeColor(color, theme.fn.primaryShade()), + transform: 'translate(-50%, -50%)', + top: '50%', + cursor: 'pointer', + borderRadius: 1000, + alignItems: 'center', + justifyContent: 'center', + transitionDuration: '100ms', + transitionProperty: 'box-shadow, transform', + transitionTimingFunction: theme.transitionTimingFunction, + zIndex: 3, + userSelect: 'none', + touchAction: 'none', + }, + + dragging: { + transform: 'translate(-50%, -50%) scale(1.05)', + boxShadow: theme.shadows.sm, + }, +})); + + +interface TrackStyles { + radius: MantineNumberSize; + color: MantineColor; + disabled: boolean; + inverted: boolean; + thumbSize?: number; + } + + export const useTrackStyles = createStyles( + (theme, { radius, color, disabled, inverted, thumbSize }: TrackStyles, { size }) => ({ + trackContainer: { + display: 'flex', + alignItems: 'center', + width: '100%', + height: `calc(${getSize({ sizes, size })} * 2)`, + cursor: 'pointer', + + '&:has(~ input:disabled)': { + '&': { + pointerEvents: 'none', + }, + + '& .mantine-Slider-thumb': { + display: 'none', + }, + + '& .mantine-Slider-track::before': { + content: '""', + backgroundColor: inverted + ? theme.colorScheme === 'dark' + ? theme.colors.dark[3] + : theme.colors.gray[4] + : theme.colorScheme === 'dark' + ? theme.colors.dark[4] + : theme.colors.gray[2], + }, + + '& .mantine-Slider-bar': { + backgroundColor: inverted + ? theme.colorScheme === 'dark' + ? theme.colors.dark[4] + : theme.colors.gray[2] + : theme.colorScheme === 'dark' + ? theme.colors.dark[3] + : theme.colors.gray[4], + }, + }, + }, + + track: { + position: 'relative', + height: getSize({ sizes, size }), + width: '100%', + marginRight: thumbSize ? rem(thumbSize / 2) : getSize({ size, sizes }), + marginLeft: thumbSize ? rem(thumbSize / 2) : getSize({ size, sizes }), + + '&::before': { + content: '""', + position: 'absolute', + top: 0, + bottom: 0, + borderRadius: theme.fn.radius(radius), + right: `calc(${thumbSize ? rem(thumbSize / 2) : getSize({ size, sizes })} * -1)`, + left: `calc(${thumbSize ? rem(thumbSize / 2) : getSize({ size, sizes })} * -1)`, + backgroundColor: inverted + ? disabled + ? theme.colorScheme === 'dark' + ? theme.colors.dark[3] + : theme.colors.gray[4] + : theme.fn.variant({ variant: 'filled', color }).background + : theme.colorScheme === 'dark' + ? theme.colors.dark[4] + : theme.colors.gray[2], + zIndex: 0, + }, + }, + + bar: { + position: 'absolute', + zIndex: 1, + top: 0, + bottom: 0, + backgroundColor: inverted + ? theme.colorScheme === 'dark' + ? theme.colors.dark[4] + : theme.colors.gray[2] + : disabled + ? theme.colorScheme === 'dark' + ? theme.colors.dark[3] + : theme.colors.gray[4] + : theme.fn.variant({ variant: 'filled', color }).background, + borderRadius: theme.fn.radius(radius), + }, + }) + ); + + interface MarksStyles { + color: MantineColor; + disabled: boolean; + thumbSize?: number; +} + +export const useMarksStyles = createStyles((theme, { color, disabled, thumbSize }: MarksStyles, { size }) => ({ + marksContainer: { + position: 'absolute', + right: thumbSize ? rem(thumbSize / 2) : getSize({ sizes, size }), + left: thumbSize ? rem(thumbSize / 2) : getSize({ sizes, size }), + + '&:has(~ input:disabled)': { + '& .mantine-Slider-markFilled': { + border: `${rem(2)} solid ${ + theme.colorScheme === 'dark' ? theme.colors.dark[4] : theme.colors.gray[2] + }`, + borderColor: theme.colorScheme === 'dark' ? theme.colors.dark[3] : theme.colors.gray[4], + }, + }, + }, + + markWrapper: { + position: 'absolute', + top: `calc(${rem(getSize({ sizes, size }))} / 2)`, + zIndex: 2, + height: 0, + }, + + mark: { + boxSizing: 'border-box', + border: `${rem(2)} solid ${ + theme.colorScheme === 'dark' ? theme.colors.dark[4] : theme.colors.gray[2] + }`, + height: getSize({ sizes, size }), + width: getSize({ sizes, size }), + borderRadius: 1000, + transform: `translateX(calc(-${getSize({ sizes, size })} / 2))`, + backgroundColor: theme.white, + pointerEvents: 'none', + }, + + markFilled: { + borderColor: disabled + ? theme.colorScheme === 'dark' + ? theme.colors.dark[3] + : theme.colors.gray[4] + : theme.fn.variant({ variant: 'filled', color }).background, + }, + + markLabel: { + transform: `translate(-50%, calc(${theme.spacing.xs} / 2))`, + fontSize: theme.fontSizes.sm, + color: theme.colorScheme === 'dark' ? theme.colors.dark[2] : theme.colors.gray[6], + whiteSpace: 'nowrap', + cursor: 'pointer', + userSelect: 'none', + }, +})); \ No newline at end of file diff --git a/src/viser/client/src/ControlPanel/MultiSlider.tsx b/src/viser/client/src/ControlPanel/MultiSlider.tsx new file mode 100644 index 000000000..3db5faa10 --- /dev/null +++ b/src/viser/client/src/ControlPanel/MultiSlider.tsx @@ -0,0 +1,774 @@ +import React, { useRef, useState, forwardRef, useEffect } from 'react'; +import { useMove, useUncontrolled } from '@mantine/hooks'; +import { + DefaultProps, + MantineNumberSize, + MantineColor, + useMantineTheme, + useComponentDefaultProps, + Selectors, + getSize, + rem, +} from '@mantine/styles'; +import { + MantineTransition, + Box, + Transition, +} from '@mantine/core'; +import { sizes, useSliderRootStyles, useThumbStyles, useTrackStyles, useMarksStyles } from './MultiSlider.styles'; + +export function getClientPosition(event: any) { + if ('TouchEvent' in window && event instanceof window.TouchEvent) { + const touch = event.touches[0]; + return touch.clientX; + } + + return event.clientX; +} + +interface GetPosition { + value: number; + min: number; + max: number; +} + +function getPosition({ value, min, max }: GetPosition) { + const position = ((value - min) / (max - min)) * 100; + return Math.min(Math.max(position, 0), 100); +} + +interface GetChangeValue { + value: number; + containerWidth?: number; + min: number; + max: number; + step: number; + precision?: number; +} + +function getChangeValue({ + value, + containerWidth, + min, + max, + step, + precision, +}: GetChangeValue) { + const left = !containerWidth + ? value + : Math.min(Math.max(value, 0), containerWidth) / containerWidth; + const dx = left * (max - min); + const nextValue = (dx !== 0 ? Math.round(dx / step) * step : 0) + min; + + const nextValueWithinStep = Math.max(nextValue, min); + + if (precision !== undefined) { + return Number(nextValueWithinStep.toFixed(precision)); + } + + return nextValueWithinStep; +} + +export type SliderRootStylesNames = Selectors; + +export interface SliderRootProps + extends DefaultProps, + React.ComponentPropsWithoutRef<'div'> { + size: MantineNumberSize; + children: React.ReactNode; + disabled: boolean; + variant: string; +} + +export const SliderRoot = forwardRef( + ( + { + className, + size, + classNames, + styles, + disabled, + unstyled, + variant, + ...others + }: SliderRootProps, + ref + ) => { + const { classes, cx } = useSliderRootStyles((null as unknown) as void,{ + name: 'Slider', + classNames, + styles, + unstyled, + variant, + size, + }); + return ; + } +); + +SliderRoot.displayName = '@mantine/core/SliderRoot'; + + +export type ThumbStylesNames = Selectors; + +export interface ThumbProps extends DefaultProps { + max: number; + min: number; + value: number; + position: number; + dragging: boolean; + color: MantineColor; + size: MantineNumberSize; + label: React.ReactNode; + onKeyDownCapture?(event: React.KeyboardEvent): void; + onMouseDown?(event: React.MouseEvent | React.TouchEvent): void; + labelTransition?: MantineTransition; + labelTransitionDuration?: number; + labelTransitionTimingFunction?: string; + labelAlwaysOn: boolean; + thumbLabel: string; + onFocus?(): void; + onBlur?(): void; + showLabelOnHover?: boolean; + isHovered?: boolean; + children?: React.ReactNode; + disabled: boolean; + thumbSize: number; + variant: string; +} + +export const Thumb = forwardRef( + ( + { + max, + min, + value, + position, + label, + dragging, + onMouseDown, + onKeyDownCapture, + color, + classNames, + styles, + size, + labelTransition, + labelTransitionDuration, + labelTransitionTimingFunction, + labelAlwaysOn, + thumbLabel, + onFocus, + onBlur, + showLabelOnHover, + isHovered, + children = null, + disabled, + unstyled, + thumbSize, + variant, + }: ThumbProps, + ref + ) => { + const { classes, cx, theme } = useThumbStyles( + { color, disabled, thumbSize }, + { name: 'Slider', classNames, styles, unstyled, variant, size } + ); + const [focused, setFocused] = useState(false); + + const isVisible = labelAlwaysOn || dragging || focused || (showLabelOnHover && isHovered); + + return ( + + tabIndex={0} + role="slider" + aria-label={thumbLabel} + aria-valuemax={max} + aria-valuemin={min} + aria-valuenow={value} + ref={ref} + className={cx(classes.thumb, { [classes.dragging]: dragging })} + onFocus={() => { + setFocused(true); + typeof onFocus === 'function' && onFocus(); + }} + onBlur={() => { + setFocused(false); + typeof onBlur === 'function' && onBlur(); + }} + onTouchStart={onMouseDown} + onMouseDown={onMouseDown} + onKeyDownCapture={onKeyDownCapture} + onClick={(event) => event.stopPropagation()} + style={{ [theme.dir === 'rtl' ? 'right' : 'left']: `${position}%` }} + > + {children} + + {(transitionStyles) => ( +
+ {label} +
+ )} + + + ); + } +); + +Thumb.displayName = '@mantine/core/SliderThumb'; + +export type MarksStylesNames = Selectors; + +export interface MarksProps extends DefaultProps { + marks: { value: number; label?: React.ReactNode }[]; + size: MantineNumberSize; + thumbSize?: number; + color: MantineColor; + min: number; + max: number; + onChange(value: number): void; + disabled: boolean; + variant: string; +} + +export function Marks({ + marks, + color, + size, + thumbSize, + min, + max, + classNames, + styles, + onChange, + disabled, + unstyled, + variant, +}: MarksProps) { + const { classes, cx } = useMarksStyles( + { color, disabled, thumbSize }, + { name: 'Slider', classNames, styles, unstyled, variant, size } + ); + + const items = marks.map((mark, index) => ( + +
+ {mark.label && ( +
{ + event.stopPropagation(); + !disabled && onChange(mark.value); + }} + onTouchStart={(event) => { + event.stopPropagation(); + !disabled && onChange(mark.value); + }} + > + {mark.label} +
+ )} + + )); + + return
{items}
; +} + +Marks.displayName = '@mantine/core/SliderMarks'; + + + +export type TrackStylesNames = Selectors | MarksStylesNames; + +export interface TrackProps extends DefaultProps { + marks: { value: number; label?: React.ReactNode }[]; + size: MantineNumberSize; + thumbSize?: number; + radius: MantineNumberSize; + color: MantineColor; + min: number; + max: number; + children: React.ReactNode; + onChange(value: number): void; + disabled: boolean; + variant: string; + containerProps?: React.PropsWithRef>; +} + +export function Track({ + size, + thumbSize, + color, + classNames, + styles, + radius, + children, + disabled, + unstyled, + variant, + containerProps, + ...others +}: TrackProps) { + const { classes } = useTrackStyles( + { color, radius, disabled, inverted: false }, + { name: 'Slider', classNames, styles, unstyled, variant, size } + ); + + return ( + <> +
+
+ {children} +
+
+ + + + ); +} + +Track.displayName = '@mantine/core/SliderTrack'; + + + +export type MultiSliderStylesNames = + | SliderRootStylesNames + | ThumbStylesNames + | TrackStylesNames + | MarksStylesNames; + +type Value = number[]; + +export interface MultiSliderProps + extends DefaultProps, + Omit, 'value' | 'onChange' | 'defaultValue'> { + variant: string; + + /** Color from theme.colors */ + color: MantineColor; + + /** Key of theme.radius or any valid CSS value to set border-radius, theme.defaultRadius by default */ + radius: MantineNumberSize; + + /** Predefined track and thumb size, number to set sizes */ + size: MantineNumberSize; + + /** Minimal possible value */ + min: number; + + /** Maximum possible value */ + max: number; + + /** Minimal range interval */ + minRange: number; + + /** Number by which value will be incremented/decremented with thumb drag and arrows */ + step: number; + + /** Amount of digits after the decimal point */ + precision?: number; + + /** Current value for controlled slider */ + value?: Value; + + /** Default value for uncontrolled slider */ + defaultValue?: Value; + + /** Called each time value changes */ + onChange?(value: Value): void; + + /** Called when user stops dragging slider or changes value with arrows */ + onChangeEnd?(value: Value): void; + + /** Hidden input name, use with uncontrolled variant */ + name?: string; + + /** Marks which will be placed on the track */ + marks: { value: number; label?: React.ReactNode }[]; + + /** Function to generate label or any react node to render instead, set to null to disable label */ + label?: React.ReactNode | ((value: number) => React.ReactNode); + + /** Label appear/disappear transition */ + labelTransition?: MantineTransition; + + /** Label appear/disappear transition duration in ms */ + labelTransitionDuration?: number; + + /** Label appear/disappear transition timing function, defaults to theme.transitionRimingFunction */ + labelTransitionTimingFunction?: string; + + /** If true label will be not be hidden when user stops dragging */ + labelAlwaysOn: boolean; + + /** First thumb aria-label */ + thumbFromLabel: string; + + /** Second thumb aria-label */ + thumbToLabel: string; + + /**If true slider label will appear on hover */ + showLabelOnHover?: boolean; + + /** Thumbs children, can be used to add icons */ + thumbChildren: React.ReactNode | React.ReactNode[] | null; + + /** Disables slider */ + disabled: boolean; + + /** Thumb width and height */ + thumbSize: number; + + /** A transformation function, to change the scale of the slider */ + scale: (value: number) => number; + + fixedEndpoints: boolean; +} + +const defaultProps: Partial = { + size: 'md', + radius: 'xl', + min: 0, + max: 100, + minRange: 2, + step: 1, + marks: [], + label: (f) => f, + labelTransition: 'skew-down', + labelTransitionDuration: 0, + labelAlwaysOn: false, + thumbFromLabel: '', + thumbChildren: null, + thumbToLabel: '', + showLabelOnHover: true, + disabled: false, + scale: (v) => v, + fixedEndpoints: false, +}; + +export const MultiSlider = forwardRef((props, ref) => { + const { + classNames, + styles, + color, + value, + onChange, + onChangeEnd, + size, + radius, + min, + max, + minRange, + step, + precision, + defaultValue, + name, + marks, + label, + labelTransition, + labelTransitionDuration, + labelTransitionTimingFunction, + labelAlwaysOn, + thumbFromLabel, + thumbToLabel, + showLabelOnHover, + thumbChildren, + disabled, + unstyled, + thumbSize, + scale, + variant, + fixedEndpoints, + ...others + } = useComponentDefaultProps('MultiSlider', defaultProps, props); + + const theme = useMantineTheme(); + const [focused, setFocused] = useState(-1); + const [hovered, setHovered] = useState(false); + const [_value, setValue] = useUncontrolled({ + value, + defaultValue, + finalValue: [min, max], + onChange, + }); + const valueRef = useRef(_value); + const thumbs = useRef<(HTMLDivElement | null)[]>([]); + const thumbIndex = useRef(-1); + const positions = _value.map(x => getPosition({ value: x, min, max})); + + const _setValue = (val: Value) => { + setValue(val); + valueRef.current = val; + }; + + useEffect( + () => { + if (Array.isArray(value)) { + valueRef.current = value; + } + }, + Array.isArray(value) ? [value[0], value[1]] : [null, null] + ); + + const setRangedValue = (val: number, index: number, triggerChangeEnd: boolean) => { + const clone: Value = [...valueRef.current]; + clone[index] = val; + + if (index < clone.length - 1) { + if (val > clone[index + 1] - (minRange - 0.000000001)) { + clone[index] = Math.max(min, clone[index + 1] - minRange); + } + + if (val > (max - (minRange - 0.000000001) || min)) { + clone[index] = valueRef.current[index]; + } + } + + if (index > 0) { + if (val < clone[index - 1] + minRange) { + clone[index] = Math.min(max, clone[index - 1] + minRange); + } + } + + if (fixedEndpoints && (index === 0 || index == clone.length - 1)) { + clone[index] = valueRef.current[index]; + } + + _setValue(clone); + + if (triggerChangeEnd) { + onChangeEnd?.(valueRef.current); + } + }; + + const handleChange = (val: number) => { + if (!disabled) { + const nextValue = getChangeValue({ value: val, min, max, step, precision }); + setRangedValue(nextValue, thumbIndex.current, false); + } + }; + + const { ref: container, active } = useMove( + ({ x }) => handleChange(x), + { onScrubEnd: () => onChangeEnd?.(valueRef.current) }, + theme.dir + ); + + function handleThumbMouseDown(index: number) { + thumbIndex.current = index; + } + + const handleTrackMouseDownCapture = ( + event: React.MouseEvent | React.TouchEvent + ) => { + container.current.focus(); + const rect = container.current.getBoundingClientRect(); + const changePosition = getClientPosition(event.nativeEvent); + const changeValue = getChangeValue({ + value: changePosition - rect.left, + max, + min, + step, + containerWidth: rect.width, + }); + + const _nearestHandle = _value.map((v) => Math.abs(v - changeValue)).indexOf(Math.min(..._value.map((v) => Math.abs(v - changeValue)))); + + thumbIndex.current = _nearestHandle; + }; + + const getFocusedThumbIndex = () => { + if (focused !== 1 && focused !== 0) { + setFocused(0); + return 0; + } + + return focused; + }; + + const handleTrackKeydownCapture = (event: React.KeyboardEvent) => { + if (!disabled) { + switch (event.key) { + case 'ArrowUp': { + event.preventDefault(); + const focusedIndex = getFocusedThumbIndex(); + thumbs.current[focusedIndex]?.focus(); + setRangedValue( + Math.min(Math.max(valueRef.current[focusedIndex] + step, min), max), + focusedIndex, + true + ); + break; + } + case 'ArrowRight': { + event.preventDefault(); + const focusedIndex = getFocusedThumbIndex(); + thumbs.current[focusedIndex]?.focus(); + setRangedValue( + Math.min( + Math.max( + theme.dir === 'rtl' + ? valueRef.current[focusedIndex] - step + : valueRef.current[focusedIndex] + step, + min + ), + max + ), + focusedIndex, + true + ); + break; + } + + case 'ArrowDown': { + event.preventDefault(); + const focusedIndex = getFocusedThumbIndex(); + thumbs.current[focusedIndex]?.focus(); + setRangedValue( + Math.min(Math.max(valueRef.current[focusedIndex] - step, min), max), + focusedIndex, + true + ); + break; + } + case 'ArrowLeft': { + event.preventDefault(); + const focusedIndex = getFocusedThumbIndex(); + thumbs.current[focusedIndex]?.focus(); + setRangedValue( + Math.min( + Math.max( + theme.dir === 'rtl' + ? valueRef.current[focusedIndex] + step + : valueRef.current[focusedIndex] - step, + min + ), + max + ), + focusedIndex, + true + ); + break; + } + + default: { + break; + } + } + } + }; + + const sharedThumbProps = { + max, + min, + color, + size, + labelTransition, + labelTransitionDuration, + labelTransitionTimingFunction, + labelAlwaysOn, + onBlur: () => setFocused(-1), + classNames, + styles, + }; + + const hasArrayThumbChildren = Array.isArray(thumbChildren); + + return ( + + { + const nearestValue = Math.abs(_value[0] - val) > Math.abs(_value[1] - val) ? 1 : 0; + const clone: Value = [..._value]; + clone[nearestValue] = val; + _setValue(clone); + }} + disabled={disabled} + unstyled={unstyled} + variant={variant} + containerProps={{ + ref: container, + onMouseEnter: showLabelOnHover ? () => setHovered(true) : undefined, + onMouseLeave: showLabelOnHover ? () => setHovered(false) : undefined, + onTouchStartCapture: handleTrackMouseDownCapture, + onTouchEndCapture: () => { + thumbIndex.current = -1; + }, + onMouseDownCapture: handleTrackMouseDownCapture, + onMouseUpCapture: () => { + thumbIndex.current = -1; + }, + onKeyDownCapture: handleTrackKeydownCapture, + }} + >{_value.map((value, index) => ( + { + thumbs.current[index] = node; + }} + thumbLabel={thumbFromLabel} + onMouseDown={() => handleThumbMouseDown(index)} + onFocus={() => setFocused(index)} + showLabelOnHover={showLabelOnHover} + isHovered={hovered} + disabled={disabled} + unstyled={unstyled} + thumbSize={thumbSize} + variant={variant} + > + {hasArrayThumbChildren ? thumbChildren[index] : thumbChildren} + ))} + + {_value.map((value, index) => ( + + ))} + + ); +}); + +MultiSlider.displayName = 'MultiSlider'; \ No newline at end of file diff --git a/src/viser/client/src/ControlPanel/curve.tsx b/src/viser/client/src/ControlPanel/curve.tsx new file mode 100644 index 000000000..4481d5cab --- /dev/null +++ b/src/viser/client/src/ControlPanel/curve.tsx @@ -0,0 +1,115 @@ +// Code for creating a curve from a set of points + +import * as THREE from 'three'; + +function get_catmull_rom_curve(list_of_3d_vectors: THREE.Vector3[], is_cycle: boolean, smoothness_value: number) { + // TODO: add some hyperparameters to this function + const curve = new THREE.CatmullRomCurve3( + list_of_3d_vectors, + is_cycle, + // 'centripetal' + 'catmullrom', + smoothness_value, + ); + return curve; +} + +export interface Camera { + position: THREE.Vector3 + fov: number + quaternion: THREE.Quaternion + renderTime?: number +} + +export function get_curve_object_from_cameras( + cameras: Camera[], + is_cycle: boolean, + smoothness_value: number, +) { + if (cameras.length === 0) { + return null; + } + // interpolate positions, lookat directions, and ups + // similar to + // https://github.com/google-research/multinerf/blob/1c8b1c552133cdb2de1c1f3c871b2813f6662265/internal/camera_utils.py#L281 + + const positions = []; + const lookats = []; + const ups = []; + const fovs = []; + const render_times = []; + + for (let i = 0; i < cameras.length; i += 1) { + const camera = cameras[i]; + + const up = new THREE.Vector3(0, 1, 0); // y is up in local space + const lookat = new THREE.Vector3(0, 0, 1); // z is forward in local space + + up.applyQuaternion(camera.quaternion); + lookat.applyQuaternion(camera.quaternion); + + positions.push(camera.position); + ups.push(up); + lookats.push(lookat); + // Reuse catmullromcurve3 for 1d values. TODO fix this + fovs.push(new THREE.Vector3(0, 0, camera.fov)); + render_times.push(new THREE.Vector3(0, 0, camera.renderTime)); + } + + let curve_positions = null; + let curve_lookats = null; + let curve_ups = null; + let curve_fovs = null; + let curve_render_times = null; + + curve_positions = get_catmull_rom_curve(positions, is_cycle, smoothness_value); + curve_lookats = get_catmull_rom_curve(lookats, is_cycle, smoothness_value); + curve_ups = get_catmull_rom_curve(ups, is_cycle, smoothness_value); + curve_fovs = get_catmull_rom_curve(fovs, is_cycle, smoothness_value / 10); + curve_render_times = get_catmull_rom_curve(render_times, is_cycle, smoothness_value); + + const curve_object = { + curve_positions, + curve_lookats, + curve_ups, + curve_fovs, + curve_render_times, + }; + return curve_object; +} + +export function get_transform_matrix(position: THREE.Vector3, lookat: THREE.Vector3, up: THREE.Vector3) { + // normalize the vectors + lookat.normalize(); + // make up orthogonal to lookat + const up_proj = lookat.clone().multiplyScalar(up.dot(lookat)); + up.sub(up_proj); + up.normalize(); + + // create a copy of the vector up + const up_copy = up.clone(); + const cross = up_copy.cross(lookat); + cross.normalize(); + + // create the camera transform matrix + const mat = new THREE.Matrix4(); + mat.set( + cross.x, + up.x, + lookat.x, + position.x, + cross.y, + up.y, + lookat.y, + position.y, + cross.z, + up.z, + lookat.z, + position.z, + 0, + 0, + 0, + 1, + ); + return mat; +} \ No newline at end of file diff --git a/src/viser/client/src/ThreeAssets.tsx b/src/viser/client/src/ThreeAssets.tsx index 69f31c339..4ff02a5cf 100644 --- a/src/viser/client/src/ThreeAssets.tsx +++ b/src/viser/client/src/ThreeAssets.tsx @@ -192,17 +192,19 @@ export const CoordinateFrame = React.forwardRef< ); }); +export interface CameraFrustumProps { + fov: number; + aspect: number; + scale: number; + color: number; + image?: THREE.Texture; +} + const lineGeom = new THREE.CylinderGeometry(1.0, 1.0, 1.0, 3, 1); /** Helper for visualizing camera frustums. */ export const CameraFrustum = React.forwardRef< THREE.Group, - { - fov: number; - aspect: number; - scale: number; - color: number; - image?: THREE.Texture; - } + CameraFrustumProps >(function CameraFrustum(props, ref) { let y = Math.tan(props.fov / 2.0); let x = y * props.aspect; From 81d47fae3e287d1ad2ec1707f2a5b59e568d29b2 Mon Sep 17 00:00:00 2001 From: Jonas Kulhanek Date: Fri, 29 Dec 2023 13:47:00 +0100 Subject: [PATCH 2/4] Fix minor bugs --- .../ControlPanel/CameraTrajectoryPanel.tsx | 97 +++++++++++-------- 1 file changed, 55 insertions(+), 42 deletions(-) diff --git a/src/viser/client/src/ControlPanel/CameraTrajectoryPanel.tsx b/src/viser/client/src/ControlPanel/CameraTrajectoryPanel.tsx index f79738603..04f68f095 100644 --- a/src/viser/client/src/ControlPanel/CameraTrajectoryPanel.tsx +++ b/src/viser/client/src/ControlPanel/CameraTrajectoryPanel.tsx @@ -139,15 +139,29 @@ export default function CameraTrajectoryPanel(props: CameraTrajectoryPanelProps) const addSceneNode = viewer.useSceneTree((state) => state.addSceneNode); const nodeFromName = viewer.useSceneTree((state) => state.nodeFromName); const isRenderMode = viewer.useGui((state) => state.isRenderMode); + + // Get default FOV from camera + const [isCycle, setIsCycle] = React.useState(false); + const [isPlaying, setIsPlaying] = React.useState(false); + const [fps, setFps] = React.useState(24); + const [smoothness, setSmoothness] = React.useState(0.5); + const [cameras, setCameras] = React.useState([]); + const [fov, setFov] = React.useState(viewer.cameraRef.current?.fov ?? 75.0); + const [renderWidth, setRenderWidth] = React.useState(1920); + const [renderHeight, setRenderHeight] = React.useState(1080); + const [playerTime, setPlayerTime] = React.useState(0.); + const aspect = renderWidth / renderHeight; const baseTreeName = "CameraTrajectory" - React.useEffect(() => { + const ensureThreeRootExists = () => { if (!(baseTreeName in nodeFromName)) { addSceneNode( new SceneNode(baseTreeName, (ref) => ( )) as SceneNode, ); + } + if (!(`${baseTreeName}/PlayerCamera` in nodeFromName)) { addSceneNode( new SceneNode( `${baseTreeName}/PlayerCamera`, @@ -161,27 +175,19 @@ export default function CameraTrajectoryPanel(props: CameraTrajectoryPanelProps) /> ), ) as SceneNode); - const attr = viewer.nodeAttributesFromName.current; - if (attr[`${baseTreeName}/PlayerCamera`] === undefined) attr[`${baseTreeName}/PlayerCamera`] = {}; - attr[`${baseTreeName}/PlayerCamera`]!.visibility = false; - return () => { - removeSceneNode(`${baseTreeName}/PlayerCamera`); - removeSceneNode(baseTreeName); - } + } + const attr = viewer.nodeAttributesFromName.current; + if (attr[`${baseTreeName}/PlayerCamera`] === undefined) attr[`${baseTreeName}/PlayerCamera`] = {}; + attr[`${baseTreeName}/PlayerCamera`]!.visibility = false; + } + React.useEffect(() => { + ensureThreeRootExists(); + return () => { + `${baseTreeName}/PlayerCamera` in nodeFromName && removeSceneNode(`${baseTreeName}/PlayerCamera`); + baseTreeName in nodeFromName && removeSceneNode(baseTreeName); } }, []); - const [isCycle, setIsCycle] = React.useState(false); - const [isPlaying, setIsPlaying] = React.useState(false); - const [fps, setFps] = React.useState(30); - const [smoothness, setSmoothness] = React.useState(0.5); - const [cameras, setCameras] = React.useState([]); - const [fov, setFov] = React.useState(1.); - const [renderWidth, setRenderWidth] = React.useState(1920); - const [renderHeight, setRenderHeight] = React.useState(1080); - const [playerTime, setPlayerTime] = React.useState(0.); - const aspect = renderWidth / renderHeight; - const curveObject = React.useMemo(() => cameras.length > 1 ? get_curve_object_from_cameras( cameras.map(({fov, wxyz, position, time}: Camera) => ({ time, @@ -192,6 +198,7 @@ export default function CameraTrajectoryPanel(props: CameraTrajectoryPanelProps) // Update cameras and trajectory React.useEffect(() => { + ensureThreeRootExists(); // Update trajectory if (!(baseTreeName in nodeFromName)) return; const children = nodeFromName[baseTreeName]!.children; @@ -228,7 +235,7 @@ export default function CameraTrajectoryPanel(props: CameraTrajectoryPanelProps) // Render camera path React.useEffect(() => { - if (!(baseTreeName in nodeFromName)) return; + ensureThreeRootExists(); const nodeName = `${baseTreeName}/Trajectory`; if (curveObject !== null) { const num_points = fps * seconds; @@ -254,6 +261,7 @@ export default function CameraTrajectoryPanel(props: CameraTrajectoryPanelProps) }, [curveObject, fps, isRenderMode]); React.useEffect(() => { + ensureThreeRootExists(); // set the camera if (curveObject !== null) { if (isRenderMode) { @@ -331,28 +339,33 @@ export default function CameraTrajectoryPanel(props: CameraTrajectoryPanelProps) const addCamera = () => { - const { position, wxyz } = getPoseFromCamera(viewer); - const hash = getCameraHash({ fov, position, wxyz }); - const name = `${mapNumberToAlphabet(hash).slice(0, 6)}`; - - if (cameras.length >= 2) { - const mult = 1 - 1/cameras.length; - setCameras([...cameras.map(x => ({...x, time: x.time * mult})), { - time: 1, - name, - position, - wxyz, - fov, - }]); - } else { - setCameras([...cameras, { - time: cameras.length === 0 ? 0 : 1, - name, - position, - wxyz, - fov, - }]); - } + setCameras((cameras) => { + const { position, wxyz } = getPoseFromCamera(viewer); + const hash = getCameraHash({ fov, position, wxyz }); + let name = `${mapNumberToAlphabet(hash).slice(0, 6)}`; + const nameNumber = cameras.filter(x => x.name.startsWith(name)).length; + if (nameNumber > 0) { + name += `-${nameNumber+1}`; + } + if (cameras.length >= 2) { + const mult = 1 - 1/cameras.length; + return [...cameras.map(x => ({...x, time: x.time * mult})), { + time: 1, + name, + position, + wxyz, + fov, + }]; + } else { + return [...cameras, { + time: cameras.length === 0 ? 0 : 1, + name, + position, + wxyz, + fov, + }]; + } + }); } From 797a6f315935a33c074fca310960f880ee8dbaf3 Mon Sep 17 00:00:00 2001 From: Jonas Kulhanek Date: Fri, 29 Dec 2023 17:19:54 +0100 Subject: [PATCH 3/4] Add python messages and handles --- examples/22_custom_gui_component.py | 37 + src/viser/_gui_api.py | 36 + src/viser/_gui_handles.py | 38 + src/viser/_messages.py | 11 + src/viser/client/src/App.tsx | 5 +- .../ControlPanel/CameraTrajectoryPanel.tsx | 1206 +---------------- .../client/src/ControlPanel/ControlPanel.tsx | 2 - .../client/src/ControlPanel/Generated.tsx | 9 +- .../client/src/ControlPanel/GuiState.tsx | 5 +- .../client/src/ControlPanel/MultiSlider.tsx | 26 +- src/viser/client/src/SceneTree.tsx | 5 + src/viser/client/src/WebsocketMessages.tsx | 14 + 12 files changed, 186 insertions(+), 1208 deletions(-) create mode 100644 examples/22_custom_gui_component.py diff --git a/examples/22_custom_gui_component.py b/examples/22_custom_gui_component.py new file mode 100644 index 000000000..e8b76d6e3 --- /dev/null +++ b/examples/22_custom_gui_component.py @@ -0,0 +1,37 @@ +"""Advanced GUI - custom GUI components""" + +import time +from pathlib import Path + +import numpy as onp + +import trimesh +import viser +import viser.transforms as tf + +mesh = trimesh.load_mesh(Path(__file__).parent / "assets/dragon.obj") +assert isinstance(mesh, trimesh.Trimesh) +mesh.apply_scale(0.05) + +vertices = mesh.vertices +faces = mesh.faces +print(f"Loaded mesh with {vertices.shape} vertices, {faces.shape} faces") + +server = viser.ViserServer() +server.add_mesh_simple( + name="/simple", + vertices=vertices, + faces=faces, + wxyz=tf.SO3.from_x_radians(onp.pi / 2).wxyz, + position=(0.0, 0.0, 0.0), +) +server.add_mesh_trimesh( + name="/trimesh", + mesh=mesh.smoothed(), + wxyz=tf.SO3.from_x_radians(onp.pi / 2).wxyz, + position=(0.0, 5.0, 0.0), +) +panel = server.add_gui_camera_trajectory_panel() + +while True: + time.sleep(10.0) \ No newline at end of file diff --git a/src/viser/_gui_api.py b/src/viser/_gui_api.py index 16b188fa3..1112b1237 100644 --- a/src/viser/_gui_api.py +++ b/src/viser/_gui_api.py @@ -38,6 +38,7 @@ GuiModalHandle, GuiTabGroupHandle, SupportsRemoveProtocol, + GuiCameraTrajectoryPanelHandle, _GuiHandleState, _GuiInputHandle, _make_unique_id, @@ -740,6 +741,41 @@ def add_gui_dropdown( ) -> GuiDropdownHandle[TString]: ... + def add_gui_camera_trajectory_panel( + self, + order: Optional[float] = None, + visible: bool = True, + ) -> GuiCameraTrajectoryPanelHandle: + """ + Add a camera trajectory panel to the GUI. + + Returns: + A handle that can be used to interact with the GUI element. + """ + id = _make_unique_id() + order = _apply_default_order(order) + + # Send add GUI input message. + self._get_api()._queue(_messages.GuiAddCameraTrajectoryPanelMessage( + order=order, + id=id, + container_id=self._get_container_id(), + )) + + # Construct handle. + handle = GuiCameraTrajectoryPanelHandle( + _gui_api=self, + _container_id=self._get_container_id(), + _visible=True, + _id=id, + _order=order, + ) + + # Set the visible field. These will queue messages under-the-hood. + if not visible: + handle.visible = visible + return handle + def add_gui_dropdown( self, label: str, diff --git a/src/viser/_gui_handles.py b/src/viser/_gui_handles.py index 614a2f839..d0a0e0539 100644 --- a/src/viser/_gui_handles.py +++ b/src/viser/_gui_handles.py @@ -596,3 +596,41 @@ def remove(self) -> None: """Permanently remove this markdown from the visualizer.""" api = self._gui_api._get_api() api._queue(GuiRemoveMessage(self._id)) + + +@dataclasses.dataclass +class GuiCameraTrajectoryPanelHandle: + _gui_api: GuiApi + _id: str + _visible: bool + _container_id: str # Parent. + _order: float + + @property + def order(self) -> float: + """Read-only order value, which dictates the position of the GUI element.""" + return self._order + + @property + def visible(self) -> bool: + """Temporarily show or hide this GUI element from the visualizer. Synchronized + automatically when assigned.""" + return self._visible + + @visible.setter + def visible(self, visible: bool) -> None: + if visible == self.visible: + return + + self._gui_api._get_api()._queue(GuiSetVisibleMessage(self._id, visible=visible)) + self._visible = visible + + def __post_init__(self) -> None: + """We need to register ourself after construction for callbacks to work.""" + parent = self._gui_api._container_handle_from_id[self._container_id] + parent._children[self._id] = self + + def remove(self) -> None: + """Permanently remove this markdown from the visualizer.""" + api = self._gui_api._get_api() + api._queue(GuiRemoveMessage(self._id)) diff --git a/src/viser/_messages.py b/src/viser/_messages.py index 86a3a34c4..5e96c048e 100644 --- a/src/viser/_messages.py +++ b/src/viser/_messages.py @@ -486,6 +486,17 @@ class GuiAddButtonGroupMessage(_GuiAddInputBase): options: Tuple[str, ...] +@dataclasses.dataclass +class GuiAddCameraTrajectoryPanelMessage(Message): + order: float + id: str + container_id: str + + fov: float = 75. + render_width: int = 1920 + render_height: int = 1080 + + @dataclasses.dataclass class GuiRemoveMessage(Message): """Sent server->client to remove a GUI element.""" diff --git a/src/viser/client/src/App.tsx b/src/viser/client/src/App.tsx index 03ffccb1f..65bf8ac01 100644 --- a/src/viser/client/src/App.tsx +++ b/src/viser/client/src/App.tsx @@ -73,6 +73,7 @@ export type ViewerContextContents = { wxyz?: [number, number, number, number]; position?: [number, number, number]; visibility?: boolean; + renderModeVisibility?: boolean; }; }>; messageQueueRef: React.MutableRefObject; @@ -266,9 +267,7 @@ function ViewerCanvas({ children }: { children: React.ReactNode }) { - {(!isRenderMode) && ( - - )} + void): void { const inputElemenet = document.createElement('input'); inputElemenet.style.display = 'none'; @@ -105,7 +87,9 @@ function getPoseFromCamera(viewer: ViewerContextContents) { const three_camera = viewer.cameraRef.current!; const R_threecam_cam = new THREE.Quaternion().setFromEuler( new THREE.Euler(Math.PI, 0.0, 0.0), - ); + ).multiply(new THREE.Quaternion().setFromEuler( + new THREE.Euler(0.0, 0.0, Math.PI), + )); const R_world_threeworld = getR_threeworld_world(viewer).invert(); const R_world_camera = R_world_threeworld.clone() .multiply(three_camera.quaternion) @@ -178,13 +162,19 @@ export default function CameraTrajectoryPanel(props: CameraTrajectoryPanelProps) } const attr = viewer.nodeAttributesFromName.current; if (attr[`${baseTreeName}/PlayerCamera`] === undefined) attr[`${baseTreeName}/PlayerCamera`] = {}; + if (attr[baseTreeName] === undefined) attr[baseTreeName] = {}; attr[`${baseTreeName}/PlayerCamera`]!.visibility = false; + attr[`${baseTreeName}/PlayerCamera`]!.renderModeVisibility = false; + attr[baseTreeName]!.renderModeVisibility = false; } React.useEffect(() => { ensureThreeRootExists(); return () => { + const attr = viewer.nodeAttributesFromName.current; `${baseTreeName}/PlayerCamera` in nodeFromName && removeSceneNode(`${baseTreeName}/PlayerCamera`); baseTreeName in nodeFromName && removeSceneNode(baseTreeName); + `${baseTreeName}/PlayerCamera` in attr && delete attr[`${baseTreeName}/PlayerCamera`]; + baseTreeName in attr && delete attr[baseTreeName]; } }, []); @@ -229,6 +219,7 @@ export default function CameraTrajectoryPanel(props: CameraTrajectoryPanelProps) if (attr[nodeName] === undefined) attr[nodeName] = {}; attr[nodeName]!.wxyz = camera.wxyz; attr[nodeName]!.position = camera.position; + attr[nodeName]!.renderModeVisibility = false; }); }, [cameras, aspect, smoothness]); @@ -255,6 +246,9 @@ export default function CameraTrajectoryPanel(props: CameraTrajectoryPanelProps) material.dispose(); }) as SceneNode); + const attr = viewer.nodeAttributesFromName.current; + if (attr[nodeName] === undefined) attr[nodeName] = {}; + attr[nodeName]!.renderModeVisibility = false; } else if (nodeName in nodeFromName) { removeSceneNode(nodeName); } @@ -278,7 +272,7 @@ export default function CameraTrajectoryPanel(props: CameraTrajectoryPanelProps) cameraControls.updateCameraUp(); cameraControls.setLookAt(...position.toArray(), ...lookat.toArray(), false); const target = position.clone().add(lookat); - // NOTE: lookat is being ignore when calling setLookAt + // NOTE: lookat is being ignored when calling setLookAt cameraControls.setTarget(...target.toArray(), false); threeCamera.setFocalLength( (0.5 * threeCamera.getFilmHeight()) / Math.tan(fov / 2.0), @@ -311,6 +305,7 @@ export default function CameraTrajectoryPanel(props: CameraTrajectoryPanelProps) attr[`${baseTreeName}/PlayerCamera`]!.visibility = true; attr[`${baseTreeName}/PlayerCamera`]!.wxyz = [quaternion.w, quaternion.x, quaternion.y, quaternion.z]; attr[`${baseTreeName}/PlayerCamera`]!.position = position.toArray(); + attr[`${baseTreeName}/PlayerCamera`]!.renderModeVisibility = false; } } else { const attr = viewer.nodeAttributesFromName.current; @@ -903,1167 +898,4 @@ export default function CameraTrajectoryPanel(props: CameraTrajectoryPanelProps) CameraTrajectoryPanel.defaultProps = { visible: true, -}; - - - -//function set_camera_position(camera, matrix) { -// const mat = new THREE.Matrix4(); -// mat.fromArray(matrix.elements); -// mat.decompose(camera.position, camera.quaternion, camera.scale); -//} -// -//function CameraList(props) { -// const throttled_time_message_sender = props.throttled_time_message_sender; -// const sceneTree = props.sceneTree; -// const cameras = props.cameras; -// const camera_main = props.camera_main; -// const transform_controls = props.transform_controls; -// const setCameras = props.setCameras; -// const swapCameras = props.swapCameras; -// const fovLabel = props.fovLabel; -// const setFovLabel = props.setFovLabel; -// const cameraProperties = props.cameraProperties; -// const setCameraProperties = props.setCameraProperties; -// const isAnimated = props.isAnimated; -// // eslint-disable-next-line no-unused-vars -// const slider_value = props.slider_value; -// const set_slider_value = props.set_slider_value; -// -// const [expanded, setExpanded] = React.useState(null); -// -// const camera_type = useSelector((state) => state.renderingState.camera_type); -// -// const handleChange = -// (cameraUUID: string) => -// (event: React.SyntheticEvent, isExpanded: boolean) => { -// setExpanded(isExpanded ? cameraUUID : false); -// }; -// -// const set_transform_controls = (index) => { -// // camera helper object so grab the camera inside -// const camera = sceneTree.find_object_no_create([ -// 'Camera Path', -// 'Cameras', -// index.toString(), -// 'Camera', -// ]); -// if (camera !== null) { -// const viewer_buttons = document.getElementsByClassName( -// 'ViewerWindow-buttons', -// )[0]; -// if (camera === transform_controls.object) { -// // double click to remove controls from object -// transform_controls.detach(); -// viewer_buttons.style.display = 'none'; -// } else { -// transform_controls.detach(); -// transform_controls.attach(camera); -// viewer_buttons.style.display = 'block'; -// } -// } -// }; -// -// const reset_slider_render_on_change = () => { -// // set slider and render camera back to 0 -// const slider_min = 0; -// const camera_render = sceneTree.find_object_no_create([ -// 'Cameras', -// 'Render Camera', -// ]); -// const camera_render_helper = sceneTree.find_object_no_create([ -// 'Cameras', -// 'Render Camera', -// 'Helper', -// ]); -// if (cameras.length >= 1) { -// let first_camera = sceneTree.find_object_no_create([ -// 'Camera Path', -// 'Cameras', -// 0, -// 'Camera', -// ]); -// if (first_camera.type !== 'PerspectiveCamera' && cameras.length > 1) { -// first_camera = sceneTree.find_object_no_create([ -// 'Camera Path', -// 'Cameras', -// 1, -// 'Camera', -// ]); -// } -// set_camera_position(camera_render, first_camera.matrix); -// camera_render_helper.set_visibility(true); -// camera_render.fov = first_camera.fov; -// camera_render.renderTime = first_camera.renderTime; -// } -// set_slider_value(slider_min); -// }; -// -// const delete_camera = (index: number) => { -// const camera_render_helper = sceneTree.find_object_no_create([ -// 'Cameras', -// 'Render Camera', -// 'Helper', -// ]); -// console.log('TODO: deleting camera: ', index); -// sceneTree.delete(['Camera Path', 'Cameras', index.toString(), 'Camera']); -// sceneTree.delete([ -// 'Camera Path', -// 'Cameras', -// index.toString(), -// 'Camera Helper', -// ]); -// -// setCameras([...cameras.slice(0, index), ...cameras.slice(index + 1)]); -// // detach and hide transform controls -// transform_controls.detach(); -// const viewer_buttons = document.getElementsByClassName( -// 'ViewerWindow-buttons', -// )[0]; -// viewer_buttons.style.display = 'none'; -// if (cameras.length < 1) { -// camera_render_helper.set_visibility(false); -// } -// reset_slider_render_on_change(); -// }; -// -// const cameraList = cameras.map((camera, index) => { -// return ( -// -// } -// aria-controls="panel1bh-content" -// id="panel1bh-header" -// > -// -// -// -// -// -// -// -// -// -// -// -// -// {isAnimated('FOV') && camera_type !== 'equirectangular' && ( -// HI -// )} -// {!isAnimated('FOV') && !isAnimated('RenderTime') && ( -//

-// Animated camera properties will show up here! -//

-// )} -// -// -// ); -// }); -// return
{cameraList}
; -//} -// -//export default function CameraPanel(props) { -// // unpack relevant information -// const sceneTree = props.sceneTree; -// const camera_main = sceneTree.find_object_no_create([ -// 'Cameras', -// 'Main Camera', -// ]); -// const camera_render = sceneTree.find_object_no_create([ -// 'Cameras', -// 'Render Camera', -// ]); -// const camera_render_helper = sceneTree.find_object_no_create([ -// 'Cameras', -// 'Render Camera', -// 'Helper', -// ]); -// const transform_controls = sceneTree.find_object_no_create([ -// 'Transform Controls', -// ]); -// -// // redux store state -// const DEFAULT_FOV = 50; -// const DEFAULT_RENDER_TIME = 0.0; -// -// interface Camera { -// time: number -// name: string -// }; -// -// // react state -// const [cameras, setCameras] = React.useState([]); -// // Mapping of camera id to each camera's properties -// const [cameraProperties, setCameraProperties] = React.useState(new Map()); -// const [slider_value, set_slider_value] = React.useState(0); -// const [smoothness_value, set_smoothness_value] = React.useState(0.5); -// const [is_playing, setIsPlaying] = React.useState(false); -// const [is_cycle, setIsCycle] = React.useState(false); -// const [seconds, setSeconds] = React.useState(4); -// const [fps, setFps] = React.useState(24); -// const [render_modal_open, setRenderModalOpen] = React.useState(false); -// const [load_path_modal_open, setLoadPathModalOpen] = React.useState(false); -// const [animate, setAnimate] = React.useState(new Set()); -// const [globalFov, setGlobalFov] = React.useState(DEFAULT_FOV); -// const [globalRenderTime, setGlobalRenderTime] = -// React.useState(DEFAULT_RENDER_TIME); -// -// const scene_state = sceneTree.get_scene_state(); -// -// // Template for sharing state between Vanilla JS Three.js and React components -// // eslint-disable-next-line no-unused-vars -// const [mouseInScene, setMouseInScene] = React.useState(false); -// React.useEffect(() => { -// scene_state.addCallback( -// (value: boolean) => setMouseInScene(value), -// 'mouse_in_scene', -// ); -// // eslint-disable-next-line react-hooks/exhaustive-deps -// }, []); -// -// // ui state -// const [fovLabel, setFovLabel] = React.useState(FOV_LABELS.FOV); -// -// // nonlinear render option -// const slider_min = 0; -// const slider_max = 1; -// -// // animation constants -// const total_num_steps = seconds * fps; -// const step_size = slider_max / total_num_steps; -// -// const reset_slider_render_on_add = (new_camera_list) => { -// // set slider and render camera back to 0 -// if (new_camera_list.length >= 1) { -// set_camera_position(camera_render, new_camera_list[0].matrix); -// setFieldOfView(new_camera_list[0].fov); -// set_slider_value(slider_min); -// } -// }; -// -// const add_camera = () => { -// const camera_main_copy = camera_main.clone(); -// camera_main_copy.aspect = 1.0; -// camera_main_copy.fov = globalFov; -// camera_main_copy.renderTime = globalRenderTime; -// const new_camera_properties = new Map(); -// camera_main_copy.properties = new_camera_properties; -// new_camera_properties.set('FOV', globalFov); -// new_camera_properties.set('NAME', `Camera ${cameras.length}`); -// // TIME VALUES ARE 0-1 -// if (cameras.length === 0) { -// new_camera_properties.set('TIME', 0.0); -// } else { -// new_camera_properties.set('TIME', 1.0); -// } -// -// const ratio = (cameras.length - 1) / cameras.length; -// -// const new_properties = new Map(cameraProperties); -// new_properties.forEach((properties) => { -// properties.set('TIME', properties.get('TIME') * ratio); -// }); -// new_properties.set(camera_main_copy.uuid, new_camera_properties); -// setCameraProperties(new_properties); -// -// const new_camera_list = cameras.concat(camera_main_copy); -// setCameras(new_camera_list); -// reset_slider_render_on_add(new_camera_list); -// }; -// -// const setCameraProperty = (property, value, index) => { -// const activeCamera = cameras[index]; -// const activeProperties = new Map(activeCamera.properties); -// activeProperties.set(property, value); -// const newProperties = new Map(cameraProperties); -// newProperties.set(activeCamera.uuid, activeProperties); -// activeCamera.properties = activeProperties; -// setCameraProperties(newProperties); -// }; -// -// const swapCameras = (index: number, new_index: number) => { -// if ( -// Math.min(index, new_index) < 0 || -// Math.max(index, new_index) >= cameras.length -// ) -// return; -// -// const swapCameraTime = cameras[index].time; -// cameras[index].time = cameras[new_index].time; -// cameras[new_index].time = swapCameraTime; -// -// const new_cameras = [ -// ...cameras.slice(0, index), -// ...cameras.slice(index + 1), -// ]; -// setCameras([ -// ...new_cameras.slice(0, new_index), -// cameras[index], -// ...new_cameras.slice(new_index), -// ]); -// -// // reset_slider_render_on_change(); -// }; -// -// // force a rerender if the cameras are dragged around -// let update_cameras_interval = null; -// // eslint-disable-next-line no-unused-vars -// transform_controls.addEventListener('mouseDown', (event) => { -// // prevent multiple loops -// if (update_cameras_interval === null) { -// // hardcoded for 100 ms per update -// update_cameras_interval = setInterval(() => {}, 100); -// } -// }); -// // eslint-disable-next-line no-unused-vars -// transform_controls.addEventListener('mouseUp', (event) => { -// if (update_cameras_interval !== null) { -// clearInterval(update_cameras_interval); -// update_cameras_interval = null; -// setCameras(cameras); -// } -// }); -// -// // draw cameras and curve to the scene -// useEffect(() => { -// // draw the cameras -// -// const labels = Array.from(document.getElementsByClassName('label')); -// labels.forEach((label) => { -// label.remove(); -// }); -// -// sceneTree.delete(['Camera Path', 'Cameras']); // delete old cameras, which is important -// if (cameras.length < 1) { -// dispatch({ -// type: 'write', -// path: 'renderingState/camera_choice', -// data: 'Main Camera', -// }); -// camera_render_helper.set_visibility(false); -// } else { -// camera_render_helper.set_visibility(true); -// } -// for (let i = 0; i < cameras.length; i += 1) { -// const camera = cameras[i]; -// // camera.aspect = render_width / render_height; -// const camera_helper = new CameraHelper(camera, 0x393e46); -// -// const labelDiv = document.createElement('div'); -// labelDiv.className = 'label'; -// labelDiv.textContent = camera.name; -// labelDiv.style.color = 'black'; -// labelDiv.style.backgroundColor = 'rgba(255, 255, 255, 0.61)'; -// labelDiv.style.backdropFilter = 'blur(5px)'; -// labelDiv.style.padding = '6px'; -// labelDiv.style.borderRadius = '6px'; -// labelDiv.style.visibility = 'visible'; -// const camera_label = new CSS2DObject(labelDiv); -// camera_label.name = 'CAMERA_LABEL'; -// camera_label.position.set(0, -0.1, -0.1); -// camera_helper.add(camera_label); -// camera_label.layers.set(0); -// -// // camera -// sceneTree.set_object_from_path( -// ['Camera Path', 'Cameras', i.toString(), 'Camera'], -// camera, -// ); -// // camera helper -// sceneTree.set_object_from_path( -// ['Camera Path', 'Cameras', i.toString(), 'Camera Helper'], -// camera_helper, -// ); -// } -// // eslint-disable-next-line react-hooks/exhaustive-deps -// }, [cameras, cameraProperties, render_width, render_height]); -// -// // update the camera curve -// const curve_object = get_curve_object_from_cameras( -// cameras, -// is_cycle, -// smoothness_value, -// ); -// -// const getKeyframePoint = (progress: Number) => { -// const times = []; -// const ratio = (cameras.length - 1) / cameras.length; -// cameras.forEach((camera) => { -// const time = camera.properties.get('TIME'); -// times.push(is_cycle ? time * ratio : time); -// }); -// -// if (is_cycle) { -// times.push(1.0); -// } -// -// let new_point = 0.0; -// if (progress <= times[0]) { -// new_point = 0.0; -// } else if (progress >= times[times.length - 1]) { -// new_point = 1.0; -// } else { -// let i = 0; -// while ( -// i < times.length - 1 && -// !(progress >= times[i] && progress < times[i + 1]) -// ) { -// i += 1; -// } -// const percentage = (progress - times[i]) / (times[i + 1] - times[i]); -// new_point = (i + percentage) / (times.length - 1); -// } -// return new_point; -// }; -// -// if (cameras.length > 1) { -// const num_points = fps * seconds; -// const points = curve_object.curve_positions.getPoints(num_points); -// const geometry = new THREE.BufferGeometry().setFromPoints(points); -// const spline = new MeshLine(); -// spline.setGeometry(geometry); -// const material = new MeshLineMaterial({ lineWidth: 0.01, color: 0xff5024 }); -// const spline_mesh = new THREE.Mesh(spline.geometry, material); -// sceneTree.set_object_from_path(['Camera Path', 'Curve'], spline_mesh); -// -// // set the camera -// -// const point = getKeyframePoint(slider_value); -// let position = null; -// let lookat = null; -// let up = null; -// let fov = null; -// position = curve_object.curve_positions.getPoint(point); -// lookat = curve_object.curve_lookats.getPoint(point); -// up = curve_object.curve_ups.getPoint(point); -// fov = curve_object.curve_fovs.getPoint(point).z; -// -// const mat = get_transform_matrix(position, lookat, up); -// set_camera_position(camera_render, mat); -// setFieldOfView(fov); -// } else { -// sceneTree.delete(['Camera Path', 'Curve']); -// } -// -// const values = []; -// cameras.forEach((camera) => { -// const time = camera.properties.get('TIME'); -// const ratio = (cameras.length - 1) / cameras.length; -// values.push(is_cycle ? time * ratio : time); -// }); -// -// if (is_cycle && cameras.length !== 0) { -// values.push(1.0); -// } -// -// const handleKeyframeSlider = ( -// newValue: number | number[], -// activeThumb: number, -// ) => { -// if (activeThumb === cameras.length) return; -// const ratio = (cameras.length - 1) / cameras.length; -// const val = newValue[activeThumb]; -// setCameraProperty( -// 'TIME', -// is_cycle ? Math.min(val / ratio, 1.0) : val, -// activeThumb, -// ); -// }; -// -// // when the slider changes, update the main camera position -// useEffect(() => { -// if (cameras.length > 1) { -// const point = getKeyframePoint(slider_value); -// let position = null; -// let lookat = null; -// let up = null; -// let fov = null; -// let render_time = null; -// position = curve_object.curve_positions.getPoint(point); -// lookat = curve_object.curve_lookats.getPoint(point); -// up = curve_object.curve_ups.getPoint(point); -// fov = curve_object.curve_fovs.getPoint(point).z; -// render_time = curve_object.curve_render_times.getPoint(point).z; -// render_time = Math.max(Math.min(render_time, 1.0), 0.0); // clamp time values to [0, 1] -// const mat = get_transform_matrix(position, lookat, up); -// set_camera_position(camera_render, mat); -// setFieldOfView(fov); -// setGlobalFov(fov); -// setGlobalRenderTime(render_time); -// } -// // eslint-disable-next-line react-hooks/exhaustive-deps -// }, [slider_value, render_height, render_width]); -// -// // call this function whenever slider state changes -// useEffect(() => { -// if (is_playing && cameras.length > 1) { -// const interval = setInterval(() => { -// set_slider_value((prev) => prev + step_size); -// }, 1000 / fps); -// return () => clearInterval(interval); -// } -// return () => {}; -// // eslint-disable-next-line react-hooks/exhaustive-deps -// }, [is_playing]); -// -// // make sure to pause if the slider reaches the end -// useEffect(() => { -// if (slider_value >= slider_max) { -// set_slider_value(slider_max); -// setIsPlaying(false); -// } -// }, [slider_value]); -// -// const get_camera_path = () => { -// // NOTE: currently assuming these are ints -// const num_points = fps * seconds; -// const camera_path = []; -// -// for (let i = 0; i < num_points; i += 1) { -// const pt = getKeyframePoint(i / num_points); -// -// const position = curve_object.curve_positions.getPoint(pt); -// const lookat = curve_object.curve_lookats.getPoint(pt); -// const up = curve_object.curve_ups.getPoint(pt); -// const fov = curve_object.curve_fovs.getPoint(pt).z; -// -// const mat = get_transform_matrix(position, lookat, up); -// -// if (display_render_time) { -// const renderTime = curve_object.curve_render_times.getPoint(pt).z; -// camera_path.push({ -// camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix -// fov, -// aspect: camera_render.aspect, -// render_time: Math.max(Math.min(renderTime, 1.0), 0.0), // clamp time values to [0, 1] -// }); -// } else { -// camera_path.push({ -// camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix -// fov, -// aspect: camera_render.aspect, -// }); -// } -// } -// -// const keyframes = []; -// for (let i = 0; i < cameras.length; i += 1) { -// const camera = cameras[i]; -// keyframes.push({ -// matrix: JSON.stringify(camera.matrix.toArray()), -// fov: camera.fov, -// aspect: camera_render.aspect, -// properties: JSON.stringify(Array.from(camera.properties.entries())), -// }); -// } -// -// // const myData -// const camera_path_object = { -// keyframes, -// camera_type, -// render_height, -// render_width, -// camera_path, -// fps, -// seconds, -// smoothness_value, -// is_cycle, -// crop: null, -// }; -// return camera_path_object; -// }; -// -// const export_camera_path = () => { -// // export the camera path -// // inspired by: -// // https://stackoverflow.com/questions/55613438/reactwrite-to-json-file-or-export-download-no-server -// -// sendWebsocketMessage(viser_websocket, { type: 'SaveCheckpointMessage' }); -// -// const camera_path_object = get_camera_path(); -// console.log(camera_render.toJSON()); -// -// // create file in browser -// const json = JSON.stringify(camera_path_object, null, 2); -// const blob = new Blob([json], { type: 'application/json' }); -// const href = URL.createObjectURL(blob); -// -// // create "a" HTLM element with href to file -// const link = document.createElement('a'); -// link.href = href; -// -// const filename = 'camera_path.json'; -// link.download = filename; -// document.body.appendChild(link); -// link.click(); -// // clean up "a" element & remove ObjectURL -// document.body.removeChild(link); -// URL.revokeObjectURL(href); -// }; -// -// const load_camera_path = (camera_path_object) => { -// const new_camera_list = []; -// const new_properties = new Map(cameraProperties); -// -// setRenderHeight(camera_path_object.render_height); -// setRenderWidth(camera_path_object.render_width); -// setCameraType(camera_path_object.camera_type); -// -// setFps(camera_path_object.fps); -// setSeconds(camera_path_object.seconds); -// -// set_smoothness_value(camera_path_object.smoothness_value); -// setIsCycle(camera_path_object.is_cycle); -// -// for (let i = 0; i < camera_path_object.keyframes.length; i += 1) { -// const keyframe = camera_path_object.keyframes[i]; -// const camera = new THREE.PerspectiveCamera( -// keyframe.fov, -// keyframe.aspect, -// 0.1, -// 1000, -// ); -// -// // properties -// camera.properties = new Map(JSON.parse(keyframe.properties)); -// new_properties.set(camera.uuid, camera.properties); -// -// const mat = new THREE.Matrix4(); -// mat.fromArray(JSON.parse(keyframe.matrix)); -// set_camera_position(camera, mat); -// new_camera_list.push(camera); -// } -// -// setCameraProperties(new_properties); -// setCameras(new_camera_list); -// reset_slider_render_on_add(new_camera_list); -// -// if ('crop' in camera_path_object && camera_path_object.crop !== null) { -// const bg_color = camera_path_object.crop.crop_bg_color; -// sendWebsocketMessage(viser_websocket, { -// type: 'CropParamsMessage', -// crop_enabled: true, -// crop_bg_color: [bg_color.r, bg_color.g, bg_color.b], -// crop_center: camera_path_object.crop.crop_center, -// crop_scale: camera_path_object.crop.crop_scale, -// }); -// } -// }; -// -// const uploadCameraPath = (e) => { -// const fileUpload = e.target.files[0]; -// -// const fr = new FileReader(); -// fr.onload = (res) => { -// const camera_path_object = JSON.parse(res.target.result); -// load_camera_path(camera_path_object); -// }; -// -// fr.readAsText(fileUpload); -// }; -// -// const open_render_modal = () => { -// setRenderModalOpen(true); -// -// const camera_path_object = get_camera_path(); -// -// sendWebsocketMessage(viser_websocket, { -// type: 'CameraPathPayloadMessage', -// camera_path_filename: export_path, -// camera_path: camera_path_object, -// }); -// sendWebsocketMessage(viser_websocket, { type: 'SaveCheckpointMessage' }); -// }; -// -// const open_load_path_modal = () => { -// sendWebsocketMessage(viser_websocket, { type: 'CameraPathOptionsRequest' }); -// setLoadPathModalOpen(true); -// }; -// -// const isAnimated = (property) => animate.has(property); -// -// const toggleAnimate = (property) => { -// const new_animate = new Set(animate); -// if (animate.has(property)) { -// new_animate.delete(property); -// setAnimate(new_animate); -// } else { -// new_animate.add(property); -// setAnimate(new_animate); -// } -// }; -// -// const setAllCameraFOV = (val) => { -// if (fovLabel === FOV_LABELS.FOV) { -// for (let i = 0; i < cameras.length; i += 1) { -// cameras[i].fov = val; -// } -// } else { -// for (let i = 0; i < cameras.length; i += 1) { -// cameras[i].setFocalLength(val / cameras[i].aspect); -// } -// } -// }; -// -// const setAllCameraRenderTime = (val) => { -// for (let i = 0; i < cameras.length; i += 1) { -// cameras[i].renderTime = val; -// } -// }; -// -// return ( -//
-//
-//
-// -// -//
-//
-// -//
-//
-// -// -//
-//
-// -// -// -// -//
-// {display_render_time && ( -//
-// -// -// -// -//
-// )} -// {camera_type !== 'equirectangular' && ( -//
-// -// -// -// -//
-// )} -//
-//
-// -//
-//
-// -// {!is_cycle ? ( -// -// ) : ( -// -// )} -// -//
-//
-// -// -// -//
-//
-//
-// -//

Smoothness

-// -// { -// set_smoothness_value(value); -// }} -// /> -// -// -//
-//
-// -// Camera Keyframes -// -// { -// if (cameras.length === 0) { -// return ''; -// } -// if (i === cameras.length && is_cycle) { -// return `${cameras[0].properties.get('NAME')} @ ${parseFloat( -// (value * seconds).toFixed(2), -// )}s`; -// } -// return `${cameras[i].properties.get('NAME')} @ ${parseFloat( -// (value * seconds).toFixed(2), -// )}s`; -// }} -// marks={marks} -// min={slider_min} -// max={slider_max} -// disabled={cameras.length < 2} -// track={false} -// onChange={handleKeyframeSlider} -// sx={{ -// '& .MuiSlider-thumb': { -// borderRadius: '6px', -// width: `${24.0 / Math.max(Math.sqrt(cameras.length), 2)}px`, -// }, -// }} -// disableSwap -// /> -// -// Playback -// -// { -// set_slider_value(value); -// }} -// /> -//
-//
-// -// -// {/* eslint-disable-next-line no-nested-ternary */} -// {!is_playing && slider_max === slider_value ? ( -// -// ) : !is_playing ? ( -// -// ) : ( -// -// )} -// -// -//
-//
-// -//
-//
-// ); -//} \ No newline at end of file +}; \ No newline at end of file diff --git a/src/viser/client/src/ControlPanel/ControlPanel.tsx b/src/viser/client/src/ControlPanel/ControlPanel.tsx index 04b527733..e2223be51 100644 --- a/src/viser/client/src/ControlPanel/ControlPanel.tsx +++ b/src/viser/client/src/ControlPanel/ControlPanel.tsx @@ -21,7 +21,6 @@ import BottomPanel from "./BottomPanel"; import FloatingPanel from "./FloatingPanel"; import { ThemeConfigurationMessage } from "../WebsocketMessages"; import SidebarPanel from "./SidebarPanel"; -import CameraTrajectoryPanel from "./CameraTrajectoryPanel"; // Must match constant in Python. const ROOT_CONTAINER_ID = "root"; @@ -78,7 +77,6 @@ export default function ControlPanel(props: { const panelContents = ( <> - diff --git a/src/viser/client/src/ControlPanel/Generated.tsx b/src/viser/client/src/ControlPanel/Generated.tsx index 093aa46bb..311f3494e 100644 --- a/src/viser/client/src/ControlPanel/Generated.tsx +++ b/src/viser/client/src/ControlPanel/Generated.tsx @@ -32,6 +32,7 @@ import Markdown from "../Markdown"; import { ErrorBoundary } from "react-error-boundary"; import { useDisclosure } from "@mantine/hooks"; import { IconChevronDown, IconChevronUp } from "@tabler/icons-react"; +import CameraTrajectoryPanel from "./CameraTrajectoryPanel"; /** Root of generated inputs. */ export default function GeneratedGuiContainer({ @@ -120,6 +121,11 @@ function GeneratedInput({ ); } + if (conf.type == "GuiAddCameraTrajectoryPanelMessage") { + return ( + + ); + } const messageSender = makeThrottledMessageSender(viewer.websocketRef, 50); function updateValue(value: any) { @@ -446,7 +452,8 @@ function GeneratedInput({ ))} ); - } + break; + } if (conf.hint !== null) input = // We need to add for inputs that we can't assign refs to. diff --git a/src/viser/client/src/ControlPanel/GuiState.tsx b/src/viser/client/src/ControlPanel/GuiState.tsx index 402e84a6d..be2a4db87 100644 --- a/src/viser/client/src/ControlPanel/GuiState.tsx +++ b/src/viser/client/src/ControlPanel/GuiState.tsx @@ -5,7 +5,7 @@ import { ColorTranslator } from "colortranslator"; import { immer } from "zustand/middleware/immer"; import { ViewerContext } from "../App"; -import { MantineThemeOverride } from "@mantine/core"; +import { MantineThemeOverride, Slider } from "@mantine/core"; export type GuiConfig = | Messages.GuiAddButtonMessage @@ -21,7 +21,8 @@ export type GuiConfig = | Messages.GuiAddTextMessage | Messages.GuiAddVector2Message | Messages.GuiAddVector3Message - | Messages.GuiAddMarkdownMessage; + | Messages.GuiAddMarkdownMessage + | Messages.GuiAddCameraTrajectoryPanelMessage; export function isGuiConfig(message: Messages.Message): message is GuiConfig { return message.type.startsWith("GuiAdd"); diff --git a/src/viser/client/src/ControlPanel/MultiSlider.tsx b/src/viser/client/src/ControlPanel/MultiSlider.tsx index 3db5faa10..1a4e2aa45 100644 --- a/src/viser/client/src/ControlPanel/MultiSlider.tsx +++ b/src/viser/client/src/ControlPanel/MultiSlider.tsx @@ -364,19 +364,19 @@ type Value = number[]; export interface MultiSliderProps extends DefaultProps, Omit, 'value' | 'onChange' | 'defaultValue'> { - variant: string; + variant?: string; /** Color from theme.colors */ - color: MantineColor; + color?: MantineColor; /** Key of theme.radius or any valid CSS value to set border-radius, theme.defaultRadius by default */ - radius: MantineNumberSize; + radius?: MantineNumberSize; /** Predefined track and thumb size, number to set sizes */ - size: MantineNumberSize; + size?: MantineNumberSize; /** Minimal possible value */ - min: number; + min?: number; /** Maximum possible value */ max: number; @@ -421,28 +421,28 @@ export interface MultiSliderProps labelTransitionTimingFunction?: string; /** If true label will be not be hidden when user stops dragging */ - labelAlwaysOn: boolean; + labelAlwaysOn?: boolean; /** First thumb aria-label */ - thumbFromLabel: string; + thumbFromLabel?: string; /** Second thumb aria-label */ - thumbToLabel: string; + thumbToLabel?: string; /**If true slider label will appear on hover */ showLabelOnHover?: boolean; /** Thumbs children, can be used to add icons */ - thumbChildren: React.ReactNode | React.ReactNode[] | null; + thumbChildren?: React.ReactNode | React.ReactNode[] | null; /** Disables slider */ - disabled: boolean; + disabled?: boolean; /** Thumb width and height */ - thumbSize: number; + thumbSize?: number; /** A transformation function, to change the scale of the slider */ - scale: (value: number) => number; + scale?: (value: number) => number; fixedEndpoints: boolean; } @@ -502,7 +502,7 @@ export const MultiSlider = forwardRef((props, variant, fixedEndpoints, ...others - } = useComponentDefaultProps('MultiSlider', defaultProps, props); + } = useComponentDefaultProps('MultiSlider', defaultProps, props) as any; const theme = useMantineTheme(); const [focused, setFocused] = useState(-1); diff --git a/src/viser/client/src/SceneTree.tsx b/src/viser/client/src/SceneTree.tsx index 475bfbeba..9978536c3 100644 --- a/src/viser/client/src/SceneTree.tsx +++ b/src/viser/client/src/SceneTree.tsx @@ -113,6 +113,7 @@ export function SceneNodeThreeObject(props: { viewer.useSceneTree((state) => state.nodeFromName[props.name]?.clickable) ?? false; const [obj, setRef] = React.useState(null); + const isRenderMode = viewer.useGui((state) => state.isRenderMode); const dragInfo = React.useRef({ dragging: false, @@ -181,6 +182,10 @@ export function SceneNodeThreeObject(props: { obj.visible = true; } + if (isRenderMode && nodeAttributes.renderModeVisibility !== undefined) { + obj.visible = nodeAttributes.renderModeVisibility; + } + let changed = false; const wxyz = nodeAttributes.wxyz; if (wxyz !== undefined) { diff --git a/src/viser/client/src/WebsocketMessages.tsx b/src/viser/client/src/WebsocketMessages.tsx index 1c3947c94..88cea29c4 100644 --- a/src/viser/client/src/WebsocketMessages.tsx +++ b/src/viser/client/src/WebsocketMessages.tsx @@ -552,6 +552,19 @@ export interface GuiCloseModalMessage { type: "GuiCloseModalMessage"; id: string; } +/** GuiAddCameraTrajectoryPanelMessage(order: 'float', id: 'str', container_id: 'str', fov: 'float' = 75.0, render_width: 'int' = 1920, render_height: 'int' = 1080) + * + * (automatically generated) + */ +export interface GuiAddCameraTrajectoryPanelMessage { + type: "GuiAddCameraTrajectoryPanelMessage"; + order: number; + id: string; + container_id: string; + fov: number; + render_width: number; + render_height: number; +} /** Sent server->client to remove a GUI element. * * (automatically generated) @@ -751,6 +764,7 @@ export type Message = | GuiAddButtonGroupMessage | GuiModalMessage | GuiCloseModalMessage + | GuiAddCameraTrajectoryPanelMessage | GuiRemoveMessage | GuiUpdateMessage | GuiSetVisibleMessage From 6370966b0e374b7fda76950655ec8c4ea8a07473 Mon Sep 17 00:00:00 2001 From: Jonas Kulhanek Date: Sat, 30 Dec 2023 22:51:05 +0100 Subject: [PATCH 4/4] Prepare design change --- examples/22_custom_gui_component.py | 28 + src/viser/_gui_api.py | 81 +- src/viser/_gui_handles.py | 9 +- src/viser/_messages.py | 26 +- src/viser/client/src/App.tsx | 28 + src/viser/client/src/CameraControls.tsx | 1 - .../ControlPanel/CameraTrajectoryPanel.tsx | 24 +- .../client/src/ControlPanel/ControlPanel.tsx | 2 + .../client/src/ControlPanel/Generated.tsx | 163 +- .../client/src/ControlPanel/GuiState.tsx | 33 +- .../client/src/ControlPanel/MultiSlider.tsx | 16 +- src/viser/client/src/WebsocketMessages.tsx | 32 +- src/viser/infra/_icons.py | 3060 +++++++++++++++++ src/viser/infra/_messages.py | 37 +- src/viser/infra/_typescript_interface_gen.py | 13 + 15 files changed, 3433 insertions(+), 120 deletions(-) create mode 100644 src/viser/infra/_icons.py diff --git a/examples/22_custom_gui_component.py b/examples/22_custom_gui_component.py index e8b76d6e3..6c1bd4a1b 100644 --- a/examples/22_custom_gui_component.py +++ b/examples/22_custom_gui_component.py @@ -8,6 +8,7 @@ import trimesh import viser import viser.transforms as tf +from viser import Icon mesh = trimesh.load_mesh(Path(__file__).parent / "assets/dragon.obj") assert isinstance(mesh, trimesh.Trimesh) @@ -18,6 +19,33 @@ print(f"Loaded mesh with {vertices.shape} vertices, {faces.shape} faces") server = viser.ViserServer() + +import_button = server.add_gui_button("Import", icon=Icon.FOLDER_OPEN) +export_button = server.add_gui_button("Export", icon=Icon.DOWNLOAD) + +fps = server.add_gui_number("FPS", 24, min=1, icon=Icon.KEYFRAMES, hint="Frames per second") +duration = server.add_gui_number("Duration", 4.0, min=0.1, icon=Icon.CLOCK_HOUR_5, hint="Duration in seconds") +width = server.add_gui_number("Width", 1920, min=100, icon=Icon.ARROWS_HORIZONTAL, hint="Width in px") +height = server.add_gui_number("Height", 1080, min=100, icon=Icon.ARROWS_VERTICAL, hint="Height in px") +fov = server.add_gui_number("FOV", 75, min=1, max=179, icon=Icon.CAMERA, hint="Field of view") +smoothness = server.add_gui_slider("Smoothness", 0.5, min=0.0, max=1.0, step=0.01, hint="Trajectory smoothing") + + +duration = 4 +cameras_slider = server.add_gui_multi_slider( + "Timeline", + min=0., + max=1., + step=0.01, + initial_value=[0.0, 0.5, 1.0], + disabled=False, + marks=[(x, f'{x*duration:.1f}s') for x in [0., 0.5, 1.0]], +) + +@duration.on_update +def _(_) -> None: + cameras_slider.marks=[(x, f'{x*duration.value:.1f}s') for x in [0., 0.5, 1.0]], + server.add_mesh_simple( name="/simple", vertices=vertices, diff --git a/src/viser/_gui_api.py b/src/viser/_gui_api.py index 1112b1237..0140f057e 100644 --- a/src/viser/_gui_api.py +++ b/src/viser/_gui_api.py @@ -43,7 +43,6 @@ _GuiInputHandle, _make_unique_id, ) -from ._icons import base64_from_icon from ._icons_enum import Icon from ._message_api import MessageApi, cast_vector @@ -273,7 +272,7 @@ def add_gui_tab_group( return GuiTabGroupHandle( _tab_group_id=tab_group_id, _labels=[], - _icons_base64=[], + _icons=[], _tabs=[], _gui_api=self, _container_id=self._get_container_id(), @@ -367,7 +366,7 @@ def add_gui_button( hint=hint, initial_value=False, color=color, - icon_base64=None if icon is None else base64_from_icon(icon), + icon=None if icon is None else icon.value, ), disabled=disabled, visible=visible, @@ -535,6 +534,7 @@ def add_gui_number( visible: bool = True, hint: Optional[str] = None, order: Optional[float] = None, + icon: Optional[Icon] = None, ) -> GuiInputHandle[IntOrFloat]: """Add a number input to the GUI, with user-specifiable bound and precision parameters. @@ -585,6 +585,7 @@ def add_gui_number( min=min, max=max, precision=_compute_precision_digits(step), + icon=icon.value if icon is not None else None, step=step, ), disabled=disabled, @@ -888,6 +889,80 @@ def add_gui_slider( is_button=False, ) + def add_gui_multi_slider( + self, + label: str, + min: IntOrFloat, + max: IntOrFloat, + step: IntOrFloat, + initial_value: List[IntOrFloat], + disabled: bool = False, + visible: bool = True, + min_range: Optional[IntOrFloat] = None, + hint: Optional[str] = None, + order: Optional[float] = None, + marks: Optional[List[Tuple[IntOrFloat, Optional[str]]]] = None, + ) -> GuiInputHandle[IntOrFloat]: + """Add a multi slider to the GUI. Types of the min, max, step, and initial value should match. + + Args: + label: Label to display on the slider. + min: Minimum value of the slider. + max: Maximum value of the slider. + step: Step size of the slider. + initial_value: Initial values of the slider. + disabled: Whether the slider is disabled. + visible: Whether the slider is visible. + min_range: Optional minimum difference between two values of the slider. + hint: Optional hint to display on hover. + order: Optional ordering, smallest values will be displayed first. + + Returns: + A handle that can be used to interact with the GUI element. + """ + assert max >= min + if step > max - min: + step = max - min + assert all(max >= x >= min for x in initial_value) + + # GUI callbacks cast incoming values to match the type of the initial value. If + # the min, max, or step is a float, we should cast to a float. + if len(initial_value) > 0 and (type(initial_value[0]) is int and ( + type(min) is float or type(max) is float or type(step) is float + )): + initial_value = [float(x) for x in initial_value] # type: ignore + + # TODO: as of 6/5/2023, this assert will break something in nerfstudio. (at + # least LERF) + # + # assert type(min) == type(max) == type(step) == type(initial_value) + + id = _make_unique_id() + order = _apply_default_order(order) + return self._create_gui_input( + initial_value=initial_value, + message=_messages.GuiAddMultiSliderMessage( + order=order, + id=id, + label=label, + container_id=self._get_container_id(), + hint=hint, + min=min, + min_range=min_range, + max=max, + step=step, + initial_value=initial_value, + precision=_compute_precision_digits(step), + marks=[ + _messages.GuiSliderMark(value=x, label=label) + for x, label in marks + ] if marks is not None else None, + ), + disabled=disabled, + visible=visible, + is_button=False, + ) + def add_gui_rgb( self, label: str, diff --git a/src/viser/_gui_handles.py b/src/viser/_gui_handles.py index d0a0e0539..7a3ab92f9 100644 --- a/src/viser/_gui_handles.py +++ b/src/viser/_gui_handles.py @@ -26,7 +26,6 @@ import numpy as onp from typing_extensions import Protocol -from ._icons import base64_from_icon from ._icons_enum import Icon from ._message_api import _encode_image_base64 from ._messages import ( @@ -332,7 +331,7 @@ def options(self, options: Iterable[StringType]) -> None: class GuiTabGroupHandle: _tab_group_id: str _labels: List[str] - _icons_base64: List[Optional[str]] + _icons: List[Optional[str]] _tabs: List[GuiTabHandle] _gui_api: GuiApi _container_id: str # Parent. @@ -352,7 +351,7 @@ def add_tab(self, label: str, icon: Optional[Icon] = None) -> GuiTabHandle: out = GuiTabHandle(_parent=self, _id=id) self._labels.append(label) - self._icons_base64.append(None if icon is None else base64_from_icon(icon)) + self._icons.append(None if icon is None else icon.value) self._tabs.append(out) self._sync_with_client() @@ -372,7 +371,7 @@ def _sync_with_client(self) -> None: id=self._tab_group_id, container_id=self._container_id, tab_labels=tuple(self._labels), - tab_icons_base64=tuple(self._icons_base64), + tab_icons=tuple(self._icons), tab_container_ids=tuple(tab._id for tab in self._tabs), ) ) @@ -495,7 +494,7 @@ def remove(self) -> None: self._parent._gui_api._container_handle_from_id.pop(self._id) self._parent._labels.pop(container_index) - self._parent._icons_base64.pop(container_index) + self._parent._icons.pop(container_index) self._parent._tabs.pop(container_index) self._parent._sync_with_client() diff --git a/src/viser/_messages.py b/src/viser/_messages.py index 5e96c048e..0d7d304da 100644 --- a/src/viser/_messages.py +++ b/src/viser/_messages.py @@ -4,7 +4,7 @@ from __future__ import annotations import dataclasses -from typing import Any, Optional, Tuple, Union +from typing import Any, Optional, Tuple, Union, List import numpy as onp import numpy.typing as onpt @@ -364,7 +364,7 @@ class GuiAddTabGroupMessage(Message): id: str container_id: str tab_labels: Tuple[str, ...] - tab_icons_base64: Tuple[Union[str, None], ...] + tab_icons: Tuple[Union[str, None], ...] tab_container_ids: Tuple[str, ...] @@ -415,7 +415,7 @@ class GuiAddButtonMessage(_GuiAddInputBase): "teal", ] ] - icon_base64: Optional[str] + icon: Optional[str] @dataclasses.dataclass @@ -427,6 +427,25 @@ class GuiAddSliderMessage(_GuiAddInputBase): precision: int +@dataclasses.dataclass +class GuiSliderMark: + value: float + label: Optional[str] = None + + +@dataclasses.dataclass +class GuiAddMultiSliderMessage(_GuiAddInputBase): + min: float + max: float + step: Optional[float] + min_range: Optional[float] + initial_value: List[float] + precision: int + fixed_endpoints: bool = False + marks: Optional[List[GuiSliderMark]] = None + + + @dataclasses.dataclass class GuiAddNumberMessage(_GuiAddInputBase): initial_value: float @@ -434,6 +453,7 @@ class GuiAddNumberMessage(_GuiAddInputBase): step: float min: Optional[float] max: Optional[float] + icon: Optional[str] @dataclasses.dataclass diff --git a/src/viser/client/src/App.tsx b/src/viser/client/src/App.tsx index 65bf8ac01..40f477199 100644 --- a/src/viser/client/src/App.tsx +++ b/src/viser/client/src/App.tsx @@ -83,6 +83,8 @@ export type ViewerContextContents = { >; getRenderRequest: React.MutableRefObject; sceneClickEnable: React.MutableRefObject; + setIsRenderMode: (isRenderMode: boolean) => void; + viewerModeCameraStateBackup: React.MutableRefObject; }; export const ViewerContext = React.createContext( null, @@ -129,6 +131,32 @@ function ViewerRoot() { getRenderRequestState: React.useRef("ready"), getRenderRequest: React.useRef(null), sceneClickEnable: React.useRef(false), + viewerModeCameraStateBackup: React.useRef(null), + setIsRenderMode: (isRenderMode: boolean) => { + viewer.useGui.setState((state) => { + state.isRenderMode = isRenderMode; + const camera = viewer.cameraRef.current!; + // Backup/restore camera state + if (isRenderMode) { + viewer.viewerModeCameraStateBackup.current = [ + [...camera.position.toArray()], + viewer.cameraControlRef.current!.toJSON() + ]; + } else if (viewer.viewerModeCameraStateBackup.current !== null) { + const [cameraData, cameraControl] = viewer.viewerModeCameraStateBackup.current!; + const position = new THREE.Vector3(...cameraData.slice(0, 3)); + const rotation = new THREE.Euler(...cameraData.slice(3, 6)); + //viewer.cameraControlRef.current?.fromJSON(cameraControl, false); + camera.position.copy(position); + camera.rotation.copy(rotation); + camera.updateProjectionMatrix(); + //viewer.cameraControlRef.current?.updateCameraUp(); + //viewer.cameraControlRef.current?.update(1); + viewer.viewerModeCameraStateBackup.current = null; + } + //console.log(camera.up); + }); + } }; return ( diff --git a/src/viser/client/src/CameraControls.tsx b/src/viser/client/src/CameraControls.tsx index 9035dd03a..961bceed6 100644 --- a/src/viser/client/src/CameraControls.tsx +++ b/src/viser/client/src/CameraControls.tsx @@ -44,7 +44,6 @@ export function SynchronizedCameraControls() { // Callback for sending cameras. const sendCamera = React.useCallback(() => { - console.log("Sending camera"); const three_camera = camera; const camera_control = viewer.cameraControlRef.current; diff --git a/src/viser/client/src/ControlPanel/CameraTrajectoryPanel.tsx b/src/viser/client/src/ControlPanel/CameraTrajectoryPanel.tsx index f83320534..99b43bf84 100644 --- a/src/viser/client/src/ControlPanel/CameraTrajectoryPanel.tsx +++ b/src/viser/client/src/ControlPanel/CameraTrajectoryPanel.tsx @@ -87,13 +87,11 @@ function getPoseFromCamera(viewer: ViewerContextContents) { const three_camera = viewer.cameraRef.current!; const R_threecam_cam = new THREE.Quaternion().setFromEuler( new THREE.Euler(Math.PI, 0.0, 0.0), - ).multiply(new THREE.Quaternion().setFromEuler( - new THREE.Euler(0.0, 0.0, Math.PI), - )); + ); const R_world_threeworld = getR_threeworld_world(viewer).invert(); const R_world_camera = R_world_threeworld.clone() - .multiply(three_camera.quaternion) - .multiply(R_threecam_cam); + .multiply(three_camera.quaternion); + //.multiply(R_threecam_cam); return { wxyz: [ @@ -265,19 +263,21 @@ export default function CameraTrajectoryPanel(props: CameraTrajectoryPanelProps) const up = curveObject.curve_ups.getPoint(point).multiplyScalar(-1).applyQuaternion(R_threeworld_world); const fov = curveObject.curve_fovs.getPoint(point).z; - const cameraControls = viewer.cameraControlRef.current!; + // const cameraControls = viewer.cameraControlRef.current!; const threeCamera = viewer.cameraRef.current!; + threeCamera.position.set(...position.toArray()); threeCamera.up.set(...up.toArray()); - cameraControls.updateCameraUp(); - cameraControls.setLookAt(...position.toArray(), ...lookat.toArray(), false); - const target = position.clone().add(lookat); + threeCamera.lookAt(...lookat.toArray()); + // cameraControls.updateCameraUp(); + // cameraControls.setLookAt(...position.toArray(), ...lookat.toArray(), false); + // const target = position.clone().add(lookat); // NOTE: lookat is being ignored when calling setLookAt - cameraControls.setTarget(...target.toArray(), false); + // cameraControls.setTarget(...target.toArray(), false); threeCamera.setFocalLength( (0.5 * threeCamera.getFilmHeight()) / Math.tan(fov / 2.0), ); - cameraControls.update(1.); + // cameraControls.update(1.); } else { const point = getKeyframePoint(playerTime); const position = curveObject.curve_positions.getPoint(point); @@ -889,7 +889,7 @@ export default function CameraTrajectoryPanel(props: CameraTrajectoryPanelProps) label="Render mode" checked={isRenderMode} onChange={(event) => { - viewer.useGui.setState({ isRenderMode: event.currentTarget.checked }); + viewer.setIsRenderMode(event.currentTarget.checked); }} size="sm" /> diff --git a/src/viser/client/src/ControlPanel/ControlPanel.tsx b/src/viser/client/src/ControlPanel/ControlPanel.tsx index e2223be51..04b527733 100644 --- a/src/viser/client/src/ControlPanel/ControlPanel.tsx +++ b/src/viser/client/src/ControlPanel/ControlPanel.tsx @@ -21,6 +21,7 @@ import BottomPanel from "./BottomPanel"; import FloatingPanel from "./FloatingPanel"; import { ThemeConfigurationMessage } from "../WebsocketMessages"; import SidebarPanel from "./SidebarPanel"; +import CameraTrajectoryPanel from "./CameraTrajectoryPanel"; // Must match constant in Python. const ROOT_CONTAINER_ID = "root"; @@ -77,6 +78,7 @@ export default function ControlPanel(props: { const panelContents = ( <> + diff --git a/src/viser/client/src/ControlPanel/Generated.tsx b/src/viser/client/src/ControlPanel/Generated.tsx index 311f3494e..6852d905e 100644 --- a/src/viser/client/src/ControlPanel/Generated.tsx +++ b/src/viser/client/src/ControlPanel/Generated.tsx @@ -7,7 +7,7 @@ import { makeThrottledMessageSender } from "../WebsocketFunctions"; import { computeRelativeLuminance } from "./GuiState"; import { Collapse, - Image, + Group, Paper, Tabs, TabsValue, @@ -31,8 +31,18 @@ import React from "react"; import Markdown from "../Markdown"; import { ErrorBoundary } from "react-error-boundary"; import { useDisclosure } from "@mantine/hooks"; -import { IconChevronDown, IconChevronUp } from "@tabler/icons-react"; +import { IconChevronDown, IconChevronUp, TablerIconsProps } from "@tabler/icons-react"; import CameraTrajectoryPanel from "./CameraTrajectoryPanel"; +import { MultiSlider } from "./MultiSlider"; +import * as TablerIcons from '@tabler/icons-react'; + + +function createIcon(icon: string) : (props: TablerIconsProps) => JSX.Element { + // Icon name is in snake-case + // We need to convert it to PascalCase + const iconPascal = icon.split('-').map((word) => word.charAt(0).toUpperCase() + word.slice(1)).join(''); + return (TablerIcons as any)[("Icon" + iconPascal) as any] as unknown as (props: TablerIconsProps) => JSX.Element; +} /** Root of generated inputs. */ export default function GeneratedGuiContainer({ @@ -154,6 +164,8 @@ function GeneratedInput({ let labeled = true; let input = null; + const iconString = (conf as { icon?: string }).icon; + const Icon = iconString && createIcon(iconString); switch (conf.type) { case "GuiAddButtonMessage": labeled = false; @@ -182,27 +194,7 @@ function GeneratedInput({ styles={{ inner: { color: inputColor + " !important" } }} disabled={disabled} size="sm" - leftIcon={ - conf.icon_base64 === null ? undefined : ( - - ) - } + leftIcon={Icon && } > {conf.label} @@ -294,6 +286,7 @@ function GeneratedInput({ height: "1.625rem", }, }} + icon={Icon && } disabled={disabled} stepHoldDelay={500} stepHoldInterval={(t) => Math.max(1000 / t ** 2, 25)} @@ -430,7 +423,7 @@ function GeneratedInput({ break; case "GuiAddButtonGroupMessage": input = ( - + {conf.options.map((option, index) => (