From e2c2116b65ee2b61bc49a902e4eee7c524cd9caf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=81=D1=83=D0=BB?= Date: Fri, 29 Nov 2024 14:04:31 +0300 Subject: [PATCH] feat: add transcript --- .../mindset/components/Scene/Board/index.tsx | 31 +----- .../mindset/components/Scene/index.tsx | 42 ++++++-- .../Sidebar/Transcript/Viewer/index.tsx | 98 +++++++++++++++++++ .../components/Sidebar/Transcript/index.tsx | 4 +- 4 files changed, 139 insertions(+), 36 deletions(-) create mode 100644 src/components/mindset/components/Sidebar/Transcript/Viewer/index.tsx diff --git a/src/components/mindset/components/Scene/Board/index.tsx b/src/components/mindset/components/Scene/Board/index.tsx index 903c89049..6bf72c1e8 100644 --- a/src/components/mindset/components/Scene/Board/index.tsx +++ b/src/components/mindset/components/Scene/Board/index.tsx @@ -1,5 +1,5 @@ import { useThree } from '@react-three/fiber' -import { Fragment, useEffect, useMemo } from 'react' +import { Fragment, useMemo } from 'react' import { Vector3 } from 'three' import { useDataStore } from '~/stores/useDataStore' import { Link } from '~/types' @@ -25,34 +25,7 @@ type LinkExtended = Link & { export const Board = () => { const state = useThree() const { dataInitial } = useDataStore((s) => s) - const { camera, viewport } = state - - useEffect(() => { - const orthoCamera = camera as THREE.OrthographicCamera - - const handleWheel = (event: WheelEvent) => { - event.preventDefault() // Prevent default scrolling behavior - - if (event.ctrlKey) { - // Zoom the camera when ctrlKey is pressed - orthoCamera.zoom += event.deltaY * -0.1 // Adjust zoom level - orthoCamera.zoom = Math.max(2, Math.min(orthoCamera.zoom, 20)) // Clamp zoom - } - - // Move the camera left/right when ctrlKey is NOT pressed - orthoCamera.position.x += event.deltaX * 0.1 // Horizontal movement - - orthoCamera.updateProjectionMatrix() // Update projection matrix - } - - // Add the event listener - window.addEventListener('wheel', handleWheel, { passive: false }) - - return () => { - // Cleanup event listener - window.removeEventListener('wheel', handleWheel) - } - }, [camera]) + const { viewport } = state const processedData = useMemo(() => { if (!dataInitial) { diff --git a/src/components/mindset/components/Scene/index.tsx b/src/components/mindset/components/Scene/index.tsx index d3ef6ccb9..a861bb744 100644 --- a/src/components/mindset/components/Scene/index.tsx +++ b/src/components/mindset/components/Scene/index.tsx @@ -1,12 +1,12 @@ import { OrthographicCamera } from '@react-three/drei' import { Canvas, useFrame, useThree } from '@react-three/fiber' -import { memo } from 'react' +import { memo, useEffect } from 'react' import { usePlayerStore } from '~/stores/usePlayerStore' import { Board } from './Board' const CameraController = () => { - const { camera, viewport } = useThree() // Access the Three.js camera - const playerRef = usePlayerStore((s) => s.playerRef) // Ref to store the current `playerRef` + const { camera, viewport } = useThree() + const playerRef = usePlayerStore((s) => s.playerRef) useFrame(() => { if (playerRef) { @@ -16,7 +16,38 @@ const CameraController = () => { } }) - return null // No React-rendered output + return null +} + +const CanvasZoomHandler = () => { + const { camera, gl } = useThree() + + useEffect(() => { + const orthoCamera = camera as THREE.OrthographicCamera + + const handleWheel = (event: WheelEvent) => { + event.preventDefault() + + if (event.ctrlKey) { + orthoCamera.zoom += event.deltaY * -0.1 + orthoCamera.zoom = Math.max(2, Math.min(orthoCamera.zoom, 20)) + } + + orthoCamera.position.x += event.deltaX * 0.1 // + + orthoCamera.updateProjectionMatrix() + } + + const canvas = gl.domElement + + canvas.addEventListener('wheel', handleWheel, { passive: false }) + + return () => { + canvas.removeEventListener('wheel', handleWheel) + } + }, [camera, gl]) + + return null } export const Scene = memo(() => { @@ -25,11 +56,10 @@ export const Scene = memo(() => { return (
- {/* */} - {/* Camera controller to manage GSAP animations */} +
) diff --git a/src/components/mindset/components/Sidebar/Transcript/Viewer/index.tsx b/src/components/mindset/components/Sidebar/Transcript/Viewer/index.tsx new file mode 100644 index 000000000..3fa2cfb2f --- /dev/null +++ b/src/components/mindset/components/Sidebar/Transcript/Viewer/index.tsx @@ -0,0 +1,98 @@ +import clsx from 'clsx' +import { useEffect, useState } from 'react' +import styled from 'styled-components' +import { usePlayerStore } from '~/stores/usePlayerStore' +import { colors } from '~/utils' + +type Props = { + transcriptString: string +} + +type Word = { + word: string + start: number + end: number + probability: number +} + +type TranscriptData = { + id: number + seek: number + start: number + end: number + text: string + tokens: number[] + temperature: number + avg_logprob: number + compression_ratio: number + no_speech_prob: number + words: Word[] +} + +export const Viewer = ({ transcriptString }: Props) => { + const [currentTime, setCurrentTime] = useState(0) + const { playerRef } = usePlayerStore((s) => s) + const cleaned = transcriptString.replace(/^["']|["']$/g, '') + + const transcriptData: TranscriptData[] = JSON.parse(cleaned) + + useEffect(() => { + const interval = setInterval(() => { + if (playerRef && setCurrentTime) { + const time = playerRef.getCurrentTime() + + setCurrentTime(time) + } + }, 100) + + return () => clearInterval(interval) + }, [playerRef, setCurrentTime]) + + return ( + + {transcriptData.map((i) => { + const start = i.start.toFixed(2) + const end = i.end.toFixed(2) + + return ( + + + {start}:{end} + + {i.text} + {i.words.map((word) => { + const isActive = word.start < currentTime && currentTime < word.end + + return ( + + {word.word} + + ) + })} + + ) + })} + + ) +} + +const Paragraph = styled.div` + font-size: 14px; + word-break: break-word; +` + +const Wrapper = styled.div` + width: 100%; +` + +const Start = styled.span`` + +const Text = styled.span`` + +const Word = styled.span` + margin: 0 2px; + + &.active { + background: ${colors.lightBlue300}; + } +` diff --git a/src/components/mindset/components/Sidebar/Transcript/index.tsx b/src/components/mindset/components/Sidebar/Transcript/index.tsx index c7b73e9a7..9e38a1053 100644 --- a/src/components/mindset/components/Sidebar/Transcript/index.tsx +++ b/src/components/mindset/components/Sidebar/Transcript/index.tsx @@ -6,6 +6,7 @@ import { useMindsetStore } from '~/stores/useMindsetStore' import { usePlayerStore } from '~/stores/usePlayerStore' import { NodeExtended } from '~/types' import { colors } from '~/utils' +import { Viewer } from './Viewer' export const Transcript = () => { const { selectedEpisodeId } = useMindsetStore((s) => s) @@ -45,7 +46,8 @@ export const Transcript = () => { // Multiply playingTime by 1000 to match millisecond format return ( - {clip?.properties?.text && {clip?.properties?.text}} + {!clip.properties?.transcript && clip?.properties?.text && {clip?.properties?.text}} + {clip.properties?.transcript && } ) }