Merge pull request #2485 from stakwork/feature/transcript

feat: add transcript

src/components/mindset/components/Scene/Board/index.tsx

@@ -1,5 +1,5 @@
 import { useThree } from '@react-three/fiber'
-import { Fragment, useEffect, useMemo } from 'react'
+import { Fragment, useMemo } from 'react'
 import { Vector3 } from 'three'
 import { useDataStore } from '~/stores/useDataStore'
 import { Link } from '~/types'
@@ -25,34 +25,7 @@ type LinkExtended = Link & {
 export const Board = () => {
   const state = useThree()
   const { dataInitial } = useDataStore((s) => s)
-  const { camera, viewport } = state
-
-  useEffect(() => {
-    const orthoCamera = camera as THREE.OrthographicCamera
-
-    const handleWheel = (event: WheelEvent) => {
-      event.preventDefault() // Prevent default scrolling behavior
-
-      if (event.ctrlKey) {
-        // Zoom the camera when ctrlKey is pressed
-        orthoCamera.zoom += event.deltaY * -0.1 // Adjust zoom level
-        orthoCamera.zoom = Math.max(2, Math.min(orthoCamera.zoom, 20)) // Clamp zoom
-      }
-
-      // Move the camera left/right when ctrlKey is NOT pressed
-      orthoCamera.position.x += event.deltaX * 0.1 // Horizontal movement
-
-      orthoCamera.updateProjectionMatrix() // Update projection matrix
-    }
-
-    // Add the event listener
-    window.addEventListener('wheel', handleWheel, { passive: false })
-
-    return () => {
-      // Cleanup event listener
-      window.removeEventListener('wheel', handleWheel)
-    }
-  }, [camera])
+  const { viewport } = state
 
   const processedData = useMemo(() => {
     if (!dataInitial) {

src/components/mindset/components/Scene/index.tsx

@@ -1,12 +1,12 @@
 import { OrthographicCamera } from '@react-three/drei'
 import { Canvas, useFrame, useThree } from '@react-three/fiber'
-import { memo } from 'react'
+import { memo, useEffect } from 'react'
 import { usePlayerStore } from '~/stores/usePlayerStore'
 import { Board } from './Board'
 
 const CameraController = () => {
-  const { camera, viewport } = useThree() // Access the Three.js camera
-  const playerRef = usePlayerStore((s) => s.playerRef) // Ref to store the current `playerRef`
+  const { camera, viewport } = useThree()
+  const playerRef = usePlayerStore((s) => s.playerRef)
 
   useFrame(() => {
     if (playerRef) {
@@ -16,7 +16,38 @@ const CameraController = () => {
     }
   })
 
-  return null // No React-rendered output
+  return null
+}
+
+const CanvasZoomHandler = () => {
+  const { camera, gl } = useThree()
+
+  useEffect(() => {
+    const orthoCamera = camera as THREE.OrthographicCamera
+
+    const handleWheel = (event: WheelEvent) => {
+      event.preventDefault()
+
+      if (event.ctrlKey) {
+        orthoCamera.zoom += event.deltaY * -0.1
+        orthoCamera.zoom = Math.max(2, Math.min(orthoCamera.zoom, 20))
+      }
+
+      orthoCamera.position.x += event.deltaX * 0.1 //
+
+      orthoCamera.updateProjectionMatrix()
+    }
+
+    const canvas = gl.domElement
+
+    canvas.addEventListener('wheel', handleWheel, { passive: false })
+
+    return () => {
+      canvas.removeEventListener('wheel', handleWheel)
+    }
+  }, [camera, gl])
+
+  return null
 }
 
 export const Scene = memo(() => {
@@ -25,11 +56,10 @@ export const Scene = memo(() => {
   return (
     <div style={{ width: '100%', height: '100%', position: 'relative' }}>
       <Canvas orthographic>
-        {/* <color args={['white']} attach="background" /> */}
-        {/* Camera controller to manage GSAP animations */}
         <CameraController />
         <OrthographicCamera far={2000} makeDefault near={1} position={[0, 0, 20]} zoom={10} />
         <Board />
+        <CanvasZoomHandler />
       </Canvas>
     </div>
   )

src/components/mindset/components/Sidebar/Transcript/Viewer/index.tsx

@@ -0,0 +1,98 @@
+import clsx from 'clsx'
+import { useEffect, useState } from 'react'
+import styled from 'styled-components'
+import { usePlayerStore } from '~/stores/usePlayerStore'
+import { colors } from '~/utils'
+
+type Props = {
+  transcriptString: string
+}
+
+type Word = {
+  word: string
+  start: number
+  end: number
+  probability: number
+}
+
+type TranscriptData = {
+  id: number
+  seek: number
+  start: number
+  end: number
+  text: string
+  tokens: number[]
+  temperature: number
+  avg_logprob: number
+  compression_ratio: number
+  no_speech_prob: number
+  words: Word[]
+}
+
+export const Viewer = ({ transcriptString }: Props) => {
+  const [currentTime, setCurrentTime] = useState(0)
+  const { playerRef } = usePlayerStore((s) => s)
+  const cleaned = transcriptString.replace(/^["']|["']$/g, '')
+
+  const transcriptData: TranscriptData[] = JSON.parse(cleaned)
+
+  useEffect(() => {
+    const interval = setInterval(() => {
+      if (playerRef && setCurrentTime) {
+        const time = playerRef.getCurrentTime()
+
+        setCurrentTime(time)
+      }
+    }, 100)
+
+    return () => clearInterval(interval)
+  }, [playerRef, setCurrentTime])
+
+  return (
+    <Wrapper>
+      {transcriptData.map((i) => {
+        const start = i.start.toFixed(2)
+        const end = i.end.toFixed(2)
+
+        return (
+          <Paragraph key={i.id}>
+            <Start>
+              {start}:{end}
+            </Start>
+            <Text>{i.text}</Text>
+            {i.words.map((word) => {
+              const isActive = word.start < currentTime && currentTime < word.end
+
+              return (
+                <Word key={`${word.start}`} className={clsx({ active: isActive })}>
+                  {word.word}
+                </Word>
+              )
+            })}
+          </Paragraph>
+        )
+      })}
+    </Wrapper>
+  )
+}
+
+const Paragraph = styled.div`
+  font-size: 14px;
+  word-break: break-word;
+`
+
+const Wrapper = styled.div`
+  width: 100%;
+`
+
+const Start = styled.span``
+
+const Text = styled.span``
+
+const Word = styled.span`
+  margin: 0 2px;
+
+  &.active {
+    background: ${colors.lightBlue300};
+  }
+`

src/components/mindset/components/Sidebar/Transcript/index.tsx

@@ -6,6 +6,7 @@ import { useMindsetStore } from '~/stores/useMindsetStore'
 import { usePlayerStore } from '~/stores/usePlayerStore'
 import { NodeExtended } from '~/types'
 import { colors } from '~/utils'
+import { Viewer } from './Viewer'
 
 export const Transcript = () => {
   const { selectedEpisodeId } = useMindsetStore((s) => s)
@@ -45,7 +46,8 @@ export const Transcript = () => {
           // Multiply playingTime by 1000 to match millisecond format
           return (
             <TranscriptWrapper key={clip.ref_id} direction="row">
-              {clip?.properties?.text && <span>{clip?.properties?.text}</span>}
+              {!clip.properties?.transcript && clip?.properties?.text && <span>{clip?.properties?.text}</span>}
+              {clip.properties?.transcript && <Viewer transcriptString={clip.properties?.transcript} />}
             </TranscriptWrapper>
           )
         }