Merge pull request #2490 from stakwork/feature/scrapper

feat: add scrapper

src/components/mindset/components/LandingPage/index.tsx

@@ -1,11 +1,13 @@
-import { useState } from 'react'
+import { useEffect, useState } from 'react'
 import { FieldValues } from 'react-hook-form'
 import styled from 'styled-components'
 import { Flex } from '~/components/common/Flex'
 import { NODE_ADD_ERROR } from '~/constants'
 import { api } from '~/network/api'
+import { getSchemaAll } from '~/network/fetchSourcesData'
 import { useDataStore } from '~/stores/useDataStore'
 import { useMindsetStore } from '~/stores/useMindsetStore'
+import { useSchemaStore } from '~/stores/useSchemaStore'
 import { SubmitErrRes } from '~/types'
 import { colors } from '~/utils/colors'
 import { ChevronRight } from '../Icon/ChevronRight'
@@ -44,6 +46,21 @@ export const LandingPage = () => {
   const [requestError, setRequestError] = useState<string>('')
   const { setRunningProjectId } = useDataStore((s) => s)
   const { setSelectedEpisodeId, setSelectedEpisodeLink } = useMindsetStore((s) => s)
+  const { setSchemas } = useSchemaStore((s) => s)
+
+  useEffect(() => {
+    const fetchSchemaData = async () => {
+      try {
+        const response = await getSchemaAll()
+
+        setSchemas(response.schemas.filter((schema) => !schema.is_deleted))
+      } catch (err) {
+        console.error('Error fetching schema:', err)
+      }
+    }
+
+    fetchSchemaData()
+  }, [setSchemas])
 
   const handleInputChange = (e: React.ChangeEvent<HTMLInputElement>) => {
     const { value } = e.target

src/components/mindset/components/Marker/index.tsx

@@ -1,13 +1,11 @@
 import { memo } from 'react'
 import styled from 'styled-components'
 import { Flex } from '~/components/common/Flex'
-import { Tooltip } from '~/components/common/ToolTip'
 import { useSchemaStore } from '~/stores/useSchemaStore'
 import { colors } from '~/utils/colors'
 
 type Props = {
   type: string
-  name: string
   left: number
   img: string
 }
@@ -18,24 +16,22 @@ type BadgeProps = {
   label: string
 }
 
-export const Marker = memo(({ type, name, left, img }: Props) => {
+export const Marker = memo(({ type, left, img }: Props) => {
   const [normalizedSchemasByType] = useSchemaStore((s) => [s.normalizedSchemasByType])
 
   const primaryColor = normalizedSchemasByType[type]?.primary_color
   const primaryIcon = normalizedSchemasByType[type]?.icon
 
-  const icon = primaryIcon ? `svg-icons/${primaryIcon}.svg` : 'thing_badge.svg'
+  const icon = primaryIcon ? `svg-icons/${primaryIcon}.svg` : ''
 
   const badgeProps: Omit<BadgeProps, 'label'> = {
-    iconStart: img ?? icon,
+    iconStart: img || icon,
     color: primaryColor ?? colors.THING,
   }
 
   return (
     <MarkerWrapper style={{ left: `${left}%` }}>
-      <Tooltip content={`${name || type}`}>
-        <Badge {...badgeProps} label={type} />
-      </Tooltip>
+      <Badge {...badgeProps} label={type} />
     </MarkerWrapper>
   )
 })
@@ -44,7 +40,7 @@ Marker.displayName = 'Marker'
 
 const Badge = ({ iconStart, color, label }: BadgeProps) => (
   <EpisodeWrapper color={color}>
-    <img alt={label} className="badge__img" src={iconStart} />
+    {iconStart && <img alt={label} className="badge__img" src={iconStart} />}
   </EpisodeWrapper>
 )
 
@@ -57,7 +53,6 @@ const EpisodeWrapper = styled(Flex).attrs({
   overflow: hidden;
   justify-content: center;
   align-items: center;
-  padding: 0 4px;
 
   .badge__img {
     width: 10px;
@@ -68,12 +63,12 @@ const EpisodeWrapper = styled(Flex).attrs({
 
 const MarkerWrapper = styled.div`
   position: absolute;
-  top: -4px; /* Adjust as needed to center above the progress bar */
-  width: 8px;
-  height: 8px;
+  top: -6px;
+  width: 12px;
+  height: 12px;
+  border-radius: 4px;
   background-color: ${colors.white};
-  border-radius: 50%;
-  transform: translateX(-50%); /* Center the marker horizontally */
+  transform: translateX(-50%);
   transform: translateX(-50%) translateY(-50%);
   top: 50%;
   display: flex;

src/components/mindset/components/PlayerContols/ProgressBar/index.tsx

@@ -12,26 +12,29 @@ type Props = {
   handleProgressChange: (_: Event, value: number | number[]) => void
 }
 
-export const ProgressBar = ({ duration, markers, handleProgressChange, playingTIme }: Props) => (
-  <ProgressWrapper>
-    <ProgressSlider max={duration} onChange={handleProgressChange} value={playingTIme} />
-    {markers.map((node) => {
-      const position = ((node?.start || 0) / duration) * 100
-      const type = node?.node_type || ''
-      const name = node?.properties?.name || ''
-      const img = node?.properties?.image_url || ''
+export const ProgressBar = ({ duration, markers, handleProgressChange, playingTIme }: Props) => {
+  const thumbWidth = (10 / duration) * 100
 
-      return <Marker key={node.ref_id} img={img} left={position} name={name} type={type} />
-    })}
-  </ProgressWrapper>
-)
+  return (
+    <ProgressWrapper>
+      <ProgressSlider max={duration} onChange={handleProgressChange} thumbWidth={thumbWidth} value={playingTIme} />
+      {markers.map((node) => {
+        const position = ((node?.start || 0) / duration) * 100
+        const type = node?.node_type || ''
+        const img = node?.properties?.image_url || ''
+
+        return <Marker key={node.ref_id} img={img} left={position} type={type} />
+      })}
+    </ProgressWrapper>
+  )
+}
 
 const ProgressWrapper = styled(Flex)`
   position: relative;
   flex: 1 1 100%;
 `
 
-const ProgressSlider = styled(Slider)`
+const ProgressSlider = styled(Slider)<{ thumbWidth: number }>`
   && {
     z-index: 20;
     color: ${colors.white};
@@ -42,9 +45,11 @@ const ProgressSlider = styled(Slider)`
       border: none;
     }
     .MuiSlider-thumb {
-      width: 10px;
-      height: 10px;
-      background-color: ${colors.white};
+      width: ${({ thumbWidth }) => `${thumbWidth}%`};
+      height: 54px;
+      border-radius: 8px;
+      background-color: ${colors.primaryBlue};
+      opacity: 0.2;
       &:before {
         box-shadow: '0 4px 8px rgba(0,0,0,0.4)';
       }

src/components/mindset/components/Sidebar/Transcript/Viewer/index.tsx

@@ -3,6 +3,7 @@ import { useEffect, useState } from 'react'
 import styled from 'styled-components'
 import { usePlayerStore } from '~/stores/usePlayerStore'
 import { colors } from '~/utils'
+import { secondsToMediaTime } from '~/utils/secondsToMediaTime'
 
 type Props = {
   transcriptString: string
@@ -51,11 +52,11 @@ export const Viewer = ({ transcriptString }: Props) => {
   return (
     <Wrapper>
       {transcriptData.map((i) => {
-        const start = i.start.toFixed(2)
-        const end = i.end.toFixed(2)
+        const start = secondsToMediaTime(i.start)
+        const end = secondsToMediaTime(i.end)
 
         return (
-          <Paragraph key={i.id}>
+          <Paragraph key={i.start}>
             <Start>
               {start}:{end}
             </Start>
@@ -82,6 +83,9 @@ const Paragraph = styled.div`
 
 const Wrapper = styled.div`
   width: 100%;
+  height: 100%;
+  overflow: scroll;
+  padding-right: 4px;
 `
 
 const Start = styled.span``

src/network/fetchGraphData/const.ts

@@ -1,31 +1,6 @@
-import { GraphStyle } from '~/stores/useDataStore'
-import { generateEarthGraphPositions } from '~/transformers/earthGraph'
-import { generateForceGraphPositions } from '~/transformers/forceGraph'
-import { generateSphereGraphPositions } from '~/transformers/sphereGraph'
-import { generateSplitGraphPositions } from '~/transformers/splitGraph'
-import { GraphData, Link, NodeExtended } from '~/types'
-
-export const shouldIncludeTopics = true
-export const maxScale = 26
+import { GraphData } from '~/types'
 
 export const defaultData: GraphData = {
   links: [],
   nodes: [],
 }
-
-export const getGraphDataPositions = (graphStyle: GraphStyle, nodes: NodeExtended[], links: Link[]) => {
-  // give nodes and links positions based on graphStyle
-  if (graphStyle === 'split') {
-    return generateSplitGraphPositions(nodes, links)
-  }
-
-  if (graphStyle === 'sphere') {
-    return generateSphereGraphPositions(nodes)
-  }
-
-  if (graphStyle === 'earth') {
-    return generateEarthGraphPositions(nodes)
-  }
-
-  return generateForceGraphPositions(nodes, links)
-}