fix(add-content): extend webpage check for RSS feeds

stakwork · Sep 18, 2024 · 7fdc308 · 7fdc308
1 parent 7ffb3a9
commit 7fdc308
Show file tree

Hide file tree

Showing 3 changed files with 60 additions and 23 deletions.
diff --git a/src/components/AddContentModal/index.tsx b/src/components/AddContentModal/index.tsx
@@ -172,7 +172,13 @@ export const AddContentModal = () => {
   const isValidSource = validateSourceURL(sourceValue)
 
   useEffect(() => {
-    setValue('inputType', getInputType(source))
+    const updateInputType = async () => {
+      const inputType = await getInputType(source)
+
+      setValue('inputType', inputType)
+    }
+
+    updateInputType()
   }, [source, setValue])
 
   const handleClose = () => {

diff --git a/src/components/AddContentModal/utils/__tests__/index.ts b/src/components/AddContentModal/utils/__tests__/index.ts
@@ -3,63 +3,67 @@ import { extractNameFromLink, getInputType } from '..'
 
 describe('youtubeRegex', () => {
   it('should assert we can check for youtube clip regex', async () => {
-    expect(getInputType('https://www.youtube.com/watch?v=83eQ9flwVS0&ab_channel=EthanChlebowski')).toBe(LINK)
+    await expect(getInputType('https://www.youtube.com/watch?v=83eQ9flwVS0&ab_channel=EthanChlebowski')).resolves.toBe(
+      LINK,
+    )
   })
 
   it('should assert we can check for youtube live clip regex', async () => {
-    expect(getInputType('https://youtube.com/live/tkdMgjEFNWs')).toBe(LINK)
+    await expect(getInputType('https://youtube.com/live/tkdMgjEFNWs')).resolves.toBe(LINK)
   })
 
   it('should assert we can check for twitter spaces regex', async () => {
-    expect(getInputType('https://twitter.com/i/spaces/1zqKVqwrVzlxB?s=20')).toBe(LINK)
+    await expect(getInputType('https://twitter.com/i/spaces/1zqKVqwrVzlxB?s=20')).resolves.toBe(LINK)
   })
 
   it('should assert we can check for youtu.be link regex', async () => {
-    expect(getInputType('https://youtu.be/HfMYOeg79dM')).toBe(LINK)
+    await expect(getInputType('https://youtu.be/HfMYOeg79dM')).resolves.toBe(LINK)
   })
 
   it('should assert we can check for youtu.be link with parameters regex', async () => {
-    expect(getInputType('https://youtu.be/HfMYOeg79dM?t=120')).toBe(LINK)
+    await expect(getInputType('https://youtu.be/HfMYOeg79dM?t=120')).resolves.toBe(LINK)
   })
 
   it('should assert we can check for twitter tweet regex', async () => {
-    expect(getInputType('https://twitter.com/LarryRuane/status/1720496960489095668')).toBe(TWITTER_SOURCE)
+    await expect(getInputType('https://twitter.com/LarryRuane/status/1720496960489095668')).resolves.toBe(
+      TWITTER_SOURCE,
+    )
   })
 
   it('should assert we can check for x.com tweet regex', async () => {
-    expect(getInputType('https://x.com/bernaaaljg/status/1795260855002583101')).toBe(TWITTER_SOURCE)
+    await expect(getInputType('https://x.com/bernaaaljg/status/1795260855002583101')).resolves.toBe(TWITTER_SOURCE)
   })
 
   it('should assert we can check for mp3 url regex', async () => {
-    expect(getInputType('https://hahaha.com/i/spaces/1zqKVqwrVzlxB?s=20.mp3')).toBe(LINK)
+    await expect(getInputType('https://hahaha.com/i/spaces/1zqKVqwrVzlxB?s=20.mp3')).resolves.toBe(LINK)
   })
 
-  it('should assert we can check for Twitter broadcast regex', () => {
-    expect(getInputType('https://twitter.com/i/broadcasts/1YqxoDbOqevKv')).toBe(LINK)
+  it('should assert we can check for Twitter broadcast regex', async () => {
+    await expect(getInputType('https://twitter.com/i/broadcasts/1YqxoDbOqevKv')).resolves.toBe(LINK)
   })
 
   it('should assert we can check for generic url regex', async () => {
-    expect(getInputType('https://idkwhat.com/routeing/tou')).toBe(WEB_PAGE)
+    await expect(getInputType('https://idkwhat.com/routeing/tou')).resolves.toBe(WEB_PAGE)
   })
 
   it('should assert we can check for youtube clip regex', async () => {
-    expect(getInputType('https://www.youtube.com/@MrBeast')).toBe(YOUTUBE_CHANNEL)
+    await expect(getInputType('https://www.youtube.com/@MrBeast')).resolves.toBe(YOUTUBE_CHANNEL)
   })
 
   it('should assert we can check for twitter handle regex', async () => {
-    expect(getInputType('https://twitter.com/@KevKevPal')).toBe(TWITTER_HANDLE)
+    await expect(getInputType('https://twitter.com/@KevKevPal')).resolves.toBe(TWITTER_HANDLE)
   })
 
   it('should assert we can check for x.com handle regex', async () => {
-    expect(getInputType('https://x.com/@KevKevPal')).toBe(TWITTER_HANDLE)
+    await expect(getInputType('https://x.com/@KevKevPal')).resolves.toBe(TWITTER_HANDLE)
   })
 
   it('should assert we can check for youtube live clip regex', async () => {
-    expect(getInputType('https://www.youtube.com/@MrBeast')).toBe(YOUTUBE_CHANNEL)
+    await expect(getInputType('https://www.youtube.com/@MrBeast')).resolves.toBe(YOUTUBE_CHANNEL)
   })
 
   it('should assert we can check for document regex', async () => {
-    expect(getInputType('some plain text')).toBe(DOCUMENT)
+    await expect(getInputType('some plain text')).resolves.toBe(DOCUMENT)
   })
 })
 
@@ -102,9 +106,23 @@ describe('extractNameFromLink', () => {
 
 describe('getInputType', () => {
   it('should assert we can check for RSS feed url regex', async () => {
-    expect(getInputType('http://example.com/feed')).toBe(RSS)
-    expect(getInputType('http://example.com/rss')).toBe(RSS)
-    expect(getInputType('http://example.com/rss.xml')).toBe(RSS)
-    expect(getInputType('http://example.com/?feed=rss')).toBe(RSS)
+    await expect(getInputType('http://example.com/feed')).resolves.toBe(RSS)
+    await expect(getInputType('http://example.com/rss')).resolves.toBe(RSS)
+    await expect(getInputType('http://example.com/rss.xml')).resolves.toBe(RSS)
+    await expect(getInputType('http://example.com/?feed=rss')).resolves.toBe(RSS)
+  })
+
+  it('should assert we can check for RSS feed by content type', async () => {
+    global.fetch = jest.fn(() =>
+      Promise.resolve({
+        headers: {
+          get: (header) => (header === 'Content-Type' ? 'application/rss+xml' : null),
+        },
+      }),
+    ) as jest.Mock
+
+    expect(await getInputType('https://rss.arxiv.org/rss/cs.AI')).toBe(RSS)
+
+    jest.restoreAllMocks()
   })
 })
diff --git a/src/components/AddContentModal/utils/index.ts b/src/components/AddContentModal/utils/index.ts
@@ -15,7 +15,18 @@ const youtubeChannelPattern = /https?:\/\/(www\.)?youtube\.com\/(user\/)?(@)?([\
 const genericUrlRegex = /^(https?|ftp):\/\/[^\s/$.?#].[^\s]*$/
 const twitterBroadcastRegex = /https:\/\/twitter\.com\/i\/broadcasts\/([A-Za-z0-9_-]+)/
 
-export function getInputType(source: string) {
+export async function checkIfRSS(url: string): Promise<boolean> {
+  try {
+    const response = await fetch(url, { method: 'HEAD' })
+    const contentType = response.headers.get('Content-Type')
+
+    return contentType?.includes('application/rss+xml') ?? false
+  } catch (error) {
+    return false
+  }
+}
+
+export async function getInputType(source: string) {
   const linkPatterns = [
     youtubeLiveRegex,
     twitterBroadcastRegex,
@@ -46,7 +57,9 @@ export function getInputType(source: string) {
   }
 
   if (genericUrlRegex.test(source)) {
-    return WEB_PAGE
+    const isRSS = await checkIfRSS(source)
+
+    return isRSS ? RSS : WEB_PAGE
   }
 
   return DOCUMENT