Future Proof with NextJs Scraping

src/api/v1/functions/verse.ts

@@ -18,7 +18,7 @@ export const getVerse = async (book: string, chapter: string, verses: string, ve
     }
 
     let bookFinder = bookList.books.find((o: bookType) => o.book.toLowerCase() === book.toLowerCase()) || bookList.books.find((o: bookType) => o.aliases.includes(book.toUpperCase()));
-    if (!bookFinder) return {code: 400, message: `Could not find book '${book}' by name or alias.`}
+    if (!bookFinder) return { code: 400, message: `Could not find book '${book}' by name or alias.` }
 
     let URL = `${baseURL}/${versionFinder.id}/${bookFinder.aliases[0]}.${chapter}.${verses}`;
 
@@ -29,20 +29,35 @@ export const getVerse = async (book: string, chapter: string, verses: string, ve
         const unavailable = $("p:contains('No Available Verses')").text();
         if (unavailable) return { code: 400, message: "Verse not found" };
 
-        const versesArray: Array<String> = [];
-        const wrapper = $(".text-17");
-
-        await wrapper.each((i, p) => {
-            let unformattedVerse = $(p).eq(0).text();
-            let formattedVerse = unformattedVerse.replace(/\n/g, ' ');
-            versesArray.push(formattedVerse)
-        })
-
-        return {
-            citation: `${bookFinder.book} ${chapter}:${verses}`,
-            passage: versesArray[0]
+        // Nextjs way :)
+        const nextWay = $("script#__NEXT_DATA__").eq(0);
+        if (nextWay) {
+            let json = JSON.parse(nextWay.html() || "");
+            const verse = json.props.pageProps.verses[0].content;
+            const reference = json.props.pageProps.verses[0].reference.human
+
+            return {
+                citation: `${reference}`,
+                passage: verse,
+            }
+        }
+        // Old way :(
+        else {
+            const versesArray: Array<String> = [];
+            const wrapper = $(".text-17");
+
+            await wrapper.each((i, p) => {
+                let unformattedVerse = $(p).eq(0).text();
+                let formattedVerse = unformattedVerse.replace(/\n/g, ' ');
+                versesArray.push(formattedVerse)
+            })
+
+            return {
+                citation: `${bookFinder.book} ${chapter}:${verses}`,
+                passage: versesArray[0]
+            }
         }
     } catch (err) {
         console.error(err);
     }
-}
+}

src/api/v1/functions/votd.ts

@@ -1,42 +1,79 @@
 import axios from 'axios';
 import * as cheerio from 'cheerio';
+import { version } from 'os';
 
 const URL = "https://www.bible.com/verse-of-the-day";
 export const getVotd = async () => {
     try {
         const { data } = await axios.get(URL);
         const $ = cheerio.load(data);
 
-        const versesArray: Array<String> = [];
-        const citationsArray: Array<String> = [];
         const imageArray: Array<String> = [];
 
-        const verses = $("a.text-text-light.w-full.no-underline");
-        const citations = $("p.text-gray-25");
-        const images = $("a.block");
-
-        await citations.each((i, p) => {
-            let citation = $(p).eq(0).text();
-            citationsArray.push(citation)
-        })
-
-        await verses.each((i, p) => {
-            let unformattedVerse = $(p).eq(0).text();
-            let formattedVerse = unformattedVerse.replace(/\n/g, ' ');
-            versesArray.push(formattedVerse)
-        })
-
-        await images.each((i, p) => {
-            let image = `https://www.bible.com${$(p).find('img').attr()?.src}`
-            imageArray.push(image);
-        })
-
-        return {
-            citation: citationsArray[0],
-            passage: versesArray[0],
-            image: imageArray ?? []
+        // Nextjs way :)
+        const nextWay = $("script#__NEXT_DATA__").eq(0);
+        if (nextWay != null) {
+            let json = JSON.parse(nextWay.html() || "");
+            const verse = json.props.pageProps.verses[0].content;
+            const reference = json.props.pageProps.verses[0].reference.human;
+            const version = json.props.pageProps.versionData.abbreviation;
+            const images = json.props.pageProps.arrayOfVerses[0].images.images;
+
+            await images.forEach((i: any) => {
+                if (i.usfm) {
+                    let image = `https://www.bible.com/_next/image?url=https:${(i.renditions[3].url as string)}&w=640&q=75`;
+                    imageArray.push(image);
+                }
+            });
+
+            return {
+                citation: `${reference}`,
+                passage: verse,
+                images: imageArray ?? [],
+                version: version
+            }
+        }
+        // Old way :(
+        else {
+            const versesArray: Array<String> = [];
+            const citationsArray: Array<String> = [];
+            let version;
+
+            const verses = $("a.text-text-light.w-full.no-underline");
+            const citations = $("p.text-gray-25");
+            const images = $("a.block");
+
+            await citations.each((i, p) => {
+                let citation = $(p).eq(0).text();
+
+                // cut the ending (ESV), (NIV), etc and store it in version
+                version = citation.slice(-4).replace(/[()]/g, '');
+
+                // cut the version from the citation
+                citation = citation.slice(0, -6);
+
+                citationsArray.push(citation)
+            })
+
+            await verses.each((i, p) => {
+                let unformattedVerse = $(p).eq(0).text();
+                let formattedVerse = unformattedVerse.replace(/\n/g, ' ');
+                versesArray.push(formattedVerse)
+            })
+
+            await images.each((i, p) => {
+                let image = `https://www.bible.com${$(p).find('img').attr()?.src}`
+                imageArray.push(image);
+            })
+
+            return {
+                citation: citationsArray[0],
+                passage: versesArray[0],
+                image: imageArray ?? [],
+                version: version
+            }
         }
     } catch (err) {
         console.error(err);
     }
-}
+}