Update link-checker.js

jacoblogan · Aug 31, 2023 · 0044f65 · 0044f65
1 parent 4264c8a
commit 0044f65
Showing 1 changed file with 31 additions and 27 deletions.
diff --git a/tasks/link-checker.js b/tasks/link-checker.js
@@ -52,34 +52,38 @@ const retrieveLinks = async (siteMapUrls, visitedLinks) => {
   for (let i = 0; i < siteMapUrls.length; i++) {
     let url = siteMapUrls[i];
 
-    let response = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 2 });
-    await page.waitForNetworkIdle();
-    if (response && response.status() && response.status() === 200) {
-      console.log(`successfully visited ${url} to retrieve links`);
-      visitedLinks[url] = true;
-
-      const urlList = await page.evaluate(async (url) => {
-        let urls = [];
-        let elements = document.getElementsByTagName('a');
-        for (let i = 0; i < elements.length; i++) {
-          let element = elements[i];
-          if (element.href) {
-            const link = {
-              url: element.href,
-              parentUrl: url,
-              linkText: element.textContent
-            };
-            urls.push(link);
+    try{
+      let response = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 2 });
+      await page.waitForNetworkIdle();
+      if (response && response.status() && response.status() === 200) {
+        console.log(`successfully visited ${url} to retrieve links`);
+        visitedLinks[url] = true;
+
+        const urlList = await page.evaluate(async (url) => {
+          let urls = [];
+          let elements = document.getElementsByTagName('a');
+          for (let i = 0; i < elements.length; i++) {
+            let element = elements[i];
+            if (element.href) {
+              const link = {
+                url: element.href,
+                parentUrl: url,
+                linkText: element.textContent
+              };
+              urls.push(link);
+            }
           }
-        }
-        return urls;
-      }, url);
-
-      urlList.forEach((link) => {
-        if (!CRAWLER_EXCEPTIONS.includes(link.url)) {
-          urlsToVisit.push(link);
-        }
-      });
+          return urls;
+        }, url);
+
+        urlList.forEach((link) => {
+          if (!CRAWLER_EXCEPTIONS.includes(link.url)) {
+            urlsToVisit.push(link);
+          }
+        });
+      }
+    } catch (e) {
+      console.log(`attempt to retrieve ${url} has failed: ${e}`);
     }
   }