Fixed retrying errors change to promise.all

google-index-checker.js

@@ -21,14 +21,25 @@ const apiKey = process.env.SCRAPERAPI_KEY
 
 let count = 1
 let notIndexedCounter = 0
-let urls = 0
+let urls = []
+let errors = []
+let len = 0
 
 // Collect URLS, get max Concurrent and run request in pool
 ;(async () => {
   urls = await getUrls()
+  len = urls.length
   const maxConcurrent = await getConcurrent()
-  await batchRequest(maxConcurrent, urls, runRequest, 'timeout')
-  finalMessage(urls.length)
+
+  await batchRequest(maxConcurrent, urls, runRequest)
+
+  while (errors.length) {
+    urls = errors
+    errors = []
+    await batchRequest(maxConcurrent, urls, runRequest)
+  }
+
+  finalMessage(len)
 })()
 
 // Gather URLS from file
@@ -74,7 +85,7 @@ async function runRequest(url) {
     const indexation = matchResponse(url, data)
 
     // Print to terminal each url, its number and status code
-    const counter = `${count++}/${urls.length}`
+    const counter = `${count++}/${len}`
     const statusPrint = green.bold(status)
     const indexPrint = white.bold(indexation)
 
@@ -96,12 +107,10 @@ async function runRequest(url) {
       process.exit(1)
     }
 
-    // Log with different color to highlight the error
-    console.error(yellow(`Error: ${url} ${red(status)} ${green('Re-trying')}`))
-
-    throw {
-      name: 'timeout',
-      error: url
+    if ([500, 501, 502, 503, 504].includes(status)) {
+      // Log with different color to highlight the error
+      console.error(yellow(`Error: ${url} ${red(status)} ${green('Re-trying')}`))
+      errors.push(url)
     }
   }
 }

lib/batchRequest.js

@@ -1,13 +1,10 @@
 // Pool request // https://www.npmjs.com/package/tiny-async-pool
-export async function batchRequest(poolLimit, array, iteratorFn, exception) {
+export async function batchRequest(poolLimit, array, iteratorFn) {
   const promises = []
   const racers = new Set()
 
-  // For each item create promises
   for (const item of array) {
-    // Start with creating an iteratorFn promise for each item
     const pro = Promise.resolve().then(() => iteratorFn(item, array))
-    // Store the promises
     promises.push(pro)
     racers.add(pro)
     const clean = () => racers.delete(pro)
@@ -17,14 +14,5 @@ export async function batchRequest(poolLimit, array, iteratorFn, exception) {
     if (racers.size >= poolLimit) await Promise.race(racers)
   }
 
-  const results = await Promise.allSettled(promises)
-
-  // Collect errors rejected by iteratorFn,
-  const rejected = results
-    .filter(({ status, reason }) => status === 'rejected' && reason.name === exception)
-    .map(({ reason }) => reason.error)
-
-  if (rejected.length) {
-    await poolRequest(poolLimit, rejected, iteratorFn, exception)
-  }
+  await Promise.all(promises)
 }