add handle to 429 error

node/middlewares/generateMiddlewares/generateProductRoutes.ts

@@ -21,6 +21,7 @@ import {
   Message,
   PRODUCT_ROUTES_INDEX,
   RAW_DATA_PREFIX,
+  retryOn429,
   SitemapEntry,
   SitemapIndex,
   slugify,
@@ -184,14 +185,24 @@ const saveRoutes = (
   )
   index.push(entry)
   const lastUpdated = currentDate()
-  await vbase.saveJSON<SitemapEntry>(bucket, entry, {
-    lastUpdated,
-    routes,
-  })
-  await vbase.saveJSON<SitemapIndex>(bucket, PRODUCT_ROUTES_INDEX, {
-    index,
-    lastUpdated,
-  })
+
+  try {
+    await retryOn429(() =>
+      vbase.saveJSON<SitemapEntry>(bucket, entry, {
+        lastUpdated,
+        routes,
+      })
+    )
+
+    await retryOn429(() =>
+      vbase.saveJSON<SitemapIndex>(bucket, PRODUCT_ROUTES_INDEX, {
+        index,
+        lastUpdated,
+      })
+    )
+  } catch (error) {
+    console.error(`Failed to save data after retries: ${error.message}`)
+  }
 }
 
 const pageWasProcessed = async (

node/middlewares/generateMiddlewares/utils.ts

@@ -154,3 +154,26 @@ export const cleanConfigBucket = async (enabledIndexFiles: string[], vbase: VBas
   ])
 
 const allTruthy = <T>(array: T[]) => !array.some(e => !e)
+
+const delay = (ms: number) => new Promise(res => setTimeout(res, ms))
+
+export const retryOn429 = async (
+  fn: () => Promise<unknown>,
+  maxRetries = 3,
+  attempt = 0
+): Promise<void> => {
+  try {
+    await fn()
+  } catch (error) {
+    if (error.status === 429 && attempt < maxRetries) {
+      const waitTime = 2 ** (attempt + 1) * 1000
+      console.warn(
+        `HTTP 429 received, retrying in ${waitTime / 1000} seconds...`
+      )
+      await delay(waitTime)
+      return retryOn429(fn, maxRetries, attempt + 1)
+    }
+
+    throw error
+  }
+}