fix: dont use missing/stale related asset data when generating the related asset index

scripts/generateAssetData/generateRelatedAssetIndex/generateRelatedAssetIndex.ts

@@ -123,28 +123,20 @@ const getZerionRelatedAssetIds = async (
   if (firstEntry === undefined) return
 
   const implementations = firstEntry.attributes.implementations
-  const primaryImplementationId = firstEntry.id
 
-  const primaryImplementation = implementations?.find(
-    implementation => implementation.address === primaryImplementationId,
-  )
-
-  const relatedAssetKey = primaryImplementation
-    ? zerionImplementationToMaybeAssetId(primaryImplementation)
-    : undefined
-
-  const relatedAssetIds = implementations
+  // Use all assetIds actually present in the dataset
+  const allRelatedAssetIds = implementations
     ?.map(zerionImplementationToMaybeAssetId)
     .filter(isSome)
-    .filter(
-      relatedAssetId =>
-        relatedAssetId !== relatedAssetKey && assetData[relatedAssetId] !== undefined,
-    )
+    .filter(relatedAssetId => assetData[relatedAssetId] !== undefined)
 
-  if (!relatedAssetKey || !relatedAssetIds || relatedAssetIds.length === 0) {
+  if (!allRelatedAssetIds || allRelatedAssetIds.length <= 1) {
     return
   }
 
+  const relatedAssetKey = allRelatedAssetIds[0]
+  const relatedAssetIds = allRelatedAssetIds.filter(assetId => assetId !== relatedAssetKey)
+
   return { relatedAssetIds, relatedAssetKey }
 }
 
@@ -161,24 +153,20 @@ const getCoingeckoRelatedAssetIds = async (
   const { data } = await axios.get<CoingeckoAssetDetails>(`${coingeckoBaseUrl}/coins/${coinUri}`)
 
   const platforms = data.platforms
-  const primaryPlatform = Object.entries(data.platforms)[0]
 
-  const relatedAssetKey = primaryPlatform
-    ? coingeckoPlatformDetailsToMaybeAssetId(primaryPlatform)
-    : undefined
-
-  const relatedAssetIds = Object.entries(platforms)
+  // Use all assetIds actually present in the dataset
+  const allRelatedAssetIds = Object.entries(platforms)
     ?.map(coingeckoPlatformDetailsToMaybeAssetId)
     .filter(isSome)
-    .filter(
-      relatedAssetId =>
-        relatedAssetId !== relatedAssetKey && assetData[relatedAssetId] !== undefined,
-    )
+    .filter(relatedAssetId => assetData[relatedAssetId] !== undefined)
 
-  if (!relatedAssetKey || !relatedAssetIds || relatedAssetIds.length === 0) {
+  if (allRelatedAssetIds.length <= 1) {
     return
   }
 
+  const relatedAssetKey = allRelatedAssetIds[0]
+  const relatedAssetIds = allRelatedAssetIds.filter(assetId => assetId !== relatedAssetKey)
+
   return { relatedAssetIds, relatedAssetKey }
 }
 
@@ -222,9 +210,11 @@ const processRelatedAssetIds = async (
     })
 
   const zerionRelatedAssetsResult = await getZerionRelatedAssetIds(
-    // DO NOT REMOVE ME - reuse the relatedAssetKey if found with coingecko fetch. cg may not have all related assetIds for a given asset, and Zerion may not have
-    // any at all the same asset. e.g USDC.SOL is found on Coingecko but with only USDC.ETH as a relatedAssetId, and is not present at all under the USDC.SOL umbrella in Zerion.
-    // Using the primary implementation ensures we use a reliable identifier for the related assets, not a more obscure one.
+    // DO NOT REMOVE ME - reuse the relatedAssetKey if found with coingecko fetch. cg may not have
+    // all related assetIds for a given asset, and Zerion may not have any at all the same asset.
+    // e.g USDC.SOL is found on Coingecko but with only USDC.ETH as a relatedAssetId, and is not
+    // present at all under the USDC.SOL umbrella in Zerion. Using the primary implementation
+    // ensures we use a reliable identifier for the related assets, not a more obscure one.
     coingeckoRelatedAssetsResult?.relatedAssetKey ?? assetId,
     assetData,
   )
@@ -299,14 +289,31 @@ export const generateRelatedAssetIndex = async (rebuildAll: boolean = false) =>
   const relatedAssetIndex: Record<AssetId, AssetId[]> = JSON.parse(
     await fs.promises.readFile(relatedAssetIndexPath, 'utf8'),
   )
-  const assetDataWithRelatedAssetKeys: Record<AssetId, PartialFields<Asset, 'relatedAssetKey'>> = {
-    ...generatedAssetData,
-  }
 
-  if (rebuildAll) {
+  // Remove stale related asset data from the assetData where:
+  // a) rebuildAll is set
+  // b) the primary related asset no longer exists in the dataset
+  Object.values(generatedAssetData).forEach(asset => {
+    const relatedAssetKey = asset.relatedAssetKey
+
+    if (!relatedAssetKey) return
+
+    const primaryRelatedAsset = generatedAssetData[relatedAssetKey]
+
     // remove relatedAssetKey from the existing data to ensure the related assets get updated
-    Object.values(assetDataWithRelatedAssetKeys).forEach(asset => delete asset.relatedAssetKey)
-  }
+    if (rebuildAll || primaryRelatedAsset === undefined) {
+      delete relatedAssetIndex[relatedAssetKey]
+      delete asset.relatedAssetKey
+    }
+  })
+
+  // Remove stale related asset data from the relatedAssetIndex where:
+  // a) a related assetId no longer exists in the dataset
+  Object.entries(relatedAssetIndex).forEach(([relatedAssetKey, relatedAssetIds]) => {
+    relatedAssetIndex[relatedAssetKey] = relatedAssetIds.filter(
+      assetId => generatedAssetData[assetId] !== undefined,
+    )
+  })
 
   const { throttle, clear: clearThrottleInterval } = createThrottle({
     capacity: 50, // Reduced initial capacity to allow for a burst but not too high
@@ -319,12 +326,7 @@ export const generateRelatedAssetIndex = async (rebuildAll: boolean = false) =>
     console.log(`Processing chunk: ${i} of ${chunks.length}`)
     await Promise.all(
       batch.map(async assetId => {
-        await processRelatedAssetIds(
-          assetId,
-          assetDataWithRelatedAssetKeys,
-          relatedAssetIndex,
-          throttle,
-        )
+        await processRelatedAssetIds(assetId, generatedAssetData, relatedAssetIndex, throttle)
         return
       }),
     )
@@ -335,7 +337,7 @@ export const generateRelatedAssetIndex = async (rebuildAll: boolean = false) =>
   await fs.promises.writeFile(
     generatedAssetsPath,
     // beautify the file for github diff.
-    JSON.stringify(assetDataWithRelatedAssetKeys, null, 2),
+    JSON.stringify(generatedAssetData, null, 2),
   )
 
   await fs.promises.writeFile(

src/lib/asset-service/service/generatedAssetData.json

@@ -9977,7 +9977,7 @@
     "explorer": "https://etherscan.io",
     "explorerAddressLink": "https://etherscan.io/address/",
     "explorerTxLink": "https://etherscan.io/tx/",
-    "relatedAssetKey": "eip155:56/bep20:0x16594930d16f3970e1a4317c6016555cb2e7b7fc"
+    "relatedAssetKey": null
   },
   "eip155:1/erc20:0x167478921b907422f8e88b43c4af2b8bea278d3a": {
     "explorer": "https://etherscan.io",
@@ -23608,7 +23608,7 @@
     "explorer": "https://etherscan.io",
     "explorerAddressLink": "https://etherscan.io/address/",
     "explorerTxLink": "https://etherscan.io/tx/",
-    "relatedAssetKey": "eip155:137/erc20:0x7d4d68f18d1be3410ab8d827fb7ebc690f938d2d"
+    "relatedAssetKey": null
   },
   "eip155:1/erc20:0x37a2f8701856a78de92dbe35df2200c355eae090": {
     "assetId": "eip155:1/erc20:0x37a2f8701856a78de92dbe35df2200c355eae090",
@@ -46195,7 +46195,7 @@
     "explorer": "https://etherscan.io",
     "explorerAddressLink": "https://etherscan.io/address/",
     "explorerTxLink": "https://etherscan.io/tx/",
-    "relatedAssetKey": "eip155:42161/erc20:0x6c249b6f6492864d914361308601a7abb32e68f8"
+    "relatedAssetKey": "eip155:43114/erc20:0xb418417374fca27bb54169d3c777492e6fe17ee7"
   },
   "eip155:1/erc20:0x6c280db098db673d30d5b34ec04b6387185d3620": {
     "explorer": "https://etherscan.io",
@@ -94078,7 +94078,7 @@
     "explorer": "https://etherscan.io",
     "explorerAddressLink": "https://etherscan.io/address/",
     "explorerTxLink": "https://etherscan.io/tx/",
-    "relatedAssetKey": "eip155:42161/erc20:0xde5ed76e7c05ec5e4572cfc88d1acea165109e44"
+    "relatedAssetKey": "eip155:1/erc20:0xde5ed76e7c05ec5e4572cfc88d1acea165109e44"
   },
   "eip155:1/erc20:0xde654f497a563dd7a121c176a125dd2f11f13a83": {
     "assetId": "eip155:1/erc20:0xde654f497a563dd7a121c176a125dd2f11f13a83",
@@ -123808,7 +123808,7 @@
     "explorer": "https://polygonscan.com/",
     "explorerAddressLink": "https://polygonscan.com/address/",
     "explorerTxLink": "https://polygonscan.com/tx/",
-    "relatedAssetKey": "eip155:1/erc20:0x9d0b65a76274645b29e4cc41b8f23081fa09f4a3"
+    "relatedAssetKey": "eip155:56/bep20:0x7bc75e291e656e8658d66be1cc8154a3769a35dd"
   },
   "eip155:137/erc20:0x7f792db54b0e580cdc755178443f0430cf799aca": {
     "assetId": "eip155:137/erc20:0x7f792db54b0e580cdc755178443f0430cf799aca",
@@ -130082,7 +130082,7 @@
     "explorer": "https://polygonscan.com/",
     "explorerAddressLink": "https://polygonscan.com/address/",
     "explorerTxLink": "https://polygonscan.com/tx/",
-    "relatedAssetKey": "eip155:42161/erc20:0xde5ed76e7c05ec5e4572cfc88d1acea165109e44"
+    "relatedAssetKey": "eip155:1/erc20:0xde5ed76e7c05ec5e4572cfc88d1acea165109e44"
   },
   "eip155:137/erc20:0xdf7837de1f2fa4631d716cf2502f8b230f1dcc32": {
     "assetId": "eip155:137/erc20:0xdf7837de1f2fa4631d716cf2502f8b230f1dcc32",
@@ -146484,7 +146484,7 @@
     "explorer": "https://arbiscan.io",
     "explorerAddressLink": "https://arbiscan.io/address/",
     "explorerTxLink": "https://arbiscan.io/tx/",
-    "relatedAssetKey": "eip155:1/erc20:0xb4371da53140417cbb3362055374b10d97e420bb"
+    "relatedAssetKey": "eip155:42161/erc20:0xf763fa322dc58dee588252fafee5f448e863b633"
   },
   "eip155:42161/erc20:0xf7693c6fd9a7172d537fa75d133d309501cbd657": {
     "assetId": "eip155:42161/erc20:0xf7693c6fd9a7172d537fa75d133d309501cbd657",
@@ -146549,7 +146549,7 @@
     "explorer": "https://arbiscan.io",
     "explorerAddressLink": "https://arbiscan.io/address/",
     "explorerTxLink": "https://arbiscan.io/tx/",
-    "relatedAssetKey": "eip155:1/erc20:0xdd50c053c096cb04a3e3362e2b622529ec5f2e8a"
+    "relatedAssetKey": null
   },
   "eip155:42161/erc20:0xf890360473c12d8015da8dbf7af11da87337a065": {
     "explorer": "https://arbiscan.io",
@@ -152692,7 +152692,7 @@
     "explorer": "https://snowtrace.dev",
     "explorerAddressLink": "https://snowtrace.dev/address/",
     "explorerTxLink": "https://snowtrace.dev/tx/",
-    "relatedAssetKey": "eip155:42161/erc20:0x6c249b6f6492864d914361308601a7abb32e68f8"
+    "relatedAssetKey": "eip155:43114/erc20:0xb418417374fca27bb54169d3c777492e6fe17ee7"
   },
   "eip155:43114/erc20:0xb44a9b6905af7c801311e8f4e76932ee959c663c": {
     "assetId": "eip155:43114/erc20:0xb44a9b6905af7c801311e8f4e76932ee959c663c",
@@ -166312,7 +166312,7 @@
     "explorer": "https://bscscan.com",
     "explorerAddressLink": "https://bscscan.com/address/",
     "explorerTxLink": "https://bscscan.com/tx/",
-    "relatedAssetKey": "eip155:1/erc20:0xcc1a8bd438bebc4b2a885a34475bb974f2124317"
+    "relatedAssetKey": null
   },
   "eip155:56/bep20:0x366d71ab095735b7dae83ce2b82d5262ef655f10": {
     "assetId": "eip155:56/bep20:0x366d71ab095735b7dae83ce2b82d5262ef655f10",
@@ -166882,7 +166882,7 @@
     "explorer": "https://bscscan.com",
     "explorerAddressLink": "https://bscscan.com/address/",
     "explorerTxLink": "https://bscscan.com/tx/",
-    "relatedAssetKey": "eip155:1/erc20:0xd9ec3ff1f8be459bb9369b4e79e9ebcf7141c093"
+    "relatedAssetKey": null
   },
   "eip155:56/bep20:0x39bff8613defd221b0410ed3d4e5c07512d55f2d": {
     "assetId": "eip155:56/bep20:0x39bff8613defd221b0410ed3d4e5c07512d55f2d",
@@ -180795,7 +180795,7 @@
     "explorer": "https://bscscan.com",
     "explorerAddressLink": "https://bscscan.com/address/",
     "explorerTxLink": "https://bscscan.com/tx/",
-    "relatedAssetKey": "eip155:1/erc20:0x9d0b65a76274645b29e4cc41b8f23081fa09f4a3"
+    "relatedAssetKey": "eip155:56/bep20:0x7bc75e291e656e8658d66be1cc8154a3769a35dd"
   },
   "eip155:56/bep20:0x7bd6fabd64813c48545c9c0e312a0099d9be2540": {
     "assetId": "eip155:56/bep20:0x7bd6fabd64813c48545c9c0e312a0099d9be2540",
@@ -192496,7 +192496,7 @@
     "explorer": "https://bscscan.com",
     "explorerAddressLink": "https://bscscan.com/address/",
     "explorerTxLink": "https://bscscan.com/tx/",
-    "relatedAssetKey": "eip155:1/erc20:0x9fc8f0ca1668e87294941b7f627e9c15ea06b459"
+    "relatedAssetKey": null
   },
   "eip155:56/bep20:0xb350aebaedb1ed3269b0e25d5e593a9bb4b9f9d5": {
     "assetId": "eip155:56/bep20:0xb350aebaedb1ed3269b0e25d5e593a9bb4b9f9d5",
@@ -195469,7 +195469,7 @@
     "explorer": "https://bscscan.com",
     "explorerAddressLink": "https://bscscan.com/address/",
     "explorerTxLink": "https://bscscan.com/tx/",
-    "relatedAssetKey": "eip155:1/erc20:0xb4371da53140417cbb3362055374b10d97e420bb"
+    "relatedAssetKey": "eip155:42161/erc20:0xf763fa322dc58dee588252fafee5f448e863b633"
   },
   "eip155:56/bep20:0xc0eff7749b125444953ef89682201fb8c6a917cd": {
     "assetId": "eip155:56/bep20:0xc0eff7749b125444953ef89682201fb8c6a917cd",
@@ -201045,7 +201045,7 @@
     "explorer": "https://bscscan.com",
     "explorerAddressLink": "https://bscscan.com/address/",
     "explorerTxLink": "https://bscscan.com/tx/",
-    "relatedAssetKey": "eip155:42161/erc20:0xde5ed76e7c05ec5e4572cfc88d1acea165109e44"
+    "relatedAssetKey": "eip155:1/erc20:0xde5ed76e7c05ec5e4572cfc88d1acea165109e44"
   },
   "eip155:56/bep20:0xde619a9e0eeeaa9f8cd39522ed788234837f3b26": {
     "assetId": "eip155:56/bep20:0xde619a9e0eeeaa9f8cd39522ed788234837f3b26",
@@ -223429,7 +223429,7 @@
     "explorer": "https://basescan.org",
     "explorerAddressLink": "https://basescan.org/address/",
     "explorerTxLink": "https://basescan.org/tx/",
-    "relatedAssetKey": "eip155:1/erc20:0xacdf0dba4b9839b96221a8487e9ca660a48212be"
+    "relatedAssetKey": null
   },
   "eip155:8453/erc20:0xcd2f22236dd9dfe2356d7c543161d4d260fd9bcb": {
     "assetId": "eip155:8453/erc20:0xcd2f22236dd9dfe2356d7c543161d4d260fd9bcb",
@@ -224603,7 +224603,7 @@
     "explorer": "https://basescan.org",
     "explorerAddressLink": "https://basescan.org/address/",
     "explorerTxLink": "https://basescan.org/tx/",
-    "relatedAssetKey": "eip155:42161/erc20:0xde5ed76e7c05ec5e4572cfc88d1acea165109e44"
+    "relatedAssetKey": "eip155:1/erc20:0xde5ed76e7c05ec5e4572cfc88d1acea165109e44"
   },
   "eip155:8453/erc20:0xde66c35e01ed8e619bf092352338ef94f2327337": {
     "explorer": "https://basescan.org",
@@ -231732,7 +231732,7 @@
     "explorer": "https://explorer.solana.com",
     "explorerAddressLink": "https://explorer.solana.com/address/",
     "explorerTxLink": "https://explorer.solana.com/tx/",
-    "relatedAssetKey": "eip155:1/erc20:0xdd50c053c096cb04a3e3362e2b622529ec5f2e8a"
+    "relatedAssetKey": null
   },
   "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp/token:4MshgHvWGvxDs8mtFqPGKC8kX6kuhniWSYPguBb1p1bh": {
     "assetId": "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp/token:4MshgHvWGvxDs8mtFqPGKC8kX6kuhniWSYPguBb1p1bh",
@@ -253988,7 +253988,7 @@
     "explorer": "https://explorer.solana.com",
     "explorerAddressLink": "https://explorer.solana.com/address/",
     "explorerTxLink": "https://explorer.solana.com/tx/",
-    "relatedAssetKey": "eip155:1/erc20:0xb6667b04cb61aa16b59617f90ffa068722cf21da"
+    "relatedAssetKey": "eip155:1/erc20:0x7cd017ca5ddb86861fa983a34b5f495c6f898c41"
   },
   "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp/token:Ed1yisBEAo8UXToSswvVFgzyJKpF48HEdq5kvz2zpump": {
     "assetId": "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp/token:Ed1yisBEAo8UXToSswvVFgzyJKpF48HEdq5kvz2zpump",