Merge pull request #314 from echovl/fix-nile-adapter

batch nile user positions fetching

adapters/nile/hourly_blocks.csv

@@ -1,2 +1,2 @@
 number,timestamp
-3976979,1713974398
+11449898,1730365248

adapters/nile/src/index.ts

@@ -1,6 +1,6 @@
 import fs from "fs";
 import { write } from "fast-csv";
-import csv from 'csv-parser';
+import csv from "csv-parser";
 import { BlockData, OutputSchemaRow, UserVote } from "./sdk/types";
 import {
   getV2UserPositionsAtBlock,
@@ -9,6 +9,7 @@ import {
 import { getTimestampAtBlock } from "./sdk/common";
 import { fetchUserVotes } from "./sdk/lensDetails";
 import BigNumber from "bignumber.js";
+import { getUserLockers } from "./sdk/lockers";
 
 const NILE_ADDRESS = "0xAAAac83751090C6ea42379626435f805DDF54DC8".toLowerCase();
 
@@ -82,14 +83,8 @@ export const getUserLiquidityTVLByBlock = async ({
     getV3UserPositionsAtBlock(blockNumber),
   ]);
 
-  const userAddresses = [...v2Positions, ...v3Positions]
-    .map((pos) => pos.user)
-    .reduce(
-      (prev, curr) => (prev.includes(curr) ? prev : [...prev, curr]),
-      [] as string[],
-    );
-
-  const userVotes = await getUserVotesTVLByBlock(blockNumber, userAddresses);
+  const userLockers = await getUserLockers(blockNumber);
+  const userVotes = await getUserVotesTVLByBlock(blockNumber, userLockers);
 
   // combine v2 & v3
   const combinedPositions = [...v2Positions, ...v3Positions];
@@ -142,24 +137,16 @@ export const getUserVotesTVLByBlock = async (
     [userAddress: string]: BigNumber;
   };
 
-  const batchSize = 300;
-  let userVotesResult: any[] = [];
-  for (let i = 0; i < userAddresses.length; i += batchSize) {
-    const batch = userAddresses.slice(i, i + batchSize);
-    userVotesResult = userVotesResult.concat(
-      await Promise.all(
-        batch.map((user) => fetchUserVotes(BigInt(blockNumber), user)),
-      ),
-    );
-  }
+  const userVotesResult = await fetchUserVotes(
+    BigInt(blockNumber),
+    userAddresses,
+  );
 
   for (const userFecthedVotes of userVotesResult) {
-    for (const userVote of userFecthedVotes) {
-      const userAddress = userVote.result.userAddress.toLowerCase();
-      tokenBalanceMap[userAddress] = BigNumber(
-        tokenBalanceMap[userAddress] ?? 0,
-      ).plus(userVote.result.amount.toString());
-    }
+    const userAddress = userFecthedVotes.result.userAddress.toLowerCase();
+    tokenBalanceMap[userAddress] = BigNumber(
+      tokenBalanceMap[userAddress] ?? 0,
+    ).plus(userFecthedVotes.result.amount.toString());
   }
 
   Object.entries(tokenBalanceMap).forEach(([userAddress, balance]) => {
@@ -178,54 +165,54 @@ export const getUserVotesTVLByBlock = async (
 //   console.log("Done");
 // });
 
-
 const readBlocksFromCSV = async (filePath: string): Promise<BlockData[]> => {
   const blocks: BlockData[] = [];
 
   await new Promise<void>((resolve, reject) => {
     fs.createReadStream(filePath)
       .pipe(csv()) // Specify the separator as '\t' for TSV files
-      .on('data', (row) => {
+      .on("data", (row) => {
         const blockNumber = parseInt(row.number, 10);
         const blockTimestamp = parseInt(row.timestamp, 10);
         if (!isNaN(blockNumber) && blockTimestamp) {
           blocks.push({ blockNumber: blockNumber, blockTimestamp });
         }
       })
-      .on('end', () => {
+      .on("end", () => {
         resolve();
       })
-      .on('error', (err) => {
+      .on("error", (err) => {
         reject(err);
       });
   });
 
   return blocks;
 };
 
+readBlocksFromCSV("hourly_blocks.csv")
+  .then(async (blocks: any[]) => {
+    console.log(blocks);
+    const allCsvRows: any[] = []; // Array to accumulate CSV rows for all blocks
 
-readBlocksFromCSV('hourly_blocks.csv').then(async (blocks: any[]) => {
-  console.log(blocks);
-  const allCsvRows: any[] = []; // Array to accumulate CSV rows for all blocks
-
-  for (const block of blocks) {
+    for (const block of blocks) {
       try {
-          const result = await getUserTVLByBlock(block);
-          // Accumulate CSV rows for all blocks
-          allCsvRows.push(...result);
+        const result = await getUserTVLByBlock(block);
+        // Accumulate CSV rows for all blocks
+        allCsvRows.push(...result);
       } catch (error) {
-          console.error(`An error occurred for block ${block}:`, error);
+        console.error(`An error occurred for block ${block}:`, error);
       }
-  }
-  await new Promise((resolve, reject) => {
-    const ws = fs.createWriteStream(`outputData.csv`, { flags: 'w' });
-    write(allCsvRows, { headers: true })
+    }
+    await new Promise((resolve, reject) => {
+      const ws = fs.createWriteStream(`outputData.csv`, { flags: "w" });
+      write(allCsvRows, { headers: true })
         .pipe(ws)
         .on("finish", () => {
-        console.log(`CSV file has been written.`);
-        resolve;
+          console.log(`CSV file has been written.`);
+          resolve;
         });
+    });
+  })
+  .catch((err) => {
+    console.error("Error reading CSV file:", err);
   });
-}).catch((err) => {
-  console.error('Error reading CSV file:', err);
-});

adapters/nile/src/sdk/config.ts

@@ -6,8 +6,11 @@ export const SUBGRAPH_URL =
 
 export const client = createPublicClient({
   chain: linea,
-  transport: http("https://rpc.linea.build", {
-    retryCount: 5,
-    timeout: 60_000,
-  }),
+  transport: http(
+    `https://linea-mainnet.infura.io/v3/${process.env.OPENBLOCK_LINEA_INFURA_API_KEY}`,
+    {
+      retryCount: 5,
+      timeout: 60_000,
+    },
+  ),
 });

adapters/nile/src/sdk/lensDetails.ts

@@ -15,80 +15,101 @@ export interface VoteResponse {
 
 export const fetchUserVotes = async (
   blockNumber: bigint,
-  userAddress: string,
+  userAddresses: string[],
 ): Promise<VoteResponse[]> => {
   const publicClient = client;
 
-  const userBalanceCall = await multicall(
+  const balanceCalls = userAddresses.map((userAddress) => ({
+    address: VE_NILE_ADDRESS,
+    name: "balanceOf",
+    params: [userAddress],
+  }));
+
+  const userBalances = await batchMulticall(
     publicClient,
     veNILEAbi as Abi,
-    [
-      {
-        address: VE_NILE_ADDRESS,
-        name: "balanceOf",
-        params: [userAddress],
-      },
-    ],
+    balanceCalls,
     blockNumber,
+    1000,
+    2000,
   );
 
-  const userBalance = userBalanceCall[0].result as number;
-
-  if (userBalance === 0) return [];
+  const tokenCalls: any = [];
+  userBalances.forEach((balance, index) => {
+    const userAddress = userAddresses[index];
+    const userBalance = balance.result as number;
 
-  const calls = [];
-  for (let i = 0; i < userBalance; i++) {
-    calls.push({
-      address: VE_NILE_ADDRESS,
-      name: "tokenOfOwnerByIndex",
-      params: [userAddress, i],
-    });
-  }
+    if (userBalance > 0) {
+      for (let i = 0; i < userBalance; i++) {
+        tokenCalls.push({
+          address: VE_NILE_ADDRESS,
+          name: "tokenOfOwnerByIndex",
+          params: [userAddress, i],
+        });
+      }
+    }
+  });
 
-  const userTokensCalls = await multicall(
+  const userTokensCalls = await batchMulticall(
     publicClient,
     veNILEAbi as Abi,
-    calls,
+    tokenCalls,
     blockNumber,
+    1000,
+    200,
   );
 
-  const detailsCall = userTokensCalls.map((call) => {
-    return {
-      address: VE_NILE_ADDRESS,
-      name: "locked",
-      params: [call.result],
-    };
-  });
+  const detailsCalls = userTokensCalls.map((call) => ({
+    address: VE_NILE_ADDRESS,
+    name: "locked",
+    params: [call.result],
+  }));
 
-  const res = (await multicall(
+  const res = (await batchMulticall(
     publicClient,
     veNILEAbi as Abi,
-    detailsCall,
+    detailsCalls,
     blockNumber,
+    1000,
+    2000,
   )) as any;
 
-  return res.map((r: any) => {
+  return res.map((r: any, index: any) => {
+    const userAddress = userAddresses[Math.floor(index / tokenCalls.length)];
     return { result: { amount: r.result[0], userAddress } };
   }) as VoteResponse[];
 };
 
-function multicall(
+async function batchMulticall(
   publicClient: PublicClient,
   abi: Abi,
   calls: any[],
   blockNumber: bigint,
+  batchSize: number,
+  delay: number,
 ) {
-  const call: MulticallParameters = {
-    contracts: calls.map((call) => {
-      return {
+  const results = [];
+
+  for (let i = 0; i < calls.length; i += batchSize) {
+    const batch = calls.slice(i, i + batchSize);
+
+    const call: MulticallParameters = {
+      contracts: batch.map((call) => ({
         address: call.address as Address,
         abi,
         functionName: call.name,
         args: call.params,
-      };
-    }),
-    blockNumber,
-  };
+      })),
+      blockNumber,
+    };
+
+    const res = await publicClient.multicall(call);
+    results.push(...res);
+
+    if (i + batchSize < calls.length) {
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+  }
 
-  return publicClient.multicall(call);
+  return results;
 }

adapters/nile/src/sdk/lockers.ts

@@ -0,0 +1,26 @@
+import { SUBGRAPH_URL } from "./config";
+
+export const getUserLockers = async (
+  blockNumber: number,
+): Promise<string[]> => {
+  const query = `query {
+            userLocker(
+                id: "lockers"
+                block: { number: ${blockNumber} }
+            ) {
+              users
+            }
+        }`;
+
+  const response = await fetch(SUBGRAPH_URL, {
+    method: "POST",
+    body: JSON.stringify({ query }),
+    headers: { "Content-Type": "application/json" },
+  });
+
+  const {
+    data: { userLocker },
+  } = await response.json();
+
+  return userLocker.users;
+};