diff --git a/adapters/mendi/src/index.ts b/adapters/mendi/src/index.ts index 9903e3eb..7c612c2b 100644 --- a/adapters/mendi/src/index.ts +++ b/adapters/mendi/src/index.ts @@ -1,7 +1,6 @@ import { CHAINS, PROTOCOLS } from "./sdk/config"; import { - getLPValueByUserAndPoolFromActivities, - getActivitiesForAddressByPoolAtBlock, + getAccountStatesForAddressByPoolAtBlock, getTimestampAtBlock, } from "./sdk/subgraphDetails"; @@ -10,9 +9,9 @@ import { }; import fs from "fs"; -import csv from 'csv-parser'; +import csv from "csv-parser"; import { write } from "fast-csv"; -import { getMarketInfos } from "./sdk/marketDetails"; +import { getMarketInfos, updateBorrowBalances } from "./sdk/marketDetails"; import { bigMath } from "./sdk/abi/helpers"; interface BlockData { @@ -38,7 +37,7 @@ export const getUserTVLByBlock = async (blocks: BlockData) => { const csvRows: OutputDataSchemaRow[] = []; const block = blocks.blockNumber; - const { tokens, accountBorrows } = await getActivitiesForAddressByPoolAtBlock( + const states = await getAccountStatesForAddressByPoolAtBlock( block, "", "", @@ -47,62 +46,51 @@ export const getUserTVLByBlock = async (blocks: BlockData) => { ); console.log(`Block: ${block}`); - console.log("Tokens: ", tokens.length); - console.log("Account Borrows: ", accountBorrows.length); + console.log("States: ", states.length); - let lpValueByUsers = getLPValueByUserAndPoolFromActivities( - tokens, - accountBorrows - ); + await updateBorrowBalances(states); - lpValueByUsers.forEach((value, owner) => { - value.forEach((amount, market) => { - if (bigMath.abs(amount) < 1) return; - - const marketInfo = marketInfos.get(market.toLowerCase()); - - // Accumulate CSV row data - csvRows.push({ - block_number: blocks.blockNumber, - timestamp: blocks.blockTimestamp, - user_address: owner, - token_address: marketInfo?.underlyingAddress ?? "", - token_balance: amount / BigInt(1e18), - token_symbol: marketInfo?.underlyingSymbol ?? "", - usd_price: 0, - }); + states.forEach((state) => { + const amount = state.lentAmount - state.borrowAmount; + + if (bigMath.abs(amount) < 1) return; + + const marketInfo = marketInfos.find( + (mi) => mi.underlyingAddress == state.token.toLowerCase() + ); + + // Accumulate CSV row data + csvRows.push({ + block_number: blocks.blockNumber, + timestamp: blocks.blockTimestamp, + user_address: state.account, + token_address: state.token, + token_balance: amount, + token_symbol: marketInfo?.underlyingSymbol ?? "", + usd_price: 0, }); }); - // Write the CSV output to a file - // const ws = fs.createWriteStream("outputData.csv"); - // write(csvRows, { headers: true }) - // .pipe(ws) - // .on("finish", () => { - // console.log("CSV file has been written."); - // }); - return csvRows; }; - const readBlocksFromCSV = async (filePath: string): Promise => { const blocks: BlockData[] = []; await new Promise((resolve, reject) => { fs.createReadStream(filePath) .pipe(csv()) // Specify the separator as '\t' for TSV files - .on('data', (row) => { + .on("data", (row) => { const blockNumber = parseInt(row.number, 10); const blockTimestamp = parseInt(row.timestamp, 10); if (!isNaN(blockNumber) && blockTimestamp) { blocks.push({ blockNumber: blockNumber, blockTimestamp }); } }) - .on('end', () => { + .on("end", () => { resolve(); }) - .on('error', (err) => { + .on("error", (err) => { reject(err); }); }); @@ -110,30 +98,31 @@ const readBlocksFromCSV = async (filePath: string): Promise => { return blocks; }; +readBlocksFromCSV("hourly_blocks.csv") + .then(async (blocks: any[]) => { + console.log(blocks); + const allCsvRows: any[] = []; // Array to accumulate CSV rows for all blocks -readBlocksFromCSV('hourly_blocks.csv').then(async (blocks: any[]) => { - console.log(blocks); - const allCsvRows: any[] = []; // Array to accumulate CSV rows for all blocks - - for (const block of blocks) { + for (const block of blocks) { try { - const result = await getUserTVLByBlock(block); - for(let i = 0; i < result.length; i++){ - allCsvRows.push(result[i]) - } + const result = await getUserTVLByBlock(block); + for (let i = 0; i < result.length; i++) { + allCsvRows.push(result[i]); + } } catch (error) { - console.error(`An error occurred for block ${block}:`, error); + console.error(`An error occurred for block ${block}:`, error); } - } - await new Promise((resolve, reject) => { - const ws = fs.createWriteStream(`outputData.csv`, { flags: 'w' }); - write(allCsvRows, { headers: true }) + } + await new Promise((resolve, reject) => { + const ws = fs.createWriteStream(`outputData.csv`, { flags: "w" }); + write(allCsvRows, { headers: true }) .pipe(ws) .on("finish", () => { - console.log(`CSV file has been written.`); - resolve; + console.log(`CSV file has been written.`); + resolve; }); + }); + }) + .catch((err) => { + console.error("Error reading CSV file:", err); }); -}).catch((err) => { - console.error('Error reading CSV file:', err); -}); diff --git a/adapters/mendi/src/sdk/config.ts b/adapters/mendi/src/sdk/config.ts index 1be86669..4283befd 100644 --- a/adapters/mendi/src/sdk/config.ts +++ b/adapters/mendi/src/sdk/config.ts @@ -9,7 +9,7 @@ export const enum PROTOCOLS { export const SUBGRAPH_URLS = { [CHAINS.LINEA]: { [PROTOCOLS.MENDI]: { - url: "https://api.goldsky.com/api/public/project_cltshpix6kkj301x1b4ilh6pm/subgraphs/mendi-linea/1.4/gn", + url: "https://api.goldsky.com/api/public/project_cltshpix6kkj301x1b4ilh6pm/subgraphs/mendi-subgraph/1.0/gn", }, }, }; diff --git a/adapters/mendi/src/sdk/marketDetails.ts b/adapters/mendi/src/sdk/marketDetails.ts index 189fd765..7b4e9b61 100644 --- a/adapters/mendi/src/sdk/marketDetails.ts +++ b/adapters/mendi/src/sdk/marketDetails.ts @@ -3,7 +3,7 @@ import { CHAINS, RPC_URLS } from "./config"; import { linea } from "viem/chains"; import comptrollerAbi from "./abi/comptroller.abi"; import ctokenAbi from "./abi/ctoken.abi"; -import { MarketActivity } from "./subgraphDetails"; +import { AccountState } from "./subgraphDetails"; export interface MarketInfo { address: string; @@ -78,52 +78,65 @@ export const getMarketInfos = async ( blockNumber, }); - const marketInfos = new Map(); + const marketInfos: MarketInfo[] = []; for (let i = 0; i < markets.length; i++) { const marketAddress = markets[i].address.toLowerCase(); - var marketInfo = marketInfos.get(marketAddress); - if (marketInfo === undefined) { - marketInfo = {} as MarketInfo; - marketInfos.set(marketAddress, marketInfo); - } - - marketInfo.underlyingAddress = underlyingResults[i].result as any; - marketInfo.exchangeRateStored = BigInt( - exchangeRateResults[i].status == "success" - ? (exchangeRateResults[i].result as any) - : 0 - ); - - marketInfo.underlyingSymbol = underlyingSymbolResults[i].result as any; + marketInfos.push({ + address: marketAddress, + underlyingAddress: (underlyingResults[i].result as any)?.toLowerCase(), + underlyingSymbol: underlyingSymbolResults[i].result as any, + exchangeRateStored: BigInt( + exchangeRateResults[i].status == "success" + ? (exchangeRateResults[i].result as any) + : 0 + ), + }); } return marketInfos; }; -export const getBorrowBalanceStoredByAccounts = async ( - activities: MarketActivity[], +export const updateBorrowBalances = async ( + states: AccountState[], blockNumber?: bigint ) => { + const marketInfos = await getMarketInfos( + "0x1b4d3b0421ddc1eb216d230bc01527422fb93103" + ); + const marketsByUnderlying: any = {}; + for (let marketInfo of marketInfos) { + marketsByUnderlying[marketInfo.underlyingAddress] = marketInfo.address; + } + const publicClient = createPublicClient({ chain: extractChain({ chains: [linea], id: CHAINS.LINEA }), transport: http(RPC_URLS[CHAINS.LINEA]), }); - const borrowBalanceResults = await publicClient.multicall({ - contracts: activities - .map((m) => [ - { - address: m.market, - abi: ctokenAbi, - functionName: "borrowBalanceStored", - args: [m.owner], - }, - ]) - .flat() as any, - blockNumber, - }); - const borrowBalances = borrowBalanceResults.map((v) => v.result as bigint); + states = states.filter((x) => x.borrowAmount > 0); - return borrowBalances; + for (var i = 0; i < states.length; i += 500) { + var subStates = states.slice(i, 500); + + const borrowBalanceResults = await publicClient.multicall({ + contracts: subStates + .map((m) => [ + { + address: marketsByUnderlying[m.token], + abi: ctokenAbi, + functionName: "borrowBalanceStored", + args: [m.account], + }, + ]) + .flat() as any, + blockNumber, + }); + + for (var j = 0; j < subStates.length; j++) { + subStates[j].borrowAmount = BigInt( + borrowBalanceResults[j].result?.toString() ?? 0 + ); + } + } }; diff --git a/adapters/mendi/src/sdk/subgraphDetails.ts b/adapters/mendi/src/sdk/subgraphDetails.ts index ea71d06c..acd28012 100644 --- a/adapters/mendi/src/sdk/subgraphDetails.ts +++ b/adapters/mendi/src/sdk/subgraphDetails.ts @@ -1,37 +1,31 @@ -import { createPublicClient, extractChain, http } from "viem"; +import { Account, createPublicClient, extractChain, http } from "viem"; import { linea } from "viem/chains"; import { CHAINS, PROTOCOLS, RPC_URLS, SUBGRAPH_URLS } from "./config"; -import { - getBorrowBalanceStoredByAccounts, - getMarketInfos, -} from "./marketDetails"; +import { getMarketInfos } from "./marketDetails"; -export interface MarketActivity { +export interface AccountState { id: string; - amount: bigint; - owner: string; - market: string; - blockNumber: number; + account: string; + token: string; + lentAmount: bigint; + borrowAmount: bigint; } -export const getActivitiesForAddressByPoolAtBlock = async ( +export const getAccountStatesForAddressByPoolAtBlock = async ( blockNumber: number, address: string, poolId: string, chainId: CHAINS, protocol: PROTOCOLS -): Promise<{ tokens: MarketActivity[]; accountBorrows: MarketActivity[] }> => { +): Promise => { const marketInfos = await getMarketInfos( "0x1b4d3b0421ddc1eb216d230bc01527422fb93103", BigInt(blockNumber) ); address = address?.toLowerCase(); - let bnWhereQuery = blockNumber ? `block_number_lte: ${blockNumber} \n` : ""; - let transferWhereQuery = address - ? orWhere(`to: "${address}"`, `from: "${address}"`) - : ""; - let borrowerWhereQuery = address ? `borrower: "${address}" \n` : ""; + let accountWhereQuery = address ? `account: "${address}" \n` : ""; + let amountWhereQuery = orWhere("lentAmount_not: 0", "borrowAmount_not: 0"); let skip = 0; let fetchNext = true; @@ -39,44 +33,29 @@ export const getActivitiesForAddressByPoolAtBlock = async ( const pageSize = 1000; const url = SUBGRAPH_URLS[chainId][protocol].url; - let tokens: MarketActivity[] = []; - let accountBorrows: MarketActivity[] = []; + let states: AccountState[] = []; while (fetchNext) { let query = ` - { - transfers( - where: { ${andWhere(bnWhereQuery, transferWhereQuery)} } - orderBy: id, first: ${pageSize}, skip: ${skip}) - { - id - from - to - amount - block_number - contractId_ - } - borrows( - where: { ${andWhere(bnWhereQuery, borrowerWhereQuery)} } - orderBy: id, first: ${pageSize}, skip: ${skip}) - { - id - borrower - accountBorrows - block_number - contractId_ + query TVLQuery { + accountStates( + block: {number: ${blockNumber}} + where: { + ${andWhere(accountWhereQuery, amountWhereQuery)} } - repayBorrows( - where: { ${andWhere(bnWhereQuery, borrowerWhereQuery)} } - orderBy: id, first: ${pageSize}, skip: ${skip}) - { + orderBy: id + first: ${pageSize} + skip: ${skip} + ) { + id + account + token { id - borrower - accountBorrows - block_number - contractId_ } + lentAmount + borrowAmount } + } `; console.log(`Fetching ${skip} - ${skip + pageSize} / unknown`); @@ -87,169 +66,41 @@ export const getActivitiesForAddressByPoolAtBlock = async ( }); let data = await response.json(); - // tokens - const mints = data.data.transfers - .filter((m: any) => m.to != m.contractId_) - .map((m: any) => ({ - id: m.id, - amount: BigInt(m.amount), - owner: m.to, - market: m.contractId_, - blockNumber: m.block_number, - })); - - const redeems = data.data.transfers - .filter((m: any) => m.from != m.contractId_) - .map((m: any) => ({ + states.push( + ...data.data.accountStates.map((m: any) => ({ id: m.id, - amount: -BigInt(m.amount), - owner: m.from, - market: m.contractId_, - blockNumber: m.block_number, - })); - - // borrows - const borrows = data.data.borrows.map((m: any) => ({ - id: m.id, - amount: -BigInt(m.accountBorrows), - owner: m.borrower, - market: m.contractId_, - blockNumber: m.block_number, - })); - const repayBorrows = data.data.repayBorrows.map((m: any) => ({ - id: m.id, - amount: -BigInt(m.accountBorrows), - owner: m.borrower, - market: m.contractId_, - blockNumber: m.block_number, - })); - - tokens.push(...mints, ...redeems); - accountBorrows.push(...borrows, ...repayBorrows); + account: m.account, + token: m.token.id, + lentAmount: BigInt(m.lentAmount), + borrowAmount: BigInt(m.borrowAmount), + })) + ); - if ( - mints.length == 0 && - redeems.length == 0 && - borrows.length == 0 && - repayBorrows.length == 0 - ) { + if (data.data.accountStates.length == 0) { fetchNext = false; } else { skip += pageSize; } } - tokens = tokens - .map((t: any) => { - const marketInfo = marketInfos.get(t.market.toLowerCase()); + states = states + .map((state: AccountState) => { + const marketInfo = marketInfos.find( + (mi) => mi.underlyingAddress == state.token.toLowerCase() + ); if (!marketInfo) { return undefined; } return { - ...t, - amount: t.amount * marketInfo.exchangeRateStored, + ...state, + lentAmount: + (state.lentAmount * marketInfo.exchangeRateStored) / BigInt(1e18), }; }) - .filter((x) => x !== undefined); - - accountBorrows = await normalizeAccountBorrows(blockNumber, accountBorrows); - accountBorrows.forEach((t: any) => (t.amount = t.amount * BigInt(1e18))); - - return { tokens, accountBorrows }; -}; - -export const normalizeAccountBorrows = async ( - blockNumber: number, - accountBorrows: MarketActivity[] -): Promise => { - const result: MarketActivity[] = []; - - accountBorrows.sort((a, b) => b.blockNumber - a.blockNumber); - - for (let i = 0; i < accountBorrows.length; i++) { - var marketActivity = accountBorrows[i]; - - var doesExist = - result.findIndex( - (x: MarketActivity) => - x.owner == marketActivity.owner && x.market == marketActivity.market - ) > -1; - if (doesExist) continue; - - result.push(marketActivity); - } - - const chuckCount = 2000; - for (let i = 0; i < result.length; i += chuckCount) { - var end = Math.min(result.length, i + chuckCount); - - var currentBorrows = await getBorrowBalanceStoredByAccounts( - result.slice(i, end), - BigInt(blockNumber) - ); - - for (let j = 0; j < currentBorrows.length; j++) { - result[i + j].amount = -BigInt(currentBorrows[j]); - } - } - - return result; -}; - -export const getLPValueByUserAndPoolFromActivities = ( - tokens: MarketActivity[], - accountBorrows: MarketActivity[] -): Map> => { - // owner => market => amount - let result = new Map>(); - - // get latest account borrows - // sort descending to block number - accountBorrows.sort((a, b) => b.blockNumber - a.blockNumber); - - for (let i = 0; i < accountBorrows.length; i++) { - let marketActivity = accountBorrows[i]; - let market = marketActivity.market; - let owner = marketActivity.owner; - let amount = marketActivity.amount; - - let userActivities = result.get(owner); - if (userActivities === undefined) { - userActivities = new Map(); - result.set(owner, userActivities); - } - - let poolActivities = userActivities.get(market); - if (poolActivities === undefined) { - userActivities.set(market, amount); - } - } - - // add token activities - for (let i = 0; i < tokens.length; i++) { - let marketActivity = tokens[i]; - - let market = marketActivity.market; - let owner = marketActivity.owner; - let amount = marketActivity.amount; - - let userActivities = result.get(owner); - if (userActivities === undefined) { - userActivities = new Map(); - result.set(owner, userActivities); - } - - let poolActivities = userActivities.get(market); - if (poolActivities === undefined) { - poolActivities = BigInt(0); - } - - poolActivities = poolActivities + amount; - userActivities.set(market, poolActivities); - } + .filter((x) => x !== undefined) as AccountState[]; - return result; + return states; }; export const getTimestampAtBlock = async (blockNumber: number) => {