diff --git a/adapters/clip-finance/hourly_blocks.csv b/adapters/clip-finance/hourly_blocks.csv new file mode 100644 index 00000000..8145066b --- /dev/null +++ b/adapters/clip-finance/hourly_blocks.csv @@ -0,0 +1,2 @@ +number timestamp +4333348 1715043599 \ No newline at end of file diff --git a/adapters/clip-finance/package.json b/adapters/clip-finance/package.json new file mode 100644 index 00000000..934e320c --- /dev/null +++ b/adapters/clip-finance/package.json @@ -0,0 +1,32 @@ +{ + "name": "clip-finance", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node dist/index.js", + "compile": "tsc", + "watch": "tsc -w", + "clear": "rm -rf dist" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "@types/big.js": "^6.2.2", + "big.js": "^6.2.1", + "bignumber.js": "^9.1.2", + "csv-parser": "^3.0.0", + "decimal.js-light": "^2.5.1", + "fast-csv": "^5.0.1", + "jsbi": "^4.3.0", + "tiny-invariant": "^1.3.1", + "toformat": "^2.0.0", + "viem": "^2.8.13" + }, + "devDependencies": { + "@types/node": "^20.11.17", + "typescript": "^5.3.3" + } +} \ No newline at end of file diff --git a/adapters/clip-finance/src/index.ts b/adapters/clip-finance/src/index.ts new file mode 100644 index 00000000..dd5fb2b8 --- /dev/null +++ b/adapters/clip-finance/src/index.ts @@ -0,0 +1,100 @@ +import { getTimestampAtBlock, getUserBalanceSnapshotAtBlock } from "./sdk/subgraphDetails"; +import fs from 'fs'; +import csv from 'csv-parser'; +import { write } from 'fast-csv'; + +interface CSVRow { + block_number: number; + timestamp: number; + user_address: string; + token_address: string; + token_balance: bigint; + token_symbol: string; + usd_price: number +} + +interface BlockData { + blockNumber: number; + blockTimestamp: number; +} + +export const getUserTVLByBlock = async (blocks: BlockData) => { + const { blockNumber, blockTimestamp } = blocks; + const snapshotBlocks: number[] = [blockNumber]; + + const csvRows: CSVRow[] = []; + + for (const block of snapshotBlocks) { + let snapshots = await getUserBalanceSnapshotAtBlock(block, ""); + + // const timestamp = await getTimestampAtBlock(block); + + for (const snapshot of snapshots) { + const csvRow: CSVRow = { + block_number: block, + timestamp: blockTimestamp, + user_address: snapshot.id, + token_address: snapshot.token, + token_balance: BigInt(snapshot.balance.toString()), + token_symbol: snapshot.tokenSymbol, + usd_price: 0 + }; + csvRows.push(csvRow); + } + } + + console.log("Total rows:", csvRows.length); + + return csvRows; +}; + +const readBlocksFromCSV = async (filePath: string): Promise => { + const blocks: BlockData[] = []; + + await new Promise((resolve, reject) => { + fs.createReadStream(filePath) + .pipe(csv()) // Specify the separator as '\t' for TSV files + .on('data', (row) => { + + const blockNumber = parseInt(row.number, 10); + const blockTimestamp = parseInt(row.timestamp, 10); + if (!isNaN(blockNumber) && blockTimestamp) { + blocks.push({ blockNumber: blockNumber, blockTimestamp }); + } + }) + .on('end', () => { + resolve(); + }) + .on('error', (err) => { + reject(err); + }); + }); + + return blocks; +}; + +readBlocksFromCSV('hourly_blocks.csv').then(async (blocks: any[]) => { + console.log(blocks); + const allCsvRows: any[] = []; + + for (const block of blocks) { + try { + const result = await getUserTVLByBlock(block); + allCsvRows.push(...result); + } catch (error) { + console.error(`An error occurred for block ${block}:`, error); + } + } + await new Promise((resolve, reject) => { + const ws = fs.createWriteStream(`outputData.csv`, { flags: 'w' }); + write(allCsvRows, { headers: true }) + .pipe(ws) + .on("finish", () => { + console.log(`CSV file has been written.`); + resolve; + }); + }); + + }).catch((err) => { + console.error('Error reading CSV file:', err); + }); \ No newline at end of file diff --git a/adapters/clip-finance/src/sdk/config.ts b/adapters/clip-finance/src/sdk/config.ts new file mode 100644 index 00000000..0b2dbb2a --- /dev/null +++ b/adapters/clip-finance/src/sdk/config.ts @@ -0,0 +1,19 @@ +export const enum CHAINS { + LINEA = 59144, +} + +export const SUBGRAPH_URLS = { + [CHAINS.LINEA]: + "https://api.goldsky.com/api/public/project_cltzfe75l0y4u01s98n3c7fmu/subgraphs/clip-finance-shares-token/v2.4/gn", +}; + +export const RESERVE_SUBGRAPH_URLS = { + [CHAINS.LINEA]: + "https://api.goldsky.com/api/public/project_cltzfe75l0y4u01s98n3c7fmu/subgraphs/clip-finance-shares-token/v2.5/gn", + +} + +export const RPC_URLS = { + [CHAINS.LINEA]: + "https://rpc.linea.build", +}; \ No newline at end of file diff --git a/adapters/clip-finance/src/sdk/subgraphDetails.ts b/adapters/clip-finance/src/sdk/subgraphDetails.ts new file mode 100644 index 00000000..4a9c2db8 --- /dev/null +++ b/adapters/clip-finance/src/sdk/subgraphDetails.ts @@ -0,0 +1,317 @@ +import { createPublicClient, extractChain, http } from "viem"; +import { linea } from "viem/chains"; +import { SUBGRAPH_URLS, CHAINS, RPC_URLS, RESERVE_SUBGRAPH_URLS } from "./config"; +import Big from "big.js"; + +export interface UserBalanceSnapshot { + id: string; + balance: Big; + token : string; + tokenSymbol: string; +} + +export interface User { + id: string; + balance: Big; + token : string; + tokenSymbol: string; +} + +interface SharePricesSnapshot { + id: string; + price0: Big; + price01: Big; + price1: Big; + price10: Big; + token0: string; + token0Symbol: string; + token1 : string; + token1Symbol: string; +} + +interface UserSharesSnapshot { + id: string; + shares0: Big; + shares1: Big; +} + +function delay(ms: number) { + return new Promise( resolve => setTimeout(resolve, ms) ); +} + +export const getUserBalanceSnapshotAtBlock = async ( + blockNumber: number, + address: string +): Promise => { + let subgraphUrl = SUBGRAPH_URLS[CHAINS.LINEA]; + let blockQuery = blockNumber !== 0 ? ` block: {number: ${blockNumber}}` : ``; + + let idQuery = address !== "" ? `id: "${address.toLowerCase()}"` : ``; + let showZeroBalances = false; + let balanceNotZeroQuery = showZeroBalances ? "" : `balance_gt: 0`; + let whereQueries = [idQuery, balanceNotZeroQuery]; + + let skip = 0; + let fetchNext = true; + let result: UserBalanceSnapshot[] = []; + const sharePricesMap = new Map(); + while (fetchNext) { + const query = `{ + sharePrices( + ${blockQuery} + first:1000, skip:${skip} + ){ + id + price0 + price01 + price1 + price10 + token0 + token0Symbol + token1 + token1Symbol + } + } + `; + let count = 0; + let response; + do { + response = await fetch(subgraphUrl, { + method: "POST", + body: JSON.stringify({ query }), + headers: { "Content-Type": "application/json" }, + }); + if (response.status != 200) { + subgraphUrl = RESERVE_SUBGRAPH_URLS[CHAINS.LINEA]; + response = await fetch(subgraphUrl, { + method: "POST", + body: JSON.stringify({ query }), + headers: { "Content-Type": "application/json" }, + }); + } + if (response.status != 200) { + console.log("sharePrices fetching failed. Try again in 15 sec"); + await delay(15000); + } + ++count + } while ((response.status != 200) && (count < 10)) + + let data = await response.json(); + let snapshots = data.data.sharePrices; + for (const snapshot of snapshots) { + const sharePriceSnapshot: SharePricesSnapshot = { + id: snapshot.id, + price0: Big(snapshot.price0), + price01: Big(snapshot.price01), + price1 : Big(snapshot.price1), + price10: Big(snapshot.price10), + token0 : snapshot.token0, + token0Symbol: snapshot.token0Symbol, + token1 : snapshot.token1, + token1Symbol: snapshot.token1Symbol, + } + sharePricesMap.set(snapshot.id, sharePriceSnapshot); + } + if (snapshots.length < 1000) { + fetchNext = false; + } else { + skip += 1000; + } + } + + skip = 0; + fetchNext = true; + const balanceMap = new Map(); + const strategyRouterSharesMap = new Map(); + let strategyRouterBalance = new Map(); + const addBalance = (balance: UserBalanceSnapshot, share: UserBalanceSnapshot) => { + const user= share.id.substring(0, 42); + const key = user.concat(balance.token); + if (user == "0xa663f143055254a503467ff8b18aa9e70b9455b6") { + strategyRouterBalance.set(key.concat(balance.token), balance); + } else if (balance.balance.gt(0)) { + if (!balanceMap.has(key)) { + balanceMap.set(key, balance); + } else { + const oldUserBalanceSnapshot = balanceMap.get(key); + if (oldUserBalanceSnapshot) { + oldUserBalanceSnapshot.balance = oldUserBalanceSnapshot.balance.plus(balance.balance); + balanceMap.set(key, oldUserBalanceSnapshot); + } + } + } + }; + + while (fetchNext) { + const query = `{ + userShares( + ${blockQuery} + first:1000, skip:${skip} + ){ + id + shares0 + shares1 + } + } + `; + let count = 0; + let response; + do { + response = await fetch(subgraphUrl, { + method: "POST", + body: JSON.stringify({ query }), + headers: { "Content-Type": "application/json" }, + }); + if (response.status != 200) { + await delay(15000); + console.log("userShares fetching failed. Try again in 15 sec"); + } + ++count; + } while ((count < 10) && (response.status != 200)) { + + } + let data = await response.json(); + let snapshots = data.data.userShares; + for (const snapshot of snapshots) { + const contract = "0x".concat(snapshot.id.substring(42)); + const sharePrice = sharePricesMap.get(contract); + const user = snapshot.id.substring(0, 42); + if (sharePrice) { + let userBalanceSnapshot: UserBalanceSnapshot = { + id: "", + balance: Big(0), + token : "", + tokenSymbol: "", + + }; + if (sharePrice.price0.gt(0)) { + userBalanceSnapshot = { + id: user.toLowerCase(), + balance: Big(Math.round(Big(snapshot.shares0).mul(sharePrice.price0).div(1e18).toNumber())), + token : sharePrice.token0.toLowerCase(), + tokenSymbol: sharePrice.token0Symbol, + + } + addBalance(userBalanceSnapshot, snapshot); + } + if (sharePrice.price01.gt(0)) { + userBalanceSnapshot = { + id: user.toLowerCase(), + balance: Big(Math.round(Big(snapshot.shares0).mul(sharePrice.price01).div(1e18).toNumber())), + token : sharePrice.token1.toLowerCase(), + tokenSymbol: sharePrice.token1Symbol, + + } + addBalance(userBalanceSnapshot, snapshot); + } + if (sharePrice.price1.gt(0)) { + userBalanceSnapshot = { + id: user.toLowerCase(), + balance: Big(Math.round(Big(snapshot.shares1).mul(sharePrice.price1).div(1e18).toNumber())), + token : sharePrice.token1.toLowerCase(), + tokenSymbol: sharePrice.token1Symbol, + } + addBalance(userBalanceSnapshot, snapshot); + } + if (sharePrice.price10.gt(0)) { + userBalanceSnapshot = { + id: user.toLowerCase(), + balance: Big(Math.round(Big(snapshot.shares1).mul(sharePrice.price10).div(1e18).toNumber())), + token : sharePrice.token0.toLowerCase(), + tokenSymbol: sharePrice.token0Symbol, + + } + addBalance(userBalanceSnapshot, snapshot); + } + } else { + if (Big(snapshot.shares0).gt(0)) { + strategyRouterSharesMap.set(snapshot.id, snapshot); + } + } + } + if (snapshots.length < 1000) { + fetchNext = false; + } else { + skip += 1000; + } + } + + const query = `{ + sharesTokenSharesCounts ( + ${blockQuery} + ){ + id + total + } + } + `; + + let count = 0; + let response; + do { + response = await fetch(subgraphUrl, { + method: "POST", + body: JSON.stringify({ query }), + headers: { "Content-Type": "application/json" }, + }); + if (response.status != 200) { + console.log("sharesTokenSharesCounts fetching failed. Try again in 15 sec"); + await delay(15000) + } + ++count; + } while ((count < 10) && (response.status != 200)); + let data = await response.json(); + let snapshots = data.data.sharesTokenSharesCounts; + let strategyRouterTotalShares: Big = Big(0); + for (const snapshot of snapshots) { + strategyRouterTotalShares = Big(snapshot.total); + } + let countedTotalShares: Big = Big(0); + if (strategyRouterTotalShares.gt(0)) { + let checkBalance = Big(0); + strategyRouterSharesMap.forEach((share: UserSharesSnapshot, id: string)=> { + const user = share.id.substring(0, 42); + for (const srbKey of strategyRouterBalance.keys()) { + const balance = strategyRouterBalance.get(srbKey); + if (balance) { + countedTotalShares = countedTotalShares.plus(Big(share.shares0)); + const userBalance : UserBalanceSnapshot = { + id: user.toLowerCase(), + balance: Big(Math.round(Big(share.shares0).mul(balance.balance).div(strategyRouterTotalShares).toNumber())), + token : balance.token.toLowerCase(), + tokenSymbol: balance.tokenSymbol + } + + checkBalance = checkBalance.plus(userBalance.balance); + const key = user.concat(balance.token); + if (!balanceMap.has(key)) { + balanceMap.set(key, userBalance); + } else { + const oldUserBalance = balanceMap.get(key); + if (oldUserBalance) { + oldUserBalance.balance = oldUserBalance.balance.plus(userBalance.balance); + balanceMap.set(key, userBalance); + } + } + } + } + }); + } + + return Array.from(balanceMap.values()); +}; + +export const getTimestampAtBlock = async (blockNumber: number) => { + const publicClient = createPublicClient({ + chain: extractChain({ chains: [linea], id: linea.id }), + transport: http(RPC_URLS[CHAINS.LINEA], { + retryCount: 5, + timeout: 60_000, + }), + }); + + const block = await publicClient.getBlock({ + blockNumber: BigInt(blockNumber), + }); + return Number(block.timestamp * 1000n); +}; \ No newline at end of file diff --git a/adapters/clip-finance/tsconfig.json b/adapters/clip-finance/tsconfig.json new file mode 100644 index 00000000..55603863 --- /dev/null +++ b/adapters/clip-finance/tsconfig.json @@ -0,0 +1,109 @@ +{ + "compilerOptions": { + /* Visit https://aka.ms/tsconfig to read more about this file */ + + /* Projects */ + // "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ + // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ + // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ + // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ + // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ + // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ + + /* Language and Environment */ + "target": "es2022", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ + // "jsx": "preserve", /* Specify what JSX code is generated. */ + // "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */ + // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ + // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */ + // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ + // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */ + // "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */ + // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ + // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ + // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ + + /* Modules */ + "module": "commonjs", /* Specify what module code is generated. */ + "rootDir": "src/", /* Specify the root folder within your source files. */ + // "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */ + // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ + // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ + // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ + // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */ + // "types": [], /* Specify type package names to be included without being referenced in a source file. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */ + // "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */ + // "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */ + // "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */ + // "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */ + // "resolveJsonModule": true, /* Enable importing .json files. */ + // "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */ + // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ + + /* JavaScript Support */ + // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ + // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ + // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ + + /* Emit */ + // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + // "declarationMap": true, /* Create sourcemaps for d.ts files. */ + // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ + // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ + // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ + "outDir": "dist/", /* Specify an output folder for all emitted files. */ + // "removeComments": true, /* Disable emitting comments. */ + // "noEmit": true, /* Disable emitting files from a compilation. */ + // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ + // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types. */ + // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ + // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ + // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ + // "newLine": "crlf", /* Set the newline character for emitting files. */ + // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ + // "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */ + // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ + // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */ + // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ + // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ + + /* Interop Constraints */ + // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ + // "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */ + // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ + "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ + // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ + "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ + + /* Type Checking */ + "strict": true, /* Enable all strict type-checking options. */ + // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ + // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ + // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + // "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ + // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ + // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ + // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ + // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ + // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ + // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ + // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + + /* Completeness */ + // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + } +} \ No newline at end of file diff --git a/adapters/package.json b/adapters/package.json index 92ded735..ec94e15a 100644 --- a/adapters/package.json +++ b/adapters/package.json @@ -13,6 +13,7 @@ "license": "UNLICENSED", "dependencies": { "csv-parser": "^3.0.0", - "fast-csv": "^5.0.1" + "fast-csv": "^5.0.1", + "viem": "^2.9.20" } } diff --git a/adapters/sparta/src/index.ts b/adapters/sparta/src/index.ts index dfc581e4..267fb661 100644 --- a/adapters/sparta/src/index.ts +++ b/adapters/sparta/src/index.ts @@ -38,7 +38,6 @@ async function processBlockData(block: number): Promise { // get reserves at block const reservesSnapshotAtBlock = await fetchReservesForPools(block); - console.log("reservesSnapshotAtBlock"); // calculate tokens based on reserves const userReserves = calculateUserReservePortion( userPositions, @@ -46,8 +45,6 @@ async function processBlockData(block: number): Promise { reservesSnapshotAtBlock ); - console.log("ok"); - const timestamp = await getTimestampAtBlock(block); // convert userReserves to userPositions @@ -59,8 +56,6 @@ function convertToUserPositions( block_number: number, timestamp: number ): UserPosition[] { - console.log(`userData`, userData); - const tempResults: Record = {}; Object.keys(userData).forEach((user) => { @@ -157,46 +152,48 @@ function processTransactions(transactions: Transaction[]): { const toAddress = transaction.to.toLowerCase(); const contractId = transaction.contractId_.toLowerCase(); - // Skip transactions where 'from' or 'to' match the contract ID, or both 'from' and 'to' are zero addresses + // Skip internal lp txs if ( - fromAddress === contractId || - toAddress === contractId || - (fromAddress === "0x0000000000000000000000000000000000000000" && - toAddress === "0x0000000000000000000000000000000000000000") + (fromAddress === contractId && + toAddress === "0x0000000000000000000000000000000000000000") || + (toAddress === contractId && + fromAddress === "0x0000000000000000000000000000000000000000") ) { return; } - // Initialize cumulativePositions if not already set + // Initialize userPositions and cumulativePositions if not already set + if (!userPositions[contractId]) { + userPositions[contractId] = {}; + } if (!cumulativePositions[contractId]) { cumulativePositions[contractId] = 0; } // Convert the transaction value from string to integer. - let value = parseInt(transaction.value.toString()); + const value = parseInt(transaction.value.toString(), 10); - // Process transactions that increase liquidity (to address isn't zero) - if (toAddress !== "0x0000000000000000000000000000000000000000") { - if (!userPositions[contractId]) { - userPositions[contractId] = {}; - } - if (!userPositions[contractId][toAddress]) { - userPositions[contractId][toAddress] = 0; - } - userPositions[contractId][toAddress] += value; - cumulativePositions[contractId] += value; - } - - // Process transactions that decrease liquidity (from address isn't zero) + // Decrease liquidity from the sender if the from address is not zero if (fromAddress !== "0x0000000000000000000000000000000000000000") { - if (!userPositions[contractId]) { - userPositions[contractId] = {}; - } if (!userPositions[contractId][fromAddress]) { userPositions[contractId][fromAddress] = 0; } userPositions[contractId][fromAddress] -= value; cumulativePositions[contractId] -= value; + + // Remove the sender from userPositions if their balance is zero + if (userPositions[contractId][fromAddress] === 0) { + delete userPositions[contractId][fromAddress]; + } + } + + // Increase liquidity for the receiver if the to address is not zero + if (toAddress !== "0x0000000000000000000000000000000000000000") { + if (!userPositions[contractId][toAddress]) { + userPositions[contractId][toAddress] = 0; + } + userPositions[contractId][toAddress] += value; + cumulativePositions[contractId] += value; } }); @@ -204,12 +201,34 @@ function processTransactions(transactions: Transaction[]): { } async function fetchTransfers(blockNumber: number) { - const { data } = await client.query({ - query: TRANSFERS_QUERY, - variables: { blockNumber }, - fetchPolicy: "no-cache", - }); - return data.transfers; + const allTransfers = []; + const pageSize = 1000; + let skip = 0; + let hasMore = true; + + while (hasMore) { + try { + const { data } = await client.query({ + query: TRANSFERS_QUERY, + variables: { blockNumber, first: pageSize, skip }, + fetchPolicy: "no-cache", + }); + + const transfers = data.transfers; + allTransfers.push(...transfers); + + if (transfers.length < pageSize) { + hasMore = false; + } else { + skip += pageSize; + } + } catch (error) { + console.error("Error fetching transfers:", error); + break; + } + } + + return allTransfers; } async function fetchReservesForPools(blockNumber: number): Promise { @@ -238,8 +257,8 @@ function convertToOutputDataSchema( return { block_number: userPosition.block_number, timestamp: userPosition.timestamp, - user_address: userPosition.user, - token_address: userPosition.token, + user_address: userPosition.user.toLowerCase(), + token_address: userPosition.token.toLowerCase(), token_balance: BigInt(userPosition.balance), // Ensure balance is treated as bigint token_symbol: "", // You may want to fill this based on additional token info you might have usd_price: 0, // Adjust if you need to calculate this value or pull from another source @@ -295,31 +314,8 @@ export const getUserTVLByBlock = async (blocks: BlockData) => { return convertToOutputDataSchema(data); }; -async function main() { - console.log(`Starting data fetching process mode: ${FIRST_TIME}`); - const blocks = await getBlockRangesToFetch(); - - let lastblock = 0; - try { - for (const block of blocks) { - lastblock = block; - const blockData = await getUserTVLByBlock({ - blockNumber: block, - blockTimestamp: 0, - }); - console.log("Processed block", block); - await saveToCSV(blockData); - } - } catch (error: any) { - console.error("Error processing block", lastblock, error.message); - } finally { - saveLastProcessedBlock(lastblock); - } -} - // IMPORTANT: config::FIRST_TIME is set to true be default // after inital fetch set it to false -// main().catch(console.error); const readBlocksFromCSV = async (filePath: string): Promise => { const blocks: BlockData[] = []; diff --git a/adapters/sparta/src/sdk/queries.ts b/adapters/sparta/src/sdk/queries.ts index bcae14e3..c0a3ec43 100644 --- a/adapters/sparta/src/sdk/queries.ts +++ b/adapters/sparta/src/sdk/queries.ts @@ -1,8 +1,12 @@ import { gql } from "@apollo/client"; export const TRANSFERS_QUERY = gql` - query GetLiquidityTransfers($blockNumber: Int!) { - transfers(where: { block_number_lte: $blockNumber }) { + query GetLiquidityTransfers($blockNumber: Int!, $first: Int!, $skip: Int!) { + transfers( + first: $first + skip: $skip + where: { block_number_lte: $blockNumber } + ) { from to value diff --git a/adapters/teahouse/src/index.ts b/adapters/teahouse/src/index.ts index 633d06be..b9b1d66d 100644 --- a/adapters/teahouse/src/index.ts +++ b/adapters/teahouse/src/index.ts @@ -39,7 +39,7 @@ const pipeline = promisify(stream.pipeline); const getData = async () => { const blocks = [ - 4174101 + 4368847 ]; //await readBlocksFromCSV('src/sdk/mode_chain_daily_blocks.csv'); const csvRows: OutputDataSchemaRow[] = []; diff --git a/adapters/teahouse/src/sdk/config.ts b/adapters/teahouse/src/sdk/config.ts index e211e2e3..14b7184d 100644 --- a/adapters/teahouse/src/sdk/config.ts +++ b/adapters/teahouse/src/sdk/config.ts @@ -2,7 +2,7 @@ import { createPublicClient, http } from "viem"; import { linea } from "viem/chains" -export const V3_SUBGRAPH_URL = "https://api.goldsky.com/api/public/project_clu5ow773st3501un98cv0861/subgraphs/TeavaultV3PairLinea-linea/1.0/gn"; +export const V3_SUBGRAPH_URL = "https://api.goldsky.com/api/public/project_clu5ow773st3501un98cv0861/subgraphs/TeavaultV3PairLinea-linea/surge/gn"; export const client = createPublicClient({ chain: linea, diff --git a/adapters/teahouse/src/sdk/vaults.ts b/adapters/teahouse/src/sdk/vaults.ts index fc40802e..a04ff07b 100644 --- a/adapters/teahouse/src/sdk/vaults.ts +++ b/adapters/teahouse/src/sdk/vaults.ts @@ -5,5 +5,6 @@ export const VAULT_ADDRESS = [ "0x07811284e36fDc45f65cd56FC7c6929855d6A0cc", "0x73d9ccd3017b41e9b29f1e4a49d5468b52bd17c6", "0x7d372Cc969211502D5C3a5721a85fc382f83bC8F", - "0x172Dba015dDfA642a3E3e0e8BaB040468D8D9879" + "0x172Dba015dDfA642a3E3e0e8BaB040468D8D9879", + "0x1adC5E10933b696FA5311DB5339F9a15E959e2B5" ]; \ No newline at end of file