From 7cbed125d79964b3065026ccce11e3dc0d1db27c Mon Sep 17 00:00:00 2001 From: timwang Date: Wed, 10 Apr 2024 16:41:40 +0800 Subject: [PATCH 1/4] Add dyson finance adapter --- adapters/dysonfinance/dex/package.json | 24 +++ adapters/dysonfinance/dex/src/index.ts | 258 ++++++++++++++++++++++++ adapters/dysonfinance/dex/tsconfig.json | 101 ++++++++++ 3 files changed, 383 insertions(+) create mode 100644 adapters/dysonfinance/dex/package.json create mode 100644 adapters/dysonfinance/dex/src/index.ts create mode 100644 adapters/dysonfinance/dex/tsconfig.json diff --git a/adapters/dysonfinance/dex/package.json b/adapters/dysonfinance/dex/package.json new file mode 100644 index 00000000..3fb155bc --- /dev/null +++ b/adapters/dysonfinance/dex/package.json @@ -0,0 +1,24 @@ +{ + "name": "dyson-finance", + "version": "1.0.0", + "description": "Accessible & Profitable Dex for All", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node dist/index.js", + "dev": "ts-node src/index.ts", + "compile": "tsc", + "watch": "tsc -w", + "clear": "rm -rf dist" + }, + "author": "eng@dyson.finance", + "license": "ISC", + "dependencies": { + "fast-csv": "^5.0.1" + }, + "devDependencies": { + "@types/node": "^20.12.6", + "ts-node": "10.9.2", + "typescript": "^5.4.4" + } +} diff --git a/adapters/dysonfinance/dex/src/index.ts b/adapters/dysonfinance/dex/src/index.ts new file mode 100644 index 00000000..7122c902 --- /dev/null +++ b/adapters/dysonfinance/dex/src/index.ts @@ -0,0 +1,258 @@ +import * as fs from "fs"; +import { write } from "fast-csv"; + +/** + * The objective is to quantify: + * - TVL on Linea (size of collateral minting GRAI on Linea) + * - GRAI stability pool deposits on Linea + * + * For that, we'll be querying an existing Gravita Subgraph deployed on TheGraph. + */ + +type OutputDataSchemaRow = { + block_number: number; + timestamp: number; + user_address: string; + token_address: string; + token_balance: number; + token_symbol: string; + usd_price: number; +}; + +const LINEA_RPC = "https://rpc.linea.build"; + + +const DYSON_SUBGRAPH_QUERY_URL = + "https://api.goldsky.com/api/public/project_clus3ghosysc701v046du9c18/subgraphs/linea-dyson/1.0.0/gn"; + +const DYSON_POOLS_QUERY = ` + query allPairs($blockNumber: Int!) { + pairEntities(first: 100, block: {number: $blockNumber}) { + id + quoteToken + token0Address + token0Decimals + token0Name + token1Address + token1Decimals + token1Name + } + } +`; + +const DYSON_POSITIONS_QUERY = ` + query DysonPositionsQuery($blockNumber: Int!) { + noteEntities(first: 1000, skip: 0, where: {isWithdrawed: false}, block: {number: $blockNumber}, orderBy: timestamp, orderDirection: desc) { + timestamp + user + pair + depositType + depositAmt + token0Amt + token1Amt + isWithdrawed + } + } +`; + + + +const post = async (url: string, data: any): Promise => { + const response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(data), + }); + return await response.json(); +}; + +const getLatestBlockNumberAndTimestamp = async () => { + const data = await post(LINEA_RPC, { + jsonrpc: "2.0", + method: "eth_getBlockByNumber", + params: ["latest", false], + id: 1, + }); + const blockNumber = parseInt(data.result.number); + const blockTimestamp = parseInt(data.result.timestamp); + return { blockNumber, blockTimestamp }; +}; + +const getBlockTimestamp = async (number: number): Promise => { + const hexBlockNumber = "0x" + number.toString(16); // Convert decimal block number to hexadecimal + console.log(hexBlockNumber); + const data = await post(LINEA_RPC, { + jsonrpc: "2.0", + method: "eth_getBlockByNumber", + params: [hexBlockNumber, false], + id: 1, + }); + const blockTimestampInt = parseInt(data.result.timestamp); + return blockTimestampInt; +}; + +const getPoolsData = async ( + blockNumber: number, + blockTimestamp: number, +): Promise> => { + const queryBlock = blockNumber > 675342 ? blockNumber : 675342; + const responseJson = await post(DYSON_SUBGRAPH_QUERY_URL, { + query: DYSON_POOLS_QUERY, + variables: { blockNumber: queryBlock }, + }); + const poolMap = new Map(); + + const poolArray = responseJson?.data?.pairEntities || []; + + for (let index = 0; index < poolArray.length; index++) { + const element = poolArray[index]; + poolMap.set(element?.id, element); + } + return poolMap; +}; + +function formatUnits(value: bigint, decimals: number) { + let display = value.toString() + + const negative = display.startsWith('-') + if (negative) display = display.slice(1) + + display = display.padStart(decimals, '0') + + let [integer, fraction] = [ + display.slice(0, display.length - decimals), + display.slice(display.length - decimals), + ] + fraction = fraction.replace(/(0+)$/, '') + return `${negative ? '-' : ''}${integer || '0'}${ + fraction ? `.${fraction}` : '' + }` + } + +interface SumPosition { + user: string; + tokenAddress: string; + tokenBalance: number; + tokenSymbol: string; +} + +const getPositionData = async ( + blockNumber: number, + blockTimestamp: number, +): Promise => { + const queryBlock = blockNumber > 675341 ? blockNumber : 675341; + const poolMap = await getPoolsData(blockNumber, blockTimestamp); + const responseJson = await post(DYSON_SUBGRAPH_QUERY_URL, { + query: DYSON_POSITIONS_QUERY, + variables: { blockNumber: queryBlock }, + }); + const userPositionMap = new Map(); + const positionsArray = responseJson?.data?.noteEntities || []; + + for (let index = 0; index < positionsArray.length; index++) { + const element = positionsArray[index]; + const pairData = poolMap.get(element.pair.toLowerCase()); + const depositToken = + element.depositType === 0 + ? { + tokenAddress: pairData.token0Address, + tokenSymbol: pairData.token0Name, + decimals: parseFloat(pairData.token0Decimals) + } + : { + tokenAddress: pairData.token1Address, + tokenSymbol: pairData.token1Name, + decimals: parseFloat(pairData.token1Decimals) + }; + const id = `${element.user}-${depositToken.tokenAddress}` + let position: SumPosition | undefined = userPositionMap.get(id) + const depositAmt = parseFloat(formatUnits(element.depositAmt, depositToken.decimals)) + if(position){ + position.tokenBalance += depositAmt + } else { + position = {user: element.user, + tokenAddress: depositToken.tokenAddress, + tokenBalance: depositAmt, + tokenSymbol: depositToken.tokenSymbol} + } + userPositionMap.set( + id, + position, + ); + } + + const keyIterator = userPositionMap.keys() + const csvRows: OutputDataSchemaRow[] = []; + for (const key of keyIterator) { + const userPosition = userPositionMap.get(key) + if(userPosition){ + csvRows.push({ + block_number: blockNumber, + timestamp: blockTimestamp, + user_address: userPosition.user, + token_address: userPosition.tokenAddress, + token_balance: userPosition.tokenBalance, + token_symbol: userPosition.tokenSymbol, + usd_price: 0, + }); + } + + } + return csvRows; +}; + +interface BlockData { + blockNumber: number; + blockTimestamp: number; +} + +export const main = async (blocks: BlockData[]) => { + const allCsvRows: any[] = []; // Array to accumulate CSV rows for all blocks + const batchSize = 10; // Size of batch to trigger writing to the file + let i = 0; + + for (const { blockNumber, blockTimestamp } of blocks) { + try { + // Retrieve data using block number and timestamp + const csvRows = await getPositionData(blockNumber, blockTimestamp); + + // Accumulate CSV rows for all blocks + allCsvRows.push(...csvRows); + + i++; + console.log(`Processed block ${i}`); + + // Write to file when batch size is reached or at the end of loop + if (i % batchSize === 0 || i === blocks.length) { + const ws = fs.createWriteStream(`outputData.csv`, { + flags: i === batchSize ? "w" : "a", + }); + write(allCsvRows, { headers: i === batchSize ? true : false }) + .pipe(ws) + .on("finish", () => { + console.log(`CSV file has been written.`); + }); + + // Clear the accumulated CSV rows + allCsvRows.length = 0; + } + } catch (error) { + console.error(`An error occurred for block ${blockNumber}:`, error); + } + } +}; + +export const getUserTVLByBlock = async (blocks: BlockData) => { + const { blockNumber, blockTimestamp } = blocks; + // Retrieve data using block number and timestamp + const csvRows = await getPositionData(blockNumber, blockTimestamp); + return csvRows; +}; + +// main([{blockNumber: 799999, +// blockTimestamp: 1201203123123}]).then(() => { +// console.log("Done"); +// }); diff --git a/adapters/dysonfinance/dex/tsconfig.json b/adapters/dysonfinance/dex/tsconfig.json new file mode 100644 index 00000000..060575df --- /dev/null +++ b/adapters/dysonfinance/dex/tsconfig.json @@ -0,0 +1,101 @@ +{ + "compilerOptions": { + /* Visit https://aka.ms/tsconfig to read more about this file */ + /* Projects */ + // "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ + // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ + // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ + // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ + // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ + // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ + /* Language and Environment */ + "target": "es2022", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ + // "jsx": "preserve", /* Specify what JSX code is generated. */ + // "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */ + // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ + // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */ + // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ + // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */ + // "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */ + // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ + // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ + // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ + /* Modules */ + "module": "commonjs", /* Specify what module code is generated. */ + "rootDir": "src/", /* Specify the root folder within your source files. */ + // "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */ + // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ + // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ + // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ + // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */ + // "types": [], /* Specify type package names to be included without being referenced in a source file. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */ + // "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */ + // "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */ + // "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */ + // "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */ + // "resolveJsonModule": true, /* Enable importing .json files. */ + // "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */ + // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ + /* JavaScript Support */ + // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ + // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ + // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ + /* Emit */ + // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + // "declarationMap": true, /* Create sourcemaps for d.ts files. */ + // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ + // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ + // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ + "outDir": "dist/", /* Specify an output folder for all emitted files. */ + // "removeComments": true, /* Disable emitting comments. */ + // "noEmit": true, /* Disable emitting files from a compilation. */ + // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ + // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types. */ + "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ + // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ + // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ + // "newLine": "crlf", /* Set the newline character for emitting files. */ + // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ + // "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */ + // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ + // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */ + // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ + // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ + /* Interop Constraints */ + // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ + // "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */ + // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ + "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ + // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ + "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ + /* Type Checking */ + "strict": true, /* Enable all strict type-checking options. */ + // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ + // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ + // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + // "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ + // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ + // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ + // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ + // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ + // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ + // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ + // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + /* Completeness */ + // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + } +} \ No newline at end of file From e98b2e8a25670e94c4a7b59505bee70503869ccd Mon Sep 17 00:00:00 2001 From: timwang Date: Thu, 18 Apr 2024 02:03:21 +0800 Subject: [PATCH 2/4] Modify folder stucture --- adapters/dysonfinance/{dex => }/package.json | 0 adapters/dysonfinance/{dex => }/src/index.ts | 0 adapters/dysonfinance/{dex => }/tsconfig.json | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename adapters/dysonfinance/{dex => }/package.json (100%) rename adapters/dysonfinance/{dex => }/src/index.ts (100%) rename adapters/dysonfinance/{dex => }/tsconfig.json (100%) diff --git a/adapters/dysonfinance/dex/package.json b/adapters/dysonfinance/package.json similarity index 100% rename from adapters/dysonfinance/dex/package.json rename to adapters/dysonfinance/package.json diff --git a/adapters/dysonfinance/dex/src/index.ts b/adapters/dysonfinance/src/index.ts similarity index 100% rename from adapters/dysonfinance/dex/src/index.ts rename to adapters/dysonfinance/src/index.ts diff --git a/adapters/dysonfinance/dex/tsconfig.json b/adapters/dysonfinance/tsconfig.json similarity index 100% rename from adapters/dysonfinance/dex/tsconfig.json rename to adapters/dysonfinance/tsconfig.json From 73592371b1a610673f63b1072ed9f274c1893b81 Mon Sep 17 00:00:00 2001 From: timwang Date: Mon, 22 Apr 2024 19:47:48 +0800 Subject: [PATCH 3/4] Use real number --- adapters/dysonfinance/src/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/adapters/dysonfinance/src/index.ts b/adapters/dysonfinance/src/index.ts index 7122c902..be7ecab4 100644 --- a/adapters/dysonfinance/src/index.ts +++ b/adapters/dysonfinance/src/index.ts @@ -14,7 +14,7 @@ type OutputDataSchemaRow = { timestamp: number; user_address: string; token_address: string; - token_balance: number; + token_balance: bigint; token_symbol: string; usd_price: number; }; @@ -135,7 +135,7 @@ function formatUnits(value: bigint, decimals: number) { interface SumPosition { user: string; tokenAddress: string; - tokenBalance: number; + tokenBalance: bigint; tokenSymbol: string; } @@ -169,7 +169,7 @@ const getPositionData = async ( }; const id = `${element.user}-${depositToken.tokenAddress}` let position: SumPosition | undefined = userPositionMap.get(id) - const depositAmt = parseFloat(formatUnits(element.depositAmt, depositToken.decimals)) + const depositAmt = BigInt(element.depositAmt) if(position){ position.tokenBalance += depositAmt } else { From 7d69c19e9b3aa272e833430271fd8bf323f79bb3 Mon Sep 17 00:00:00 2001 From: timwang Date: Wed, 5 Jun 2024 17:29:38 +0800 Subject: [PATCH 4/4] Fix pagination issue --- adapters/dysonfinance/src/index.ts | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/adapters/dysonfinance/src/index.ts b/adapters/dysonfinance/src/index.ts index be7ecab4..a4c77508 100644 --- a/adapters/dysonfinance/src/index.ts +++ b/adapters/dysonfinance/src/index.ts @@ -41,8 +41,8 @@ const DYSON_POOLS_QUERY = ` `; const DYSON_POSITIONS_QUERY = ` - query DysonPositionsQuery($blockNumber: Int!) { - noteEntities(first: 1000, skip: 0, where: {isWithdrawed: false}, block: {number: $blockNumber}, orderBy: timestamp, orderDirection: desc) { + query DysonPositionsQuery($blockNumber: Int!, $interval: Int!, $offset: Int!) { + noteEntities(first: $interval, skip: $offset, where: {isWithdrawed: false}, block: {number: $blockNumber}, orderBy: timestamp, orderDirection: desc) { timestamp user pair @@ -83,7 +83,6 @@ const getLatestBlockNumberAndTimestamp = async () => { const getBlockTimestamp = async (number: number): Promise => { const hexBlockNumber = "0x" + number.toString(16); // Convert decimal block number to hexadecimal - console.log(hexBlockNumber); const data = await post(LINEA_RPC, { jsonrpc: "2.0", method: "eth_getBlockByNumber", @@ -139,18 +138,31 @@ interface SumPosition { tokenSymbol: string; } +const fetchingAllPositionData = async (queryBlock: number, interval = 1000) => { + let hasMore = true; + let offset = 0; + let positionsArray: any[] = []; + while (hasMore) { + const responseJson = await post(DYSON_SUBGRAPH_QUERY_URL, { + query: DYSON_POSITIONS_QUERY, + variables: { blockNumber: queryBlock, interval, offset }, + }); + const partPositionsArray = responseJson?.data?.noteEntities as any[] || []; + hasMore = partPositionsArray.length === interval; + offset += interval; + positionsArray = positionsArray.concat(partPositionsArray); + } + return positionsArray; +}; + const getPositionData = async ( blockNumber: number, blockTimestamp: number, ): Promise => { const queryBlock = blockNumber > 675341 ? blockNumber : 675341; const poolMap = await getPoolsData(blockNumber, blockTimestamp); - const responseJson = await post(DYSON_SUBGRAPH_QUERY_URL, { - query: DYSON_POSITIONS_QUERY, - variables: { blockNumber: queryBlock }, - }); const userPositionMap = new Map(); - const positionsArray = responseJson?.data?.noteEntities || []; + const positionsArray = await fetchingAllPositionData(queryBlock) for (let index = 0; index < positionsArray.length; index++) { const element = positionsArray[index];