From ca72e8983bd6ac072629a48e2536ae33fadf6c32 Mon Sep 17 00:00:00 2001 From: trungbach Date: Mon, 21 Aug 2023 15:23:24 +0700 Subject: [PATCH 01/46] style: remove comment not use in sync --- packages/oraidex-sync/src/index.ts | 49 ++++++------------------------ 1 file changed, 9 insertions(+), 40 deletions(-) diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index b5e6897b..72e10cfb 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -58,24 +58,6 @@ class WriteOrders extends WriteData { async process(chunk: any): Promise { try { - // // first time calling of the application then we query past data and be ready to store them into the db for prefix sum - // // this helps the flow go smoothly and remove dependency between different streams - // if (this.firstWrite) { - // console.log("initial data: ", this.initialData); - // const { height, time } = this.initialData.blockHeader; - // await this.duckDb.insertPriceInfos( - // this.initialData.tokenPrices.map( - // (tokenPrice) => - // ({ - // txheight: height, - // timestamp: time, - // assetInfo: parseAssetInfo(tokenPrice.info), - // price: parseInt(tokenPrice.amount) - // } as PriceInfo) - // ) - // ); - // this.firstWrite = false; - // } const { txs, offset: newOffset } = chunk as Txs; const currentOffset = await this.duckDb.loadHeightSnapshot(); // edge case. If no new block has been found, then we skip processing to prevent duplication handling @@ -84,8 +66,6 @@ class WriteOrders extends WriteData { // accumulate liquidity pool amount await this.accumulatePoolAmount([...result.provideLiquidityOpsData, ...result.withdrawLiquidityOpsData]); - // process volume infos to insert price - // result.volumeInfos = insertVolumeInfos(result.swapOpsData); // collect the latest offer & ask volume to accumulate the results // insert txs @@ -105,6 +85,7 @@ class WriteOrders extends WriteData { } } +// we need to create a new table with name PoolInfo, whenever order table class OraiDexSync { protected constructor( private readonly duckDb: DuckDb, @@ -154,24 +135,16 @@ class OraiDexSync { this.duckDb.createLiquidityOpsTable(), this.duckDb.createSwapOpsTable(), this.duckDb.createPairInfosTable(), - // this.duckDb.createPriceInfoTable(), this.duckDb.createSwapOhlcv() ]); let currentInd = await this.duckDb.loadHeightSnapshot(); let initialData: InitialData = { tokenPrices: [], blockHeader: undefined }; const initialSyncHeight = parseInt(process.env.INITIAL_SYNC_HEIGHT) || 12388825; - // // if its' the first time, then we use the height 12388825 since its the safe height for the rpc nodes to include timestamp & new indexing logic + // if its' the first time, then we use the height 12388825 since its the safe height for the rpc nodes to include timestamp & new indexing logic if (currentInd <= initialSyncHeight) { currentInd = initialSyncHeight; } console.log("current ind: ", currentInd); - - // const tokenPrices = await Promise.all( - // extractUniqueAndFlatten(pairs).map((info) => this.simulateSwapPrice(info, currentInd)) - // ); - // const initialBlockHeader = (await this.cosmwasmClient.getBlock(currentInd)).header; - // initialData.tokenPrices = tokenPrices; - // initialData.blockHeader = initialBlockHeader; await this.updateLatestPairInfos(); new SyncData({ offset: currentInd, @@ -187,17 +160,13 @@ class OraiDexSync { } } -// async function initSync() { -// const duckDb = await DuckDb.create(process.env.DUCKDB_PROD_FILENAME || "oraidex-sync-data"); -// const oraidexSync = await OraiDexSync.create( -// duckDb, -// process.env.RPC_URL || "https://rpc.orai.io", -// process.env as any -// ); -// oraidexSync.sync(); -// } - -// initSync(); +async function initSync() { + const duckDb = await DuckDb.create("oraidex-only-sync-data"); + const oraidexSync = await OraiDexSync.create(duckDb, "http://35.237.59.125:26657", process.env as any); + oraidexSync.sync(); +} + +initSync(); export { OraiDexSync }; export * from "./types"; From cd752d9e35f90743e4d0549822d73369e1e9af34 Mon Sep 17 00:00:00 2001 From: trungbach Date: Mon, 21 Aug 2023 18:14:27 +0700 Subject: [PATCH 02/46] add: added api get pools --- package.json | 2 +- packages/oraidex-server/src/index.ts | 9 +++++++ packages/oraidex-sync/src/db.ts | 21 ++++++++-------- packages/oraidex-sync/src/helper.ts | 11 ++++++++- packages/oraidex-sync/src/index.ts | 37 +++++++++++++++++++++++++--- packages/oraidex-sync/src/types.ts | 7 ++++++ 6 files changed, 72 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index 8b62e0da..cf17abc1 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "docs": "typedoc --entryPointStrategy expand --name 'Oraidex SDK' --readme none --tsconfig packages/contracts-sdk/tsconfig.json packages/contracts-sdk/src", "build": "tsc -p", "deploy": "yarn publish --access public", - "start:server": "yarn build packages/oraidex-sync/ && npx ts-node packages/oraidex-server/src/index.ts" + "start:server": "yarn build packages/oraidex-sync/ && npx ts-node-dev packages/oraidex-server/src/index.ts" }, "workspaces": [ "packages/*" diff --git a/packages/oraidex-server/src/index.ts b/packages/oraidex-server/src/index.ts index 20e62c04..9667d781 100644 --- a/packages/oraidex-server/src/index.ts +++ b/packages/oraidex-server/src/index.ts @@ -231,6 +231,15 @@ app.get("/v1/candles/", async (req: Request<{}, {}, {}, GetCandlesQuery>, res) = } }); +app.get("/v1/pools/", async (req, res) => { + try { + const pools = await duckDb.getPools(); + res.status(200).send(pools); + } catch (error) { + res.status(500).send(error.message); + } +}); + app.listen(port, hostname, async () => { // sync data for the service to read // console.dir(pairInfos, { depth: null }); diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index b2cd90ab..92bf3eb1 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -123,6 +123,13 @@ export class DuckDb { pairAddr VARCHAR, liquidityAddr VARCHAR, oracleAddr VARCHAR, + symbols VARCHAR, + fromIconUrl VARCHAR, + toIconUrl VARCHAR, + volume24Hour UBIGINT, + apr DOUBLE, + totalLiquidity UBIGINT, + fee7Days UBIGINT, PRIMARY KEY (pairAddr) )` ); } @@ -131,16 +138,6 @@ export class DuckDb { await this.insertBulkData(ops, "pair_infos", true); } - async createPriceInfoTable() { - await this.conn.exec( - `CREATE TABLE IF NOT EXISTS price_infos ( - txheight UINTEGER, - timestamp UINTEGER, - assetInfo VARCHAR, - price UINTEGER)` - ); - } - async insertPriceInfos(ops: PriceInfo[]) { await this.insertBulkData(ops, "price_infos", false, `price_infos-${Math.random() * 1000}`); } @@ -363,4 +360,8 @@ export class DuckDb { time: new Date(res.time * tf * 1000).toISOString() })) as VolumeRange[]; } + + async getPools(): Promise { + return (await this.conn.all("SELECT * from pair_infos")).map((data) => data as PairInfoData); + } } diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index 84ff99f2..f333dfa6 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -378,6 +378,14 @@ export function findPairIndexFromDenoms(offerDenom: string, askDenom: string): n // return { baseIndex: 1, targetIndex: 0, target: infos[0] }; // default we calculate the first info in the asset info list // } +function getSymbolFromAsset(asset_infos: [AssetInfo, AssetInfo]): string { + const findedPair = pairs.find((p) => JSON.stringify(p.asset_infos) === JSON.stringify(asset_infos)); + if (!findedPair) { + throw new Error(`cannot found pair with asset_infos: ${asset_infos}`); + } + return findedPair.symbols.join("/"); +} + export { findMappedTargetedAssetInfo, findAssetInfoPathToUsdt, @@ -386,5 +394,6 @@ export { parseAssetInfoOnlyDenom, delay, findPairAddress, - calculatePriceByPool + calculatePriceByPool, + getSymbolFromAsset }; diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index 72e10cfb..13bff219 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -2,7 +2,7 @@ import "dotenv/config"; import { parseAssetInfo, parseTxs } from "./tx-parsing"; import { DuckDb } from "./db"; import { WriteData, SyncData, Txs } from "@oraichain/cosmos-rpc-sync"; -import { CosmWasmClient, OraiswapFactoryQueryClient, PairInfo } from "@oraichain/oraidex-contracts-sdk"; +import { AssetInfo, CosmWasmClient, OraiswapFactoryQueryClient, PairInfo } from "@oraichain/oraidex-contracts-sdk"; import { ProvideLiquidityOperationData, TxAnlysisResult, @@ -14,7 +14,7 @@ import { import { MulticallQueryClient } from "@oraichain/common-contracts-sdk"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; import { getAllPairInfos, getPoolInfos } from "./query"; -import { collectAccumulateLpData } from "./helper"; +import { collectAccumulateLpData, getSymbolFromAsset } from "./helper"; class WriteOrders extends WriteData { private firstWrite: boolean; @@ -112,6 +112,30 @@ class OraiDexSync { } private async updateLatestPairInfos() { + const pairInfos = await this.getAllPairInfos(); + await this.duckDb.insertPairInfos( + pairInfos.map((pair) => { + const symbols = getSymbolFromAsset(pair.asset_infos); + return { + firstAssetInfo: parseAssetInfo(pair.asset_infos[0]), + secondAssetInfo: parseAssetInfo(pair.asset_infos[1]), + commissionRate: pair.commission_rate, + pairAddr: pair.contract_addr, + liquidityAddr: pair.liquidity_token, + oracleAddr: pair.oracle_addr, + symbols, + fromIconUrl: "url1", + toIconUrl: "url2", + volume24Hour: 1n, + apr: 2, + totalLiquidity: 1n, + fee7Days: 1n + } as PairInfoData; + }) + ); + } + + private async initPairInfos() { const pairInfos = await this.getAllPairInfos(); await this.duckDb.insertPairInfos( pairInfos.map( @@ -122,7 +146,14 @@ class OraiDexSync { commissionRate: pair.commission_rate, pairAddr: pair.contract_addr, liquidityAddr: pair.liquidity_token, - oracleAddr: pair.oracle_addr + oracleAddr: pair.oracle_addr, + symbols: "orai", + fromIconUrl: "url1", + toIconUrl: "url2", + volume24Hour: 1n, + apr: 2, + totalLiquidity: 1n, + fee7Days: 1n } as PairInfoData) ) ); diff --git a/packages/oraidex-sync/src/types.ts b/packages/oraidex-sync/src/types.ts index d6e3cfd4..40288196 100644 --- a/packages/oraidex-sync/src/types.ts +++ b/packages/oraidex-sync/src/types.ts @@ -41,6 +41,13 @@ export type PairInfoData = { pairAddr: string; liquidityAddr: string; oracleAddr: string; + symbols: string; + fromIconUrl: string; + toIconUrl: string; + volume24Hour: bigint; + apr: number; + totalLiquidity: bigint; + fee7Days: bigint; }; export type PriceInfo = { From c19b076c44aed81f13e436489f025973c9e1111a Mon Sep 17 00:00:00 2001 From: trungbach Date: Tue, 22 Aug 2023 17:31:53 +0700 Subject: [PATCH 03/46] add: calculated total liquidity for pool infos --- packages/oraidex-sync/src/db.ts | 2 +- packages/oraidex-sync/src/helper.ts | 69 ++++++++++++++++++---- packages/oraidex-sync/src/index.ts | 37 ++++-------- packages/oraidex-sync/src/query.ts | 2 +- packages/oraidex-sync/src/types.ts | 2 +- packages/oraidex-sync/tests/helper.spec.ts | 25 ++++++-- 6 files changed, 91 insertions(+), 46 deletions(-) diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index 92bf3eb1..bef81745 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -128,7 +128,7 @@ export class DuckDb { toIconUrl VARCHAR, volume24Hour UBIGINT, apr DOUBLE, - totalLiquidity UBIGINT, + totalLiquidity UINT64, fee7Days UBIGINT, PRIMARY KEY (pairAddr) )` ); diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index f333dfa6..46b80ef2 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -1,4 +1,11 @@ -import { AssetInfo, SwapOperation } from "@oraichain/oraidex-contracts-sdk"; +import { + AssetInfo, + CosmWasmClient, + OraiswapPairQueryClient, + OraiswapPairTypes, + OraiswapRouterQueryClient, + SwapOperation +} from "@oraichain/oraidex-contracts-sdk"; import { pairs, pairsOnlyDenom } from "./pairs"; import { ORAI, atomic, tenAmountInDecimalSix, truncDecimals, usdtCw20Address } from "./constants"; import { @@ -11,7 +18,8 @@ import { WithdrawLiquidityOperationData } from "./types"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; -import { minBy, maxBy } from "lodash"; +import { minBy, maxBy, isEqual } from "lodash"; +import { getPoolInfos, simulateSwapPriceWithUsdt } from "./query"; export function toObject(data: any) { return JSON.parse( @@ -161,15 +169,15 @@ function findPairAddress(pairInfos: PairInfoData[], infos: [AssetInfo, AssetInfo } function calculatePriceByPool( - basePool: bigint, - quotePool: bigint, + offerPool: bigint, + askPool: bigint, commissionRate?: number, offerAmount?: number ): number { const finalOfferAmount = offerAmount || tenAmountInDecimalSix; - let bigIntAmount = Number( - (basePool - (quotePool * basePool) / (quotePool + BigInt(finalOfferAmount))) * BigInt(1 - commissionRate || 0) - ); + let bigIntAmount = + Number(offerPool - (askPool * offerPool) / (askPool + BigInt(finalOfferAmount))) * (1 - commissionRate || 0); + return bigIntAmount / finalOfferAmount; } @@ -379,13 +387,51 @@ export function findPairIndexFromDenoms(offerDenom: string, askDenom: string): n // } function getSymbolFromAsset(asset_infos: [AssetInfo, AssetInfo]): string { - const findedPair = pairs.find((p) => JSON.stringify(p.asset_infos) === JSON.stringify(asset_infos)); + const findedPair = pairs.find( + (p) => + JSON.stringify(p.asset_infos) === JSON.stringify(asset_infos) || + JSON.stringify(p.asset_infos) === JSON.stringify(asset_infos.reverse()) + ); if (!findedPair) { - throw new Error(`cannot found pair with asset_infos: ${asset_infos}`); + throw new Error(`cannot found pair with asset_infos: ${JSON.stringify(asset_infos)}`); } return findedPair.symbols.join("/"); } +async function getCosmwasmClient(): Promise { + const rpcUrl = process.env.RPC_URL || "http://35.237.59.125:26657"; + const client = await CosmWasmClient.connect(rpcUrl); + return client; +} + +function parsePoolAmount(poolInfo: OraiswapPairTypes.PoolResponse, trueAsset: AssetInfo): bigint { + return BigInt(poolInfo.assets.find((asset) => isEqual(asset.info, trueAsset))?.amount || "0"); +} + +type PoolInfo = { + offerPoolAmount: bigint; + askPoolAmount: bigint; +}; +async function fetchPoolInfoAmount(fromInfo: AssetInfo, toInfo: AssetInfo, pairAddr: string): Promise { + const client = await getCosmwasmClient(); + const pairContract = new OraiswapPairQueryClient(client, pairAddr); + const poolInfo = await pairContract.pool(); + const offerPoolAmount = parsePoolAmount(poolInfo, fromInfo); + const askPoolAmount = parsePoolAmount(poolInfo, toInfo); + return { offerPoolAmount, askPoolAmount }; +} + +async function getPairLiquidity([fromInfo, toInfo]: [AssetInfo, AssetInfo], pairAddr: string): Promise { + const { offerPoolAmount, askPoolAmount } = await fetchPoolInfoAmount(fromInfo, toInfo, pairAddr); + const routerContract = new OraiswapRouterQueryClient( + await getCosmwasmClient(), + process.env.ROUTER_CONTRACT_ADDRESS || "orai1j0r67r9k8t34pnhy00x3ftuxuwg0r6r4p8p6rrc8az0ednzr8y9s3sj2sf" + ); + const { amount } = await simulateSwapPriceWithUsdt(fromInfo, routerContract); + const totalLiquid = Number(amount) * Number(offerPoolAmount) * 2; + return totalLiquid; +} + export { findMappedTargetedAssetInfo, findAssetInfoPathToUsdt, @@ -395,5 +441,8 @@ export { delay, findPairAddress, calculatePriceByPool, - getSymbolFromAsset + getSymbolFromAsset, + getPoolInfos, + fetchPoolInfoAmount, + getPairLiquidity }; diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index 13bff219..916758ec 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -14,7 +14,7 @@ import { import { MulticallQueryClient } from "@oraichain/common-contracts-sdk"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; import { getAllPairInfos, getPoolInfos } from "./query"; -import { collectAccumulateLpData, getSymbolFromAsset } from "./helper"; +import { collectAccumulateLpData, getPairLiquidity, getSymbolFromAsset } from "./helper"; class WriteOrders extends WriteData { private firstWrite: boolean; @@ -113,8 +113,15 @@ class OraiDexSync { private async updateLatestPairInfos() { const pairInfos = await this.getAllPairInfos(); - await this.duckDb.insertPairInfos( + console.time("timer"); + const allLiquidities = await Promise.all( pairInfos.map((pair) => { + return getPairLiquidity(pair.asset_infos, pair.contract_addr); + }) + ); + console.timeEnd("timer"); + await this.duckDb.insertPairInfos( + pairInfos.map((pair, index) => { const symbols = getSymbolFromAsset(pair.asset_infos); return { firstAssetInfo: parseAssetInfo(pair.asset_infos[0]), @@ -128,37 +135,13 @@ class OraiDexSync { toIconUrl: "url2", volume24Hour: 1n, apr: 2, - totalLiquidity: 1n, + totalLiquidity: allLiquidities[index], fee7Days: 1n } as PairInfoData; }) ); } - private async initPairInfos() { - const pairInfos = await this.getAllPairInfos(); - await this.duckDb.insertPairInfos( - pairInfos.map( - (pair) => - ({ - firstAssetInfo: parseAssetInfo(pair.asset_infos[0]), - secondAssetInfo: parseAssetInfo(pair.asset_infos[1]), - commissionRate: pair.commission_rate, - pairAddr: pair.contract_addr, - liquidityAddr: pair.liquidity_token, - oracleAddr: pair.oracle_addr, - symbols: "orai", - fromIconUrl: "url1", - toIconUrl: "url2", - volume24Hour: 1n, - apr: 2, - totalLiquidity: 1n, - fee7Days: 1n - } as PairInfoData) - ) - ); - } - public async sync() { try { await Promise.all([ diff --git a/packages/oraidex-sync/src/query.ts b/packages/oraidex-sync/src/query.ts index 68fc1f28..4ba8ec2a 100644 --- a/packages/oraidex-sync/src/query.ts +++ b/packages/oraidex-sync/src/query.ts @@ -57,7 +57,7 @@ async function simulateSwapPriceWithUsdt(info: AssetInfo, router: OraiswapRouter * Simulate price for pair[0]/pair[pair.length - 1] where the amount of pair[0] is 10^7. This is a multihop simulate swap function. The asset infos in between of the array are for hopping * @param pairPath - the path starting from the offer asset info to the ask asset info * @param router - router contract - * @returns - pricea fter simulating + * @returns - price after simulating */ async function simulateSwapPrice(pairPath: AssetInfo[], router: OraiswapRouterReadOnlyInterface): Promise { // usdt case, price is always 1 diff --git a/packages/oraidex-sync/src/types.ts b/packages/oraidex-sync/src/types.ts index 40288196..5f297315 100644 --- a/packages/oraidex-sync/src/types.ts +++ b/packages/oraidex-sync/src/types.ts @@ -46,7 +46,7 @@ export type PairInfoData = { toIconUrl: string; volume24Hour: bigint; apr: number; - totalLiquidity: bigint; + totalLiquidity: number; fee7Days: bigint; }; diff --git a/packages/oraidex-sync/tests/helper.spec.ts b/packages/oraidex-sync/tests/helper.spec.ts index 1de1cc18..3c21159e 100644 --- a/packages/oraidex-sync/tests/helper.spec.ts +++ b/packages/oraidex-sync/tests/helper.spec.ts @@ -189,7 +189,14 @@ describe("test-helper", () => { commissionRate: "", pairAddr: "orai1c5s03c3l336dgesne7dylnmhszw8554tsyy9yt", liquidityAddr: "", - oracleAddr: "" + oracleAddr: "", + symbols: "1", + fromIconUrl: "1", + toIconUrl: "1", + volume24Hour: 1n, + apr: 1, + totalLiquidity: 1, + fee7Days: 1n } ]; let assetInfos: [AssetInfo, AssetInfo] = [{ native_token: { denom: ORAI } }, assetInfo]; @@ -346,11 +353,17 @@ describe("test-helper", () => { ]); }); - it("test-calculatePriceByPool-ORAI/USDT-pool-when-1ORAI=2.74USDT", () => { - // base denom is ORAI, quote denom is USDT => base pool is ORAI, quote pool is USDT. - const result = calculatePriceByPool(BigInt(639997269712), BigInt(232967274783), 0, 10 ** 6); - expect(result.toString()).toEqual("2.747144"); - }); + it.each([ + [0, "2.747144"], + [0.003, "2.738902568"] + ])( + "test-calculatePriceByPool-ORAI/USDT-pool-with-commision-rate=%s-should-return-price-%s-USDT", + (commisionRate, expectedPrice) => { + // base denom is ORAI, quote denom is USDT => base pool is ORAI, quote pool is USDT. + const result = calculatePriceByPool(BigInt(639997269712), BigInt(232967274783), commisionRate, 10 ** 6); + expect(result.toString()).toEqual(expectedPrice); + } + ); it("test-collectAccumulateLpData-should-aggregate-ops-with-same-pairs", () => { const poolResponses: PoolResponse[] = [ From 4505883834beebc860275dfeabb7be85fe850957 Mon Sep 17 00:00:00 2001 From: trungbach Date: Tue, 22 Aug 2023 18:43:26 +0700 Subject: [PATCH 04/46] setup: get fee swap of pair --- packages/oraidex-sync/src/db.ts | 25 +++++++++++++++++++++++- packages/oraidex-sync/src/helper.ts | 20 ++++++++++++++++++- packages/oraidex-sync/src/index.ts | 30 ++++++++++++++++++++++++++++- packages/oraidex-sync/src/types.ts | 7 +++++++ 4 files changed, 79 insertions(+), 3 deletions(-) diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index bef81745..4fe85abc 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -9,7 +9,8 @@ import { VolumeData, VolumeRange, WithdrawLiquidityOperationData, - GetCandlesQuery + GetCandlesQuery, + GetFeeSwap } from "./types"; import fs, { rename } from "fs"; import { isoToTimestampNumber, renameKey, replaceAllNonAlphaBetChar, toObject } from "./helper"; @@ -364,4 +365,26 @@ export class DuckDb { async getPools(): Promise { return (await this.conn.all("SELECT * from pair_infos")).map((data) => data as PairInfoData); } + + async getFeeSwap(payload: GetFeeSwap): Promise { + const { offerDenom, askDenom, startTime, endTime } = payload; + console.log({ payload }); + const result = await this.conn.all( + ` + SELECT + sum(commissionAmount + taxAmount) as totalFee, + FROM swap_ops_data + WHERE timestamp >= ? + AND timestamp <= ? + AND offerDenom = ? + AND askDenom = ? + `, + startTime, + endTime, + offerDenom, + askDenom + ); + console.log({ result }); + return 1n; + } } diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index 46b80ef2..7ebc6626 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -432,6 +432,22 @@ async function getPairLiquidity([fromInfo, toInfo]: [AssetInfo, AssetInfo], pair return totalLiquid; } +/** + * + * @param time + * @param tf in seconds + * @returns + */ +function getSpecificDateBeforeNow(time: Date, tf: number) { + const timeInMs = tf * 1000; + const dateBeforeNow = new Date(time.getTime() - timeInMs); + return dateBeforeNow; +} + +function convertDateToSecond(date: Date): number { + return Math.round(date.valueOf() / 1000); +} + export { findMappedTargetedAssetInfo, findAssetInfoPathToUsdt, @@ -444,5 +460,7 @@ export { getSymbolFromAsset, getPoolInfos, fetchPoolInfoAmount, - getPairLiquidity + getPairLiquidity, + getSpecificDateBeforeNow, + convertDateToSecond }; diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index 916758ec..d2056f8d 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -14,7 +14,14 @@ import { import { MulticallQueryClient } from "@oraichain/common-contracts-sdk"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; import { getAllPairInfos, getPoolInfos } from "./query"; -import { collectAccumulateLpData, getPairLiquidity, getSymbolFromAsset } from "./helper"; +import { + collectAccumulateLpData, + convertDateToSecond, + getPairLiquidity, + getSpecificDateBeforeNow, + getSymbolFromAsset, + parseAssetInfoOnlyDenom +} from "./helper"; class WriteOrders extends WriteData { private firstWrite: boolean; @@ -111,6 +118,11 @@ class OraiDexSync { return getAllPairInfos(firstFactoryClient, secondFactoryClient); } + // fromIconUrl, toIconUrl: upload to other server + // volume24Hour: volume ohlcv + volume liquidity (?) + // apr: oraidex + // totalLiquidity: get liquidity in lp_ops_data with last block of pair + // fee7Days: sum of fee in swap_ops ( taxAmount + commissionAmount ) + & lp_ops fee of scatom/atom private async updateLatestPairInfos() { const pairInfos = await this.getAllPairInfos(); console.time("timer"); @@ -120,6 +132,22 @@ class OraiDexSync { }) ); console.timeEnd("timer"); + + const tf = 7 * 24 * 60 * 60; // second of 7 days + const currentDate = new Date(); + const oneWeekBeforeNow = getSpecificDateBeforeNow(new Date(), tf); + const allPairFees = await Promise.all( + pairInfos.map((pair) => { + return this.duckDb.getFeeSwap({ + offerDenom: parseAssetInfoOnlyDenom(pair.asset_infos[0]), + askDenom: parseAssetInfoOnlyDenom(pair.asset_infos[1]), + startTime: convertDateToSecond(oneWeekBeforeNow), + endTime: convertDateToSecond(currentDate) + }); + }) + ); + console.table(allPairFees); + await this.duckDb.insertPairInfos( pairInfos.map((pair, index) => { const symbols = getSymbolFromAsset(pair.asset_infos); diff --git a/packages/oraidex-sync/src/types.ts b/packages/oraidex-sync/src/types.ts index 5f297315..55619db4 100644 --- a/packages/oraidex-sync/src/types.ts +++ b/packages/oraidex-sync/src/types.ts @@ -195,3 +195,10 @@ export type GetCandlesQuery = { startTime: number; endTime: number; }; + +export type GetFeeSwap = { + offerDenom: string; + askDenom: string; + startTime: number; + endTime: number; +}; From 23b5f20a3b2353c108d9cb1daa12a6e22ecfc088 Mon Sep 17 00:00:00 2001 From: trungbach Date: Wed, 23 Aug 2023 18:43:41 +0700 Subject: [PATCH 05/46] update: calculate fee when provide/withdraw liquidity --- packages/oraidex-sync/src/db.ts | 19 ++- packages/oraidex-sync/src/helper.ts | 29 ++--- packages/oraidex-sync/src/index.ts | 106 +++++++++-------- packages/oraidex-sync/src/parse.ts | 42 +++++++ packages/oraidex-sync/src/poolHelper.ts | 137 ++++++++++++++++++++++ packages/oraidex-sync/src/tx-parsing.ts | 60 +++++++--- packages/oraidex-sync/tests/parse.spec.ts | 73 ++++++++++++ 7 files changed, 381 insertions(+), 85 deletions(-) create mode 100644 packages/oraidex-sync/src/parse.ts create mode 100644 packages/oraidex-sync/src/poolHelper.ts create mode 100644 packages/oraidex-sync/tests/parse.spec.ts diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index 4fe85abc..4421615b 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -13,7 +13,8 @@ import { GetFeeSwap } from "./types"; import fs, { rename } from "fs"; -import { isoToTimestampNumber, renameKey, replaceAllNonAlphaBetChar, toObject } from "./helper"; +import { isoToTimestampNumber, parseAssetInfo, renameKey, replaceAllNonAlphaBetChar, toObject } from "./helper"; +import { AssetInfo } from "@oraichain/oraidex-contracts-sdk"; export class DuckDb { protected constructor(public readonly conn: Connection, private db: Database) {} @@ -366,9 +367,20 @@ export class DuckDb { return (await this.conn.all("SELECT * from pair_infos")).map((data) => data as PairInfoData); } + async getPoolByAssetInfos(assetInfos: [AssetInfo, AssetInfo]): Promise { + const firstAssetInfo = parseAssetInfo(assetInfos[0]); + const secondAssetInfo = parseAssetInfo(assetInfos[1]); + return ( + await this.conn.all( + `SELECT * from pair_infos WHERE firstAssetInfo = ? AND secondAssetInfo = ?`, + firstAssetInfo, + secondAssetInfo + ) + ).map((data) => data as PairInfoData)[0]; + } + async getFeeSwap(payload: GetFeeSwap): Promise { const { offerDenom, askDenom, startTime, endTime } = payload; - console.log({ payload }); const result = await this.conn.all( ` SELECT @@ -384,7 +396,6 @@ export class DuckDb { offerDenom, askDenom ); - console.log({ result }); - return 1n; + return BigInt(result[0].totalFee ?? 0); } } diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index 7ebc6626..5d41be25 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -6,8 +6,11 @@ import { OraiswapRouterQueryClient, SwapOperation } from "@oraichain/oraidex-contracts-sdk"; -import { pairs, pairsOnlyDenom } from "./pairs"; +import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; +import { isEqual, maxBy, minBy } from "lodash"; import { ORAI, atomic, tenAmountInDecimalSix, truncDecimals, usdtCw20Address } from "./constants"; +import { pairs, pairsOnlyDenom } from "./pairs"; +import { simulateSwapPriceWithUsdt } from "./query"; import { Ohlcv, OraiDexType, @@ -17,9 +20,6 @@ import { SwapOperationData, WithdrawLiquidityOperationData } from "./types"; -import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; -import { minBy, maxBy, isEqual } from "lodash"; -import { getPoolInfos, simulateSwapPriceWithUsdt } from "./query"; export function toObject(data: any) { return JSON.parse( @@ -423,6 +423,7 @@ async function fetchPoolInfoAmount(fromInfo: AssetInfo, toInfo: AssetInfo, pairA async function getPairLiquidity([fromInfo, toInfo]: [AssetInfo, AssetInfo], pairAddr: string): Promise { const { offerPoolAmount, askPoolAmount } = await fetchPoolInfoAmount(fromInfo, toInfo, pairAddr); + const routerContract = new OraiswapRouterQueryClient( await getCosmwasmClient(), process.env.ROUTER_CONTRACT_ADDRESS || "orai1j0r67r9k8t34pnhy00x3ftuxuwg0r6r4p8p6rrc8az0ednzr8y9s3sj2sf" @@ -449,18 +450,18 @@ function convertDateToSecond(date: Date): number { } export { - findMappedTargetedAssetInfo, - findAssetInfoPathToUsdt, - generateSwapOperations, - parseAssetInfo, - parseAssetInfoOnlyDenom, - delay, - findPairAddress, calculatePriceByPool, - getSymbolFromAsset, - getPoolInfos, + convertDateToSecond, + delay, fetchPoolInfoAmount, + findAssetInfoPathToUsdt, + findMappedTargetedAssetInfo, + findPairAddress, + generateSwapOperations, + getCosmwasmClient, getPairLiquidity, getSpecificDateBeforeNow, - convertDateToSecond + getSymbolFromAsset, + parseAssetInfo, + parseAssetInfoOnlyDenom }; diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index d2056f8d..f1cac4e0 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -1,19 +1,7 @@ +import { SyncData, Txs, WriteData } from "@oraichain/cosmos-rpc-sync"; +import { AssetInfo, CosmWasmClient, OraiswapFactoryQueryClient, PairInfo } from "@oraichain/oraidex-contracts-sdk"; import "dotenv/config"; -import { parseAssetInfo, parseTxs } from "./tx-parsing"; import { DuckDb } from "./db"; -import { WriteData, SyncData, Txs } from "@oraichain/cosmos-rpc-sync"; -import { AssetInfo, CosmWasmClient, OraiswapFactoryQueryClient, PairInfo } from "@oraichain/oraidex-contracts-sdk"; -import { - ProvideLiquidityOperationData, - TxAnlysisResult, - WithdrawLiquidityOperationData, - InitialData, - PairInfoData, - Env -} from "./types"; -import { MulticallQueryClient } from "@oraichain/common-contracts-sdk"; -import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; -import { getAllPairInfos, getPoolInfos } from "./query"; import { collectAccumulateLpData, convertDateToSecond, @@ -22,6 +10,17 @@ import { getSymbolFromAsset, parseAssetInfoOnlyDenom } from "./helper"; +import { getPoolInfos } from "./poolHelper"; +import { getAllPairInfos } from "./query"; +import { parseAssetInfo, parseTxs } from "./tx-parsing"; +import { + Env, + InitialData, + PairInfoData, + ProvideLiquidityOperationData, + TxAnlysisResult, + WithdrawLiquidityOperationData +} from "./types"; class WriteOrders extends WriteData { private firstWrite: boolean; @@ -40,23 +39,10 @@ class WriteOrders extends WriteData { await this.duckDb.insertLpOps(txs.withdrawLiquidityOpsData); } - private async getPoolInfos(pairAddrs: string[], wantedHeight?: number): Promise { - // adjust the query height to get data from the past - const cosmwasmClient = await CosmWasmClient.connect(this.rpcUrl); - cosmwasmClient.setQueryClientWithHeight(wantedHeight); - const multicall = new MulticallQueryClient( - cosmwasmClient, - this.env.MULTICALL_CONTRACT_ADDRESS || "orai1q7x644gmf7h8u8y6y8t9z9nnwl8djkmspypr6mxavsk9ual7dj0sxpmgwd" - ); - const res = await getPoolInfos(pairAddrs, multicall); - // reset query client to latest for other functions to call - return res; - } - private async accumulatePoolAmount(data: ProvideLiquidityOperationData[] | WithdrawLiquidityOperationData[]) { if (data.length === 0) return; // guard. If theres no data then we wont process anything const pairInfos = await this.duckDb.queryPairInfos(); - const poolInfos = await this.getPoolInfos( + const poolInfos = await getPoolInfos( pairInfos.map((pair) => pair.pairAddr), data[0].txheight // assume data is sorted by height and timestamp ); @@ -69,7 +55,7 @@ class WriteOrders extends WriteData { const currentOffset = await this.duckDb.loadHeightSnapshot(); // edge case. If no new block has been found, then we skip processing to prevent duplication handling if (currentOffset === newOffset) return true; - let result = parseTxs(txs); + let result = await parseTxs(txs, this.duckDb); // accumulate liquidity pool amount await this.accumulatePoolAmount([...result.provideLiquidityOpsData, ...result.withdrawLiquidityOpsData]); @@ -92,7 +78,6 @@ class WriteOrders extends WriteData { } } -// we need to create a new table with name PoolInfo, whenever order table class OraiDexSync { protected constructor( private readonly duckDb: DuckDb, @@ -118,35 +103,50 @@ class OraiDexSync { return getAllPairInfos(firstFactoryClient, secondFactoryClient); } + async getSwapFeePair(asset_infos: [AssetInfo, AssetInfo], startTime: Date, endTime: Date): Promise { + const [swapFee, swapFeeReverse] = await Promise.all([ + this.duckDb.getFeeSwap({ + offerDenom: parseAssetInfoOnlyDenom(asset_infos[0]), + askDenom: parseAssetInfoOnlyDenom(asset_infos[1]), + startTime: convertDateToSecond(startTime), + endTime: convertDateToSecond(endTime) + }), + this.duckDb.getFeeSwap({ + offerDenom: parseAssetInfoOnlyDenom(asset_infos[1]), + askDenom: parseAssetInfoOnlyDenom(asset_infos[0]), + startTime: convertDateToSecond(startTime), + endTime: convertDateToSecond(endTime) + }) + ]); + return swapFee + swapFeeReverse; + } + + async getAllFees(pairInfos: PairInfo[]): Promise { + const tf = 7 * 24 * 60 * 60; // second of 7 days + const currentDate = new Date(); + const oneWeekBeforeNow = getSpecificDateBeforeNow(new Date(), tf); + const swapFees = await Promise.all( + pairInfos.map((pair) => this.getSwapFeePair(pair.asset_infos, oneWeekBeforeNow, currentDate)) + ); + + // const lpFees = + return swapFees; + } + // fromIconUrl, toIconUrl: upload to other server // volume24Hour: volume ohlcv + volume liquidity (?) // apr: oraidex // totalLiquidity: get liquidity in lp_ops_data with last block of pair // fee7Days: sum of fee in swap_ops ( taxAmount + commissionAmount ) + & lp_ops fee of scatom/atom private async updateLatestPairInfos() { - const pairInfos = await this.getAllPairInfos(); console.time("timer"); + const pairInfos = await this.getAllPairInfos(); const allLiquidities = await Promise.all( pairInfos.map((pair) => { return getPairLiquidity(pair.asset_infos, pair.contract_addr); }) ); - console.timeEnd("timer"); - - const tf = 7 * 24 * 60 * 60; // second of 7 days - const currentDate = new Date(); - const oneWeekBeforeNow = getSpecificDateBeforeNow(new Date(), tf); - const allPairFees = await Promise.all( - pairInfos.map((pair) => { - return this.duckDb.getFeeSwap({ - offerDenom: parseAssetInfoOnlyDenom(pair.asset_infos[0]), - askDenom: parseAssetInfoOnlyDenom(pair.asset_infos[1]), - startTime: convertDateToSecond(oneWeekBeforeNow), - endTime: convertDateToSecond(currentDate) - }); - }) - ); - console.table(allPairFees); + const allFee7Days = await this.getAllFees(pairInfos); await this.duckDb.insertPairInfos( pairInfos.map((pair, index) => { @@ -164,10 +164,11 @@ class OraiDexSync { volume24Hour: 1n, apr: 2, totalLiquidity: allLiquidities[index], - fee7Days: 1n + fee7Days: allFee7Days[index] } as PairInfoData; }) ); + console.timeEnd("timer"); } public async sync() { @@ -211,9 +212,10 @@ async function initSync() { initSync(); export { OraiDexSync }; -export * from "./types"; -export * from "./query"; -export * from "./helper"; +export * from "./constants"; export * from "./db"; +export * from "./helper"; export * from "./pairs"; -export * from "./constants"; +export * from "./parse"; +export * from "./query"; +export * from "./types"; diff --git a/packages/oraidex-sync/src/parse.ts b/packages/oraidex-sync/src/parse.ts new file mode 100644 index 00000000..e3734ac2 --- /dev/null +++ b/packages/oraidex-sync/src/parse.ts @@ -0,0 +1,42 @@ +/** + * Functions that help parse from this type to other type. + */ + +import { AssetInfo } from "@oraichain/oraidex-contracts-sdk"; +import { parseAssetInfoOnlyDenom } from "./helper"; +import { pairs } from "./pairs"; +import { ORAI } from "./constants"; + +function parseDenomToAssetLiquidity([fromDenom, toDenom]: [string, string]): [AssetInfo, AssetInfo] { + const findedPair = pairs.find((pair) => { + const denoms = [parseAssetInfoOnlyDenom(pair.asset_infos[0]), parseAssetInfoOnlyDenom(pair.asset_infos[1])]; + return denoms.includes(fromDenom) && denoms.includes(toDenom); + }); + if (!findedPair) return null; + + return findedPair.asset_infos; +} + +/** + * Check pool: + * if pool has native_token ORAI -> dont has fee + * else if pool has native_token not ORAI -> has fee + * otherwise it has 2 cw20 token -> dont has fee + * @param [fromDenom, toDenom]: denom in a pool + * @returns is pool has fee: boolean + */ +function isPoolHasFee(assetInfos: [AssetInfo, AssetInfo]): boolean { + let hasNative = false; + for (const asset of assetInfos) { + if ("native_token" in asset) { + hasNative = true; + if (asset.native_token.denom === "orai") { + return false; + } + } + } + if (hasNative) return true; + return false; +} + +export { parseDenomToAssetLiquidity, isPoolHasFee }; diff --git a/packages/oraidex-sync/src/poolHelper.ts b/packages/oraidex-sync/src/poolHelper.ts new file mode 100644 index 00000000..43ba204b --- /dev/null +++ b/packages/oraidex-sync/src/poolHelper.ts @@ -0,0 +1,137 @@ +import { MulticallQueryClient } from "@oraichain/common-contracts-sdk"; +import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; +import { getAllPairInfos as queryAllPairInfos, getPoolInfos as queryPoolInfos } from "./query"; +import { + calculatePriceByPool, + fetchPoolInfoAmount, + findAssetInfoPathToUsdt, + getCosmwasmClient, + parseAssetInfo, + parseAssetInfoOnlyDenom +} from "./helper"; +import { + Asset, + AssetInfo, + OraiswapFactoryQueryClient, + OraiswapPairQueryClient, + PairInfo +} from "@oraichain/oraidex-contracts-sdk"; +import { ORAI, usdtCw20Address } from "./constants"; +import { PairInfoData, PairMapping } from "./types"; +import { pairs } from "./pairs"; + +async function getPoolInfos(pairAddrs: string[], wantedHeight?: number): Promise { + // adjust the query height to get data from the past + const cosmwasmClient = await getCosmwasmClient(); + cosmwasmClient.setQueryClientWithHeight(wantedHeight); + const multicall = new MulticallQueryClient( + cosmwasmClient, + process.env.MULTICALL_CONTRACT_ADDRESS || "orai1q7x644gmf7h8u8y6y8t9z9nnwl8djkmspypr6mxavsk9ual7dj0sxpmgwd" + ); + const res = await queryPoolInfos(pairAddrs, multicall); + // reset query client to latest for other functions to call + return res; +} + +function calculateFeeByAsset(asset: Asset, shareRatio: number): Asset { + const TAX_CAP = 10 ** 6; + const TAX_RATE = 0.3; + if (!("native_token" in asset.info)) return null; + const amount = +asset.amount; + const refundAmount = amount * shareRatio; + const fee = Math.min(refundAmount - (refundAmount * 1) / (TAX_RATE + 1), TAX_CAP); + return { + amount: fee.toString(), + info: asset.info + }; +} + +// find pool match this asset with orai => calculate price this asset vs orai => calculate price this asset vs usdt. +async function getPriceAssetByUsdt(asset: AssetInfo): Promise { + const foundPair = pairs.find((pair) => { + const denoms = [parseAssetInfoOnlyDenom(pair.asset_infos[0]), parseAssetInfoOnlyDenom(pair.asset_infos[1])]; + return denoms.includes(parseAssetInfoOnlyDenom(asset)) && denoms.includes(ORAI); + }); + if (!foundPair) return 0; + const priceByOrai = getPriceByAsset(foundPair.asset_infos); + console.log({ priceByOrai }); + + const usdtInfo = { token: { contract_addr: usdtCw20Address } }; + const oraiInfo = { native_token: { denom: ORAI } }; + const priceByUsdt = getPriceByAsset([oraiInfo, usdtInfo]); + console.log({ priceByUsdt }); + return priceByUsdt; +} + +async function calculateFeeByUsdt(fee: Asset): Promise { + if (!fee) return 0; + const priceByUsdt = await getPriceAssetByUsdt(fee.info); + return priceByUsdt * +fee.amount; +} + +/** + * TODO: explain this function + * + * @param txHeight + */ +async function calculateLiquidityFee(pair: PairInfoData, txHeight: number, withdrawnShare: number): Promise { + const cosmwasmClient = await getCosmwasmClient(); + cosmwasmClient.setQueryClientWithHeight(txHeight); + + const pairContract = new OraiswapPairQueryClient(cosmwasmClient, pair.pairAddr); + const poolInfo = await pairContract.pool(); + const totalShare = +poolInfo.total_share; + const shareRatio = withdrawnShare / totalShare; + + const [feeByAssetFrom, feeByAssetTo] = [ + calculateFeeByAsset(poolInfo.assets[0], shareRatio), + calculateFeeByAsset(poolInfo.assets[1], shareRatio) + ]; + + const feeByUsdt = (await calculateFeeByUsdt(feeByAssetFrom)) + (await calculateFeeByUsdt(feeByAssetTo)); + return BigInt(Math.round(feeByUsdt)); +} + +async function getPairInfos(): Promise { + const cosmwasmClient = await getCosmwasmClient(); + const firstFactoryClient = new OraiswapFactoryQueryClient( + cosmwasmClient, + "orai1hemdkz4xx9kukgrunxu3yw0nvpyxf34v82d2c8" + ); + const secondFactoryClient = new OraiswapFactoryQueryClient( + cosmwasmClient, + "orai167r4ut7avvgpp3rlzksz6vw5spmykluzagvmj3ht845fjschwugqjsqhst" + ); + return queryAllPairInfos(firstFactoryClient, secondFactoryClient); +} + +async function getOraiPrice() { + const usdtInfo = { token: { contract_addr: usdtCw20Address } }; + const oraiInfo = { native_token: { denom: ORAI } }; + // TODO: currently we get all pairinfo then find orai/usdt pair, but it slow, so need to refactor this ops + const allPairInfos = await getPairInfos(); + const oraiUsdtPool = allPairInfos.find( + (pair) => + parseAssetInfo(pair.asset_infos[0]) === parseAssetInfo(oraiInfo) && + parseAssetInfo(pair.asset_infos[1]) === parseAssetInfo(usdtInfo) + ); + const { offerPoolAmount, askPoolAmount } = await fetchPoolInfoAmount(oraiInfo, usdtInfo, oraiUsdtPool.contract_addr); + const oraiPrice = calculatePriceByPool(askPoolAmount, offerPoolAmount, +oraiUsdtPool.commission_rate); + return oraiPrice; +} + +async function getPriceByAsset([baseAsset, quoteAsset]: [AssetInfo, AssetInfo]): Promise { + // TODO: currently we get all pairinfo then find orai/usdt pair, but it slow, so need to refactor this ops + const allPairInfos = await getPairInfos(); + const pool = allPairInfos.find( + (pair) => + parseAssetInfo(pair.asset_infos[0]) === parseAssetInfo(baseAsset) && + parseAssetInfo(pair.asset_infos[1]) === parseAssetInfo(quoteAsset) + ); + if (pool) return 0; + const { offerPoolAmount, askPoolAmount } = await fetchPoolInfoAmount(baseAsset, quoteAsset, pool.contract_addr); + const assetPrice = calculatePriceByPool(askPoolAmount, offerPoolAmount, +pool.commission_rate); + return assetPrice; +} + +export { getPoolInfos, calculateLiquidityFee, getOraiPrice }; diff --git a/packages/oraidex-sync/src/tx-parsing.ts b/packages/oraidex-sync/src/tx-parsing.ts index e3124d12..49f4ae82 100644 --- a/packages/oraidex-sync/src/tx-parsing.ts +++ b/packages/oraidex-sync/src/tx-parsing.ts @@ -30,6 +30,9 @@ import { removeOpsDuplication } from "./helper"; import { pairs } from "./pairs"; +import { isPoolHasFee } from "./parse"; +import { DuckDb } from "./db"; +import { calculateLiquidityFee } from "./poolHelper"; function parseWasmEvents(events: readonly Event[]): (readonly Attribute[])[] { return events.filter((event) => event.type === "wasm").map((event) => event.attributes); @@ -167,11 +170,12 @@ function parseWithdrawLiquidityAssets(assets: string): string[] { return matches.slice(1, 5); } -function extractMsgWithdrawLiquidity( +async function extractMsgWithdrawLiquidity( txData: BasicTxData, wasmAttributes: (readonly Attribute[])[], - txCreator: string -): WithdrawLiquidityOperationData[] { + txCreator: string, + duckDb: DuckDb +): Promise { const withdrawData: WithdrawLiquidityOperationData[] = []; for (let attrs of wasmAttributes) { @@ -185,18 +189,35 @@ function extractMsgWithdrawLiquidity( let quoteAsset = assets[3]; let quoteAssetAmount = parseInt(assets[2]); // we only have one pair order. If the order is reversed then we also reverse the order - if ( - pairs.find((pair) => - isEqual( - pair.asset_infos.map((info) => parseAssetInfoOnlyDenom(info)), - [quoteAsset, baseAsset] - ) + let findedPair = pairs.find((pair) => + isEqual( + pair.asset_infos.map((info) => parseAssetInfoOnlyDenom(info)), + [quoteAsset, baseAsset] ) - ) { + ); + if (findedPair) { baseAsset = assets[3]; quoteAsset = assets[1]; + } else { + // otherwise find in reverse order + findedPair = pairs.find((pair) => + isEqual( + pair.asset_infos.map((info) => parseAssetInfoOnlyDenom(info)), + [baseAsset, quoteAsset] + ) + ); } if (assets.length !== 4) continue; + + let fee = 0n; + const isHasFee = isPoolHasFee(findedPair.asset_infos); + console.log({ isHasFee }); + if (isHasFee) { + const withdrawnShare = attrs.find((attr) => attr.key === "withdrawn_share").value; + const pair = await duckDb.getPoolByAssetInfos(findedPair.asset_infos); + fee = await calculateLiquidityFee(pair, txData.txheight, +withdrawnShare); + } + withdrawData.push({ basePrice: calculatePriceByPool(BigInt(baseAssetAmount), BigInt(quoteAssetAmount)), baseTokenAmount: baseAssetAmount, @@ -253,7 +274,7 @@ function parseExecuteContractToOraidexMsgs(msgs: MsgExecuteContractWithLogs[]): return objs; } -function parseTxs(txs: Tx[]): TxAnlysisResult { +async function parseTxs(txs: Tx[], duckDb: DuckDb): Promise { let transactions: Tx[] = []; let swapOpsData: SwapOperationData[] = []; let accountTxs: AccountTx[] = []; @@ -268,20 +289,22 @@ function parseTxs(txs: Tx[]): TxAnlysisResult { txhash: tx.hash, txheight: tx.height }; + for (let msg of msgs) { const sender = msg.sender; const wasmAttributes = parseWasmEvents(msg.logs.events); swapOpsData.push(...extractSwapOperations(basicTxData, wasmAttributes)); const provideLiquidityData = extractMsgProvideLiquidity(basicTxData, msg.msg, sender); if (provideLiquidityData) provideLiquidityOpsData.push(provideLiquidityData); - withdrawLiquidityOpsData.push(...extractMsgWithdrawLiquidity(basicTxData, wasmAttributes, sender)); + withdrawLiquidityOpsData.push( + ...(await extractMsgWithdrawLiquidity(basicTxData, wasmAttributes, sender, duckDb)) + ); accountTxs.push({ txhash: basicTxData.txhash, accountAddress: sender }); } } swapOpsData = swapOpsData.filter((i) => i.direction); swapOpsData = removeOpsDuplication(swapOpsData) as SwapOperationData[]; return { - // transactions: txs, swapOpsData: groupByTime(swapOpsData) as SwapOperationData[], ohlcv: buildOhlcv(swapOpsData), accountTxs, @@ -289,9 +312,16 @@ function parseTxs(txs: Tx[]): TxAnlysisResult { removeOpsDuplication(provideLiquidityOpsData) ) as ProvideLiquidityOperationData[], withdrawLiquidityOpsData: groupByTime( - removeOpsDuplication(provideLiquidityOpsData) + removeOpsDuplication(withdrawLiquidityOpsData) ) as WithdrawLiquidityOperationData[] }; } -export { parseAssetInfo, parseWasmEvents, parseTxs, parseWithdrawLiquidityAssets, parseTxToMsgExecuteContractMsgs }; +export { + parseAssetInfo, + parseWasmEvents, + parseTxs, + parseWithdrawLiquidityAssets, + parseTxToMsgExecuteContractMsgs, + calculateLiquidityFee +}; diff --git a/packages/oraidex-sync/tests/parse.spec.ts b/packages/oraidex-sync/tests/parse.spec.ts new file mode 100644 index 00000000..d2cb9bd6 --- /dev/null +++ b/packages/oraidex-sync/tests/parse.spec.ts @@ -0,0 +1,73 @@ +import { AssetInfo } from "@oraichain/oraidex-contracts-sdk"; +import { isPoolHasFee, parseDenomToAssetLiquidity } from "../src/parse"; +import { + ORAI, + airiCw20Adress, + atomIbcDenom, + milkyCw20Address, + scAtomCw20Address, + usdtCw20Address +} from "../src/constants"; + +describe("test-parse", () => { + it.each<[[string, string], [AssetInfo, AssetInfo] | null]>([ + [["invalidDenom", "invalidDenom"], null], + [[ORAI, "invalidDenom"], null], + [ + [ORAI, airiCw20Adress], + [{ token: { contract_addr: airiCw20Adress } }, { native_token: { denom: ORAI } }] + ] + ])( + "test-parseDenomToAssetLiquidity-with-denom-%s-should-return-correctly-asset-%o", + (denoms: [string, string], expectedResult: [AssetInfo, AssetInfo] | null) => { + const result = parseDenomToAssetLiquidity(denoms); + // ✅ PASS + expect(result).toStrictEqual(expectedResult); + } + ); + + it.each<[string, [AssetInfo, AssetInfo], boolean]>([ + [ + "has-both-native-token-that-contain-ORAI-should-return: false", + [{ native_token: { denom: ORAI } }, { native_token: { denom: atomIbcDenom } }], + false + ], + // [ + // // NOTE: currently this case not exist, but in future maybe + // "has-both-native-token-that-NOT-contain-ORAI-should-return: true", + // [osmosisIbcDenom, atomIbcDenom], + // true + // ], + [ + "has-one-native-token-that-NOT-contain-ORAI-should-return: true", + [ + { native_token: { denom: atomIbcDenom } }, + { + token: { + contract_addr: scAtomCw20Address + } + } + ], + true + ], + [ + "NOT-has-native-token-should-return-is-has-fee: false", + [ + { + token: { + contract_addr: milkyCw20Address + } + }, + { + token: { + contract_addr: usdtCw20Address + } + } + ], + false + ] + ])("test-isPoolHasFee-with-pool-%s", (_caseName, assetInfos, expectIsHasFee) => { + const result = isPoolHasFee(assetInfos); + expect(result).toBe(expectIsHasFee); + }); +}); From 1c554c671c1f6260168d5d76eaf2192518c58833 Mon Sep 17 00:00:00 2001 From: trungbach Date: Wed, 23 Aug 2023 23:22:06 +0700 Subject: [PATCH 06/46] update: calculated fee when withdraw liquidity --- packages/oraidex-sync/src/db.ts | 3 +- packages/oraidex-sync/src/index.ts | 64 ++++++------ packages/oraidex-sync/src/poolHelper.ts | 113 ++++++++++++--------- packages/oraidex-sync/src/tx-parsing.ts | 6 +- packages/oraidex-sync/src/types.ts | 1 + packages/oraidex-sync/tests/helper.spec.ts | 18 ++-- 6 files changed, 118 insertions(+), 87 deletions(-) diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index 4421615b..775eb982 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -107,7 +107,8 @@ export class DuckDb { timestamp UINTEGER, txCreator VARCHAR, txhash VARCHAR, - txheight UINTEGER)` + txheight UINTEGER, + taxRate UBIGINT)` ); } diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index f1cac4e0..e12fc382 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -139,36 +139,40 @@ class OraiDexSync { // totalLiquidity: get liquidity in lp_ops_data with last block of pair // fee7Days: sum of fee in swap_ops ( taxAmount + commissionAmount ) + & lp_ops fee of scatom/atom private async updateLatestPairInfos() { - console.time("timer"); - const pairInfos = await this.getAllPairInfos(); - const allLiquidities = await Promise.all( - pairInfos.map((pair) => { - return getPairLiquidity(pair.asset_infos, pair.contract_addr); - }) - ); - const allFee7Days = await this.getAllFees(pairInfos); - - await this.duckDb.insertPairInfos( - pairInfos.map((pair, index) => { - const symbols = getSymbolFromAsset(pair.asset_infos); - return { - firstAssetInfo: parseAssetInfo(pair.asset_infos[0]), - secondAssetInfo: parseAssetInfo(pair.asset_infos[1]), - commissionRate: pair.commission_rate, - pairAddr: pair.contract_addr, - liquidityAddr: pair.liquidity_token, - oracleAddr: pair.oracle_addr, - symbols, - fromIconUrl: "url1", - toIconUrl: "url2", - volume24Hour: 1n, - apr: 2, - totalLiquidity: allLiquidities[index], - fee7Days: allFee7Days[index] - } as PairInfoData; - }) - ); - console.timeEnd("timer"); + try { + console.time("timer-updateLatestPairInfos"); + const pairInfos = await this.getAllPairInfos(); + const allLiquidities = await Promise.all( + pairInfos.map((pair) => { + return getPairLiquidity(pair.asset_infos, pair.contract_addr); + }) + ); + const allFee7Days = await this.getAllFees(pairInfos); + + await this.duckDb.insertPairInfos( + pairInfos.map((pair, index) => { + const symbols = getSymbolFromAsset(pair.asset_infos); + return { + firstAssetInfo: parseAssetInfo(pair.asset_infos[0]), + secondAssetInfo: parseAssetInfo(pair.asset_infos[1]), + commissionRate: pair.commission_rate, + pairAddr: pair.contract_addr, + liquidityAddr: pair.liquidity_token, + oracleAddr: pair.oracle_addr, + symbols, + fromIconUrl: "url1", + toIconUrl: "url2", + volume24Hour: 1n, + apr: 2, + totalLiquidity: allLiquidities[index], + fee7Days: allFee7Days[index] + } as PairInfoData; + }) + ); + console.timeEnd("timer-updateLatestPairInfos"); + } catch (error) { + console.log("error in updateLatestPairInfos: ", error); + } } public async sync() { diff --git a/packages/oraidex-sync/src/poolHelper.ts b/packages/oraidex-sync/src/poolHelper.ts index 43ba204b..9f7e714d 100644 --- a/packages/oraidex-sync/src/poolHelper.ts +++ b/packages/oraidex-sync/src/poolHelper.ts @@ -4,7 +4,6 @@ import { getAllPairInfos as queryAllPairInfos, getPoolInfos as queryPoolInfos } import { calculatePriceByPool, fetchPoolInfoAmount, - findAssetInfoPathToUsdt, getCosmwasmClient, parseAssetInfo, parseAssetInfoOnlyDenom @@ -20,6 +19,8 @@ import { ORAI, usdtCw20Address } from "./constants"; import { PairInfoData, PairMapping } from "./types"; import { pairs } from "./pairs"; +// use this type to determine the ratio of price of base to the quote or vice versa +export type RatioDirection = "base_in_quote" | "quote_in_base"; async function getPoolInfos(pairAddrs: string[], wantedHeight?: number): Promise { // adjust the query height to get data from the past const cosmwasmClient = await getCosmwasmClient(); @@ -36,6 +37,7 @@ async function getPoolInfos(pairAddrs: string[], wantedHeight?: number): Promise function calculateFeeByAsset(asset: Asset, shareRatio: number): Asset { const TAX_CAP = 10 ** 6; const TAX_RATE = 0.3; + // just native_token not ORAI has fee if (!("native_token" in asset.info)) return null; const amount = +asset.amount; const refundAmount = amount * shareRatio; @@ -46,50 +48,27 @@ function calculateFeeByAsset(asset: Asset, shareRatio: number): Asset { }; } -// find pool match this asset with orai => calculate price this asset vs orai => calculate price this asset vs usdt. +// find pool match this asset with orai => calculate price this asset token in ORAI. +// then, calculate price of this asset token in USDT based on price ORAI in USDT. async function getPriceAssetByUsdt(asset: AssetInfo): Promise { const foundPair = pairs.find((pair) => { const denoms = [parseAssetInfoOnlyDenom(pair.asset_infos[0]), parseAssetInfoOnlyDenom(pair.asset_infos[1])]; return denoms.includes(parseAssetInfoOnlyDenom(asset)) && denoms.includes(ORAI); }); if (!foundPair) return 0; - const priceByOrai = getPriceByAsset(foundPair.asset_infos); - console.log({ priceByOrai }); - const usdtInfo = { token: { contract_addr: usdtCw20Address } }; - const oraiInfo = { native_token: { denom: ORAI } }; - const priceByUsdt = getPriceByAsset([oraiInfo, usdtInfo]); - console.log({ priceByUsdt }); - return priceByUsdt; + const ratioDirection: RatioDirection = + parseAssetInfoOnlyDenom(foundPair.asset_infos[0]) === ORAI ? "quote_in_base" : "base_in_quote"; + const priceInOrai = await getPriceByAsset(foundPair.asset_infos, ratioDirection); + const priceOraiInUsdt = await getOraiPrice(); + console.log({ asset, priceInOrai, priceOraiInUsdt }); + return priceInOrai * priceOraiInUsdt; } async function calculateFeeByUsdt(fee: Asset): Promise { if (!fee) return 0; - const priceByUsdt = await getPriceAssetByUsdt(fee.info); - return priceByUsdt * +fee.amount; -} - -/** - * TODO: explain this function - * - * @param txHeight - */ -async function calculateLiquidityFee(pair: PairInfoData, txHeight: number, withdrawnShare: number): Promise { - const cosmwasmClient = await getCosmwasmClient(); - cosmwasmClient.setQueryClientWithHeight(txHeight); - - const pairContract = new OraiswapPairQueryClient(cosmwasmClient, pair.pairAddr); - const poolInfo = await pairContract.pool(); - const totalShare = +poolInfo.total_share; - const shareRatio = withdrawnShare / totalShare; - - const [feeByAssetFrom, feeByAssetTo] = [ - calculateFeeByAsset(poolInfo.assets[0], shareRatio), - calculateFeeByAsset(poolInfo.assets[1], shareRatio) - ]; - - const feeByUsdt = (await calculateFeeByUsdt(feeByAssetFrom)) + (await calculateFeeByUsdt(feeByAssetTo)); - return BigInt(Math.round(feeByUsdt)); + const priceInUsdt = await getPriceAssetByUsdt(fee.info); + return priceInUsdt * +fee.amount; } async function getPairInfos(): Promise { @@ -105,22 +84,30 @@ async function getPairInfos(): Promise { return queryAllPairInfos(firstFactoryClient, secondFactoryClient); } -async function getOraiPrice() { +function getPairByAssetInfos(assetInfos: [AssetInfo, AssetInfo]): PairMapping { + return pairs.find((pair) => { + const [baseAsset, quoteAsset] = pair.asset_infos; + const denoms = [parseAssetInfoOnlyDenom(baseAsset), parseAssetInfoOnlyDenom(quoteAsset)]; + return ( + denoms.includes(parseAssetInfoOnlyDenom(assetInfos[0])) && denoms.includes(parseAssetInfoOnlyDenom(assetInfos[1])) + ); + }); +} + +// get price ORAI in USDT base on ORAI/USDT pool. +async function getOraiPrice(): Promise { const usdtInfo = { token: { contract_addr: usdtCw20Address } }; const oraiInfo = { native_token: { denom: ORAI } }; - // TODO: currently we get all pairinfo then find orai/usdt pair, but it slow, so need to refactor this ops - const allPairInfos = await getPairInfos(); - const oraiUsdtPool = allPairInfos.find( - (pair) => - parseAssetInfo(pair.asset_infos[0]) === parseAssetInfo(oraiInfo) && - parseAssetInfo(pair.asset_infos[1]) === parseAssetInfo(usdtInfo) - ); - const { offerPoolAmount, askPoolAmount } = await fetchPoolInfoAmount(oraiInfo, usdtInfo, oraiUsdtPool.contract_addr); - const oraiPrice = calculatePriceByPool(askPoolAmount, offerPoolAmount, +oraiUsdtPool.commission_rate); - return oraiPrice; + const oraiUsdtPair = getPairByAssetInfos([oraiInfo, usdtInfo]); + const ratioDirection: RatioDirection = + parseAssetInfoOnlyDenom(oraiUsdtPair.asset_infos[0]) === ORAI ? "base_in_quote" : "quote_in_base"; + return getPriceByAsset([oraiInfo, usdtInfo], ratioDirection); } -async function getPriceByAsset([baseAsset, quoteAsset]: [AssetInfo, AssetInfo]): Promise { +async function getPriceByAsset( + [baseAsset, quoteAsset]: [AssetInfo, AssetInfo], + ratioDirection: RatioDirection +): Promise { // TODO: currently we get all pairinfo then find orai/usdt pair, but it slow, so need to refactor this ops const allPairInfos = await getPairInfos(); const pool = allPairInfos.find( @@ -128,10 +115,40 @@ async function getPriceByAsset([baseAsset, quoteAsset]: [AssetInfo, AssetInfo]): parseAssetInfo(pair.asset_infos[0]) === parseAssetInfo(baseAsset) && parseAssetInfo(pair.asset_infos[1]) === parseAssetInfo(quoteAsset) ); - if (pool) return 0; + if (!pool) return 0; + // offer: orai, ask: usdt -> price offer in ask = calculatePriceByPool([ask, offer]) + // offer: orai, ask: atom -> price ask in offer = calculatePriceByPool([offer, ask]) const { offerPoolAmount, askPoolAmount } = await fetchPoolInfoAmount(baseAsset, quoteAsset, pool.contract_addr); const assetPrice = calculatePriceByPool(askPoolAmount, offerPoolAmount, +pool.commission_rate); - return assetPrice; + + return ratioDirection === "base_in_quote" ? assetPrice : 1 / assetPrice; +} + +/** + * First, calculate fee by offer asset & askAsset + * then, calculate fee by those asset to ORAI + * finally, convert this fee in ORAI to USDT. + * @param pair + * @param txHeight + * @param withdrawnShare + * @returns + */ +async function calculateLiquidityFee(pair: PairInfoData, txHeight: number, withdrawnShare: number): Promise { + const cosmwasmClient = await getCosmwasmClient(); + cosmwasmClient.setQueryClientWithHeight(txHeight); + + const pairContract = new OraiswapPairQueryClient(cosmwasmClient, pair.pairAddr); + const poolInfo = await pairContract.pool(); + const totalShare = +poolInfo.total_share; + const shareRatio = withdrawnShare / totalShare; + + const [feeByAssetFrom, feeByAssetTo] = [ + calculateFeeByAsset(poolInfo.assets[0], shareRatio), + calculateFeeByAsset(poolInfo.assets[1], shareRatio) + ]; + + const feeByUsdt = (await calculateFeeByUsdt(feeByAssetFrom)) + (await calculateFeeByUsdt(feeByAssetTo)); + return BigInt(Math.round(feeByUsdt)); } export { getPoolInfos, calculateLiquidityFee, getOraiPrice }; diff --git a/packages/oraidex-sync/src/tx-parsing.ts b/packages/oraidex-sync/src/tx-parsing.ts index 49f4ae82..ac6f7c95 100644 --- a/packages/oraidex-sync/src/tx-parsing.ts +++ b/packages/oraidex-sync/src/tx-parsing.ts @@ -156,7 +156,8 @@ function extractMsgProvideLiquidity( timestamp: txData.timestamp, txCreator, txhash: txData.txhash, - txheight: txData.txheight + txheight: txData.txheight, + taxRate: 1n }; } return undefined; @@ -237,7 +238,8 @@ async function extractMsgWithdrawLiquidity( timestamp: txData.timestamp, txCreator, txhash: txData.txhash, - txheight: txData.txheight + txheight: txData.txheight, + taxRate: fee }); } return withdrawData; diff --git a/packages/oraidex-sync/src/types.ts b/packages/oraidex-sync/src/types.ts index 55619db4..16689aed 100644 --- a/packages/oraidex-sync/src/types.ts +++ b/packages/oraidex-sync/src/types.ts @@ -75,6 +75,7 @@ export type ProvideLiquidityOperationData = { opType: LiquidityOpType; uniqueKey: string; // concat of first, second denom, amount, and timestamp => should be unique. unique key is used to override duplication only. txCreator: string; + taxRate: bigint; } & BasicTxData; export type WithdrawLiquidityOperationData = ProvideLiquidityOperationData; diff --git a/packages/oraidex-sync/tests/helper.spec.ts b/packages/oraidex-sync/tests/helper.spec.ts index 3c21159e..3b00ab00 100644 --- a/packages/oraidex-sync/tests/helper.spec.ts +++ b/packages/oraidex-sync/tests/helper.spec.ts @@ -396,7 +396,8 @@ describe("test-helper", () => { timestamp: 1, txCreator: "a", txhash: "a", - txheight: 1 + txheight: 1, + taxRate: 1n }, { basePrice: 1, @@ -411,7 +412,8 @@ describe("test-helper", () => { timestamp: 1, txCreator: "a", txhash: "a", - txheight: 1 + txheight: 1, + taxRate: 1n }, { basePrice: 1, @@ -426,7 +428,8 @@ describe("test-helper", () => { timestamp: 1, txCreator: "a", txhash: "a", - txheight: 1 + txheight: 1, + taxRate: 1n } ]; @@ -469,7 +472,8 @@ describe("test-helper", () => { timestamp: 1, txCreator: "a", txhash: "a", - txheight: 1 + txheight: 1, + taxRate: 1n }, { basePrice: 1, @@ -484,7 +488,8 @@ describe("test-helper", () => { timestamp: 1, txCreator: "a", txhash: "a", - txheight: 1 + txheight: 1, + taxRate: 1n }, { basePrice: 1, @@ -499,7 +504,8 @@ describe("test-helper", () => { timestamp: 1, txCreator: "a", txhash: "a", - txheight: 1 + txheight: 1, + taxRate: 1n } ]; const newOps = removeOpsDuplication(ops); From e5a8d46dbba0b40ee5a0e0d2e6ac157fc32812ea Mon Sep 17 00:00:00 2001 From: trungbach Date: Thu, 24 Aug 2023 00:21:30 +0700 Subject: [PATCH 07/46] fee: finished calculate when provide/withdraw liquidity without test --- packages/oraidex-sync/src/parse.ts | 1 - packages/oraidex-sync/src/poolHelper.ts | 2 +- packages/oraidex-sync/src/tx-parsing.ts | 148 +++++++++++++++--------- 3 files changed, 97 insertions(+), 54 deletions(-) diff --git a/packages/oraidex-sync/src/parse.ts b/packages/oraidex-sync/src/parse.ts index e3734ac2..bfd4f0b4 100644 --- a/packages/oraidex-sync/src/parse.ts +++ b/packages/oraidex-sync/src/parse.ts @@ -5,7 +5,6 @@ import { AssetInfo } from "@oraichain/oraidex-contracts-sdk"; import { parseAssetInfoOnlyDenom } from "./helper"; import { pairs } from "./pairs"; -import { ORAI } from "./constants"; function parseDenomToAssetLiquidity([fromDenom, toDenom]: [string, string]): [AssetInfo, AssetInfo] { const findedPair = pairs.find((pair) => { diff --git a/packages/oraidex-sync/src/poolHelper.ts b/packages/oraidex-sync/src/poolHelper.ts index 9f7e714d..42c305ce 100644 --- a/packages/oraidex-sync/src/poolHelper.ts +++ b/packages/oraidex-sync/src/poolHelper.ts @@ -126,7 +126,7 @@ async function getPriceByAsset( /** * First, calculate fee by offer asset & askAsset - * then, calculate fee by those asset to ORAI + * then, calculate fee of those asset to ORAI * finally, convert this fee in ORAI to USDT. * @param pair * @param txHeight diff --git a/packages/oraidex-sync/src/tx-parsing.ts b/packages/oraidex-sync/src/tx-parsing.ts index ac6f7c95..cb820973 100644 --- a/packages/oraidex-sync/src/tx-parsing.ts +++ b/packages/oraidex-sync/src/tx-parsing.ts @@ -6,6 +6,7 @@ import { Tx as CosmosTx } from "cosmjs-types/cosmos/tx/v1beta1/tx"; import { AccountTx, BasicTxData, + LiquidityOpType, ModifiedMsgExecuteContract, MsgExecuteContractWithLogs, MsgType, @@ -119,47 +120,99 @@ function extractSwapOperations(txData: BasicTxData, wasmAttributes: (readonly At return swapData; } -function extractMsgProvideLiquidity( +async function getFeeLiquidity( + [baseDenom, quoteDenom]: [string, string], + opType: LiquidityOpType, + attrs: readonly Attribute[], + txheight: number, + duckDb: DuckDb +): Promise { + // we only have one pair order. If the order is reversed then we also reverse the order + let findedPair = pairs.find((pair) => + isEqual( + pair.asset_infos.map((info) => parseAssetInfoOnlyDenom(info)), + [quoteDenom, baseDenom] + ) + ); + if (findedPair) { + [baseDenom, quoteDenom] = [quoteDenom, baseDenom]; + } else { + // otherwise find in reverse order + findedPair = pairs.find((pair) => + isEqual( + pair.asset_infos.map((info) => parseAssetInfoOnlyDenom(info)), + [baseDenom, quoteDenom] + ) + ); + } + let fee = 0n; + const isHasFee = isPoolHasFee(findedPair.asset_infos); + if (isHasFee) { + let lpShare = + opType === "provide" + ? attrs.find((attr) => attr.key === "share").value + : attrs.find((attr) => attr.key === "withdrawn_share").value; + const pair = await duckDb.getPoolByAssetInfos(findedPair.asset_infos); + fee = await calculateLiquidityFee(pair, txheight, +lpShare); + console.log(`fee ${opType} liquidity: $${fee}`); + } + return fee; +} + +async function extractMsgProvideLiquidity( txData: BasicTxData, msg: MsgType, - txCreator: string -): ProvideLiquidityOperationData | undefined { + txCreator: string, + wasmAttributes: (readonly Attribute[])[], + duckDb: DuckDb +): Promise { if ("provide_liquidity" in msg) { - const assetInfos = msg.provide_liquidity.assets.map((asset) => asset.info); - let baseAsset = msg.provide_liquidity.assets[0]; - let quoteAsset = msg.provide_liquidity.assets[1]; - if (isAssetInfoPairReverse(assetInfos)) { - baseAsset = msg.provide_liquidity.assets[1]; - quoteAsset = msg.provide_liquidity.assets[0]; - } - const firstDenom = parseAssetInfoOnlyDenom(baseAsset.info); - const secDenom = parseAssetInfoOnlyDenom(quoteAsset.info); - const firstAmount = parseInt(baseAsset.amount); - const secAmount = parseInt(quoteAsset.amount); + for (let attrs of wasmAttributes) { + const assetInfos = msg.provide_liquidity.assets.map((asset) => asset.info); - return { - basePrice: calculatePriceByPool(BigInt(firstAmount), BigInt(secAmount)), - baseTokenAmount: firstAmount, - baseTokenDenom: firstDenom, - baseTokenReserve: firstAmount, - opType: "provide", - uniqueKey: concatDataToUniqueKey({ + let baseAsset = msg.provide_liquidity.assets[0]; + let quoteAsset = msg.provide_liquidity.assets[1]; + if (isAssetInfoPairReverse(assetInfos)) { + baseAsset = msg.provide_liquidity.assets[1]; + quoteAsset = msg.provide_liquidity.assets[0]; + } + const firstDenom = parseAssetInfoOnlyDenom(baseAsset.info); + const secDenom = parseAssetInfoOnlyDenom(quoteAsset.info); + const firstAmount = parseInt(baseAsset.amount); + const secAmount = parseInt(quoteAsset.amount); + + const fee = await getFeeLiquidity( + [parseAssetInfoOnlyDenom(baseAsset.info), parseAssetInfoOnlyDenom(quoteAsset.info)], + "provide", + attrs, + txData.txheight, + duckDb + ); + return { + basePrice: calculatePriceByPool(BigInt(firstAmount), BigInt(secAmount)), + baseTokenAmount: firstAmount, + baseTokenDenom: firstDenom, + baseTokenReserve: firstAmount, + opType: "provide", + uniqueKey: concatDataToUniqueKey({ + txheight: txData.txheight, + firstAmount, + firstDenom, + secondAmount: secAmount, + secondDenom: secDenom + }), + quoteTokenAmount: secAmount, + quoteTokenDenom: secDenom, + quoteTokenReserve: secAmount, + timestamp: txData.timestamp, + txCreator, + txhash: txData.txhash, txheight: txData.txheight, - firstAmount, - firstDenom, - secondAmount: secAmount, - secondDenom: secDenom - }), - quoteTokenAmount: secAmount, - quoteTokenDenom: secDenom, - quoteTokenReserve: secAmount, - timestamp: txData.timestamp, - txCreator, - txhash: txData.txhash, - txheight: txData.txheight, - taxRate: 1n - }; + taxRate: fee + }; + } } + return undefined; } @@ -199,25 +252,10 @@ async function extractMsgWithdrawLiquidity( if (findedPair) { baseAsset = assets[3]; quoteAsset = assets[1]; - } else { - // otherwise find in reverse order - findedPair = pairs.find((pair) => - isEqual( - pair.asset_infos.map((info) => parseAssetInfoOnlyDenom(info)), - [baseAsset, quoteAsset] - ) - ); } if (assets.length !== 4) continue; - let fee = 0n; - const isHasFee = isPoolHasFee(findedPair.asset_infos); - console.log({ isHasFee }); - if (isHasFee) { - const withdrawnShare = attrs.find((attr) => attr.key === "withdrawn_share").value; - const pair = await duckDb.getPoolByAssetInfos(findedPair.asset_infos); - fee = await calculateLiquidityFee(pair, txData.txheight, +withdrawnShare); - } + const fee = await getFeeLiquidity([baseAsset, quoteAsset], "withdraw", attrs, txData.txheight, duckDb); withdrawData.push({ basePrice: calculatePriceByPool(BigInt(baseAssetAmount), BigInt(quoteAssetAmount)), @@ -296,7 +334,13 @@ async function parseTxs(txs: Tx[], duckDb: DuckDb): Promise { const sender = msg.sender; const wasmAttributes = parseWasmEvents(msg.logs.events); swapOpsData.push(...extractSwapOperations(basicTxData, wasmAttributes)); - const provideLiquidityData = extractMsgProvideLiquidity(basicTxData, msg.msg, sender); + const provideLiquidityData = await extractMsgProvideLiquidity( + basicTxData, + msg.msg, + sender, + wasmAttributes, + duckDb + ); if (provideLiquidityData) provideLiquidityOpsData.push(provideLiquidityData); withdrawLiquidityOpsData.push( ...(await extractMsgWithdrawLiquidity(basicTxData, wasmAttributes, sender, duckDb)) From 37d00c3e5981e11f441b7ce159c7f1f3575ae6b7 Mon Sep 17 00:00:00 2001 From: trungbach Date: Thu, 24 Aug 2023 18:34:46 +0700 Subject: [PATCH 08/46] test: added testcase for pool helper --- packages/oraidex-sync/src/constants.ts | 1 + packages/oraidex-sync/src/helper.ts | 21 +- packages/oraidex-sync/src/index.ts | 2 +- packages/oraidex-sync/src/pairs.ts | 21 +- packages/oraidex-sync/src/parse.ts | 41 --- packages/oraidex-sync/src/poolHelper.ts | 153 +++++----- packages/oraidex-sync/src/query.ts | 4 +- packages/oraidex-sync/src/tx-parsing.ts | 3 +- packages/oraidex-sync/src/types.ts | 1 + packages/oraidex-sync/tests/parse.spec.ts | 73 ----- .../oraidex-sync/tests/pool-helper.spec.ts | 274 ++++++++++++++++++ 11 files changed, 388 insertions(+), 206 deletions(-) delete mode 100644 packages/oraidex-sync/src/parse.ts delete mode 100644 packages/oraidex-sync/tests/parse.spec.ts create mode 100644 packages/oraidex-sync/tests/pool-helper.spec.ts diff --git a/packages/oraidex-sync/src/constants.ts b/packages/oraidex-sync/src/constants.ts index 69012d9d..be57ac7a 100644 --- a/packages/oraidex-sync/src/constants.ts +++ b/packages/oraidex-sync/src/constants.ts @@ -13,3 +13,4 @@ export const osmosisIbcDenom = "ibc/9C4DCD21B48231D0BC2AC3D1B74A864746B37E429269 export const tenAmountInDecimalSix = 10000000; export const truncDecimals = 6; export const atomic = 10 ** truncDecimals; +export const oraiInfo = { native_token: { denom: ORAI } }; diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index 5d41be25..b5d75522 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -1,9 +1,11 @@ import { AssetInfo, CosmWasmClient, + OraiswapFactoryQueryClient, OraiswapPairQueryClient, OraiswapPairTypes, OraiswapRouterQueryClient, + PairInfo, SwapOperation } from "@oraichain/oraidex-contracts-sdk"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; @@ -20,6 +22,7 @@ import { SwapOperationData, WithdrawLiquidityOperationData } from "./types"; +import { getPairByAssetInfos } from "./poolHelper"; export function toObject(data: any) { return JSON.parse( @@ -449,6 +452,21 @@ function convertDateToSecond(date: Date): number { return Math.round(date.valueOf() / 1000); } +async function getPairInfoFromAssets( + assetInfos: [AssetInfo, AssetInfo] +): Promise> { + const pair = getPairByAssetInfos(assetInfos); + const factoryClient = new OraiswapFactoryQueryClient( + await getCosmwasmClient(), + pair.factoryV1 ? process.env.FACTORY_CONTACT_ADDRESS_V1 : process.env.FACTORY_CONTACT_ADDRESS_V2 + ); + const pairInfo = await factoryClient.pair({ assetInfos }); + return { + contract_addr: pairInfo.contract_addr, + commission_rate: pairInfo.contract_addr + }; +} + export { calculatePriceByPool, convertDateToSecond, @@ -463,5 +481,6 @@ export { getSpecificDateBeforeNow, getSymbolFromAsset, parseAssetInfo, - parseAssetInfoOnlyDenom + parseAssetInfoOnlyDenom, + getPairInfoFromAssets }; diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index e12fc382..f7919f1a 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -220,6 +220,6 @@ export * from "./constants"; export * from "./db"; export * from "./helper"; export * from "./pairs"; -export * from "./parse"; export * from "./query"; export * from "./types"; +export * from "./poolHelper"; diff --git a/packages/oraidex-sync/src/pairs.ts b/packages/oraidex-sync/src/pairs.ts index 0226e824..c31ed19b 100644 --- a/packages/oraidex-sync/src/pairs.ts +++ b/packages/oraidex-sync/src/pairs.ts @@ -21,11 +21,13 @@ import { PairMapping } from "./types"; export const pairs: PairMapping[] = [ { asset_infos: [{ token: { contract_addr: airiCw20Adress } }, { native_token: { denom: ORAI } }], - symbols: ["AIRI", "ORAI"] + symbols: ["AIRI", "ORAI"], + factoryV1: true }, { asset_infos: [{ token: { contract_addr: oraixCw20Address } }, { native_token: { denom: ORAI } }], - symbols: ["ORAIX", "ORAI"] + symbols: ["ORAIX", "ORAI"], + factoryV1: true }, { asset_infos: [{ token: { contract_addr: scOraiCw20Address } }, { native_token: { denom: ORAI } }], @@ -33,15 +35,18 @@ export const pairs: PairMapping[] = [ }, { asset_infos: [{ native_token: { denom: ORAI } }, { native_token: { denom: atomIbcDenom } }], - symbols: ["ORAI", "ATOM"] + symbols: ["ORAI", "ATOM"], + factoryV1: true }, { asset_infos: [{ native_token: { denom: ORAI } }, { token: { contract_addr: usdtCw20Address } }], - symbols: ["ORAI", "USDT"] + symbols: ["ORAI", "USDT"], + factoryV1: true }, { asset_infos: [{ token: { contract_addr: kwtCw20Address } }, { native_token: { denom: ORAI } }], - symbols: ["KWT", "ORAI"] + symbols: ["KWT", "ORAI"], + factoryV1: true }, { asset_infos: [ @@ -50,11 +55,13 @@ export const pairs: PairMapping[] = [ native_token: { denom: osmosisIbcDenom } } ], - symbols: ["ORAI", "OSMO"] + symbols: ["ORAI", "OSMO"], + factoryV1: true }, { asset_infos: [{ token: { contract_addr: milkyCw20Address } }, { token: { contract_addr: usdtCw20Address } }], - symbols: ["MILKY", "USDT"] + symbols: ["MILKY", "USDT"], + factoryV1: true }, { asset_infos: [{ native_token: { denom: ORAI } }, { token: { contract_addr: usdcCw20Address } }], diff --git a/packages/oraidex-sync/src/parse.ts b/packages/oraidex-sync/src/parse.ts deleted file mode 100644 index bfd4f0b4..00000000 --- a/packages/oraidex-sync/src/parse.ts +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Functions that help parse from this type to other type. - */ - -import { AssetInfo } from "@oraichain/oraidex-contracts-sdk"; -import { parseAssetInfoOnlyDenom } from "./helper"; -import { pairs } from "./pairs"; - -function parseDenomToAssetLiquidity([fromDenom, toDenom]: [string, string]): [AssetInfo, AssetInfo] { - const findedPair = pairs.find((pair) => { - const denoms = [parseAssetInfoOnlyDenom(pair.asset_infos[0]), parseAssetInfoOnlyDenom(pair.asset_infos[1])]; - return denoms.includes(fromDenom) && denoms.includes(toDenom); - }); - if (!findedPair) return null; - - return findedPair.asset_infos; -} - -/** - * Check pool: - * if pool has native_token ORAI -> dont has fee - * else if pool has native_token not ORAI -> has fee - * otherwise it has 2 cw20 token -> dont has fee - * @param [fromDenom, toDenom]: denom in a pool - * @returns is pool has fee: boolean - */ -function isPoolHasFee(assetInfos: [AssetInfo, AssetInfo]): boolean { - let hasNative = false; - for (const asset of assetInfos) { - if ("native_token" in asset) { - hasNative = true; - if (asset.native_token.denom === "orai") { - return false; - } - } - } - if (hasNative) return true; - return false; -} - -export { parseDenomToAssetLiquidity, isPoolHasFee }; diff --git a/packages/oraidex-sync/src/poolHelper.ts b/packages/oraidex-sync/src/poolHelper.ts index 42c305ce..a177ab96 100644 --- a/packages/oraidex-sync/src/poolHelper.ts +++ b/packages/oraidex-sync/src/poolHelper.ts @@ -1,26 +1,39 @@ import { MulticallQueryClient } from "@oraichain/common-contracts-sdk"; +import { Asset, AssetInfo, OraiswapPairQueryClient } from "@oraichain/oraidex-contracts-sdk"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; -import { getAllPairInfos as queryAllPairInfos, getPoolInfos as queryPoolInfos } from "./query"; +import { ORAI, oraiInfo, usdtCw20Address } from "./constants"; import { calculatePriceByPool, fetchPoolInfoAmount, getCosmwasmClient, - parseAssetInfo, + getPairInfoFromAssets, parseAssetInfoOnlyDenom } from "./helper"; -import { - Asset, - AssetInfo, - OraiswapFactoryQueryClient, - OraiswapPairQueryClient, - PairInfo -} from "@oraichain/oraidex-contracts-sdk"; -import { ORAI, usdtCw20Address } from "./constants"; -import { PairInfoData, PairMapping } from "./types"; import { pairs } from "./pairs"; +import { queryPoolInfos } from "./query"; +import { PairInfoData, PairMapping } from "./types"; // use this type to determine the ratio of price of base to the quote or vice versa export type RatioDirection = "base_in_quote" | "quote_in_base"; + +/** + * Check pool if has native token is not ORAI -> has fee + * @returns boolean + */ +function isPoolHasFee(assetInfos: [AssetInfo, AssetInfo]): boolean { + let hasNative = false; + for (const asset of assetInfos) { + if ("native_token" in asset) { + hasNative = true; + if (asset.native_token.denom === "orai") { + return false; + } + } + } + if (hasNative) return true; + return false; +} + async function getPoolInfos(pairAddrs: string[], wantedHeight?: number): Promise { // adjust the query height to get data from the past const cosmwasmClient = await getCosmwasmClient(); @@ -30,60 +43,9 @@ async function getPoolInfos(pairAddrs: string[], wantedHeight?: number): Promise process.env.MULTICALL_CONTRACT_ADDRESS || "orai1q7x644gmf7h8u8y6y8t9z9nnwl8djkmspypr6mxavsk9ual7dj0sxpmgwd" ); const res = await queryPoolInfos(pairAddrs, multicall); - // reset query client to latest for other functions to call return res; } -function calculateFeeByAsset(asset: Asset, shareRatio: number): Asset { - const TAX_CAP = 10 ** 6; - const TAX_RATE = 0.3; - // just native_token not ORAI has fee - if (!("native_token" in asset.info)) return null; - const amount = +asset.amount; - const refundAmount = amount * shareRatio; - const fee = Math.min(refundAmount - (refundAmount * 1) / (TAX_RATE + 1), TAX_CAP); - return { - amount: fee.toString(), - info: asset.info - }; -} - -// find pool match this asset with orai => calculate price this asset token in ORAI. -// then, calculate price of this asset token in USDT based on price ORAI in USDT. -async function getPriceAssetByUsdt(asset: AssetInfo): Promise { - const foundPair = pairs.find((pair) => { - const denoms = [parseAssetInfoOnlyDenom(pair.asset_infos[0]), parseAssetInfoOnlyDenom(pair.asset_infos[1])]; - return denoms.includes(parseAssetInfoOnlyDenom(asset)) && denoms.includes(ORAI); - }); - if (!foundPair) return 0; - - const ratioDirection: RatioDirection = - parseAssetInfoOnlyDenom(foundPair.asset_infos[0]) === ORAI ? "quote_in_base" : "base_in_quote"; - const priceInOrai = await getPriceByAsset(foundPair.asset_infos, ratioDirection); - const priceOraiInUsdt = await getOraiPrice(); - console.log({ asset, priceInOrai, priceOraiInUsdt }); - return priceInOrai * priceOraiInUsdt; -} - -async function calculateFeeByUsdt(fee: Asset): Promise { - if (!fee) return 0; - const priceInUsdt = await getPriceAssetByUsdt(fee.info); - return priceInUsdt * +fee.amount; -} - -async function getPairInfos(): Promise { - const cosmwasmClient = await getCosmwasmClient(); - const firstFactoryClient = new OraiswapFactoryQueryClient( - cosmwasmClient, - "orai1hemdkz4xx9kukgrunxu3yw0nvpyxf34v82d2c8" - ); - const secondFactoryClient = new OraiswapFactoryQueryClient( - cosmwasmClient, - "orai167r4ut7avvgpp3rlzksz6vw5spmykluzagvmj3ht845fjschwugqjsqhst" - ); - return queryAllPairInfos(firstFactoryClient, secondFactoryClient); -} - function getPairByAssetInfos(assetInfos: [AssetInfo, AssetInfo]): PairMapping { return pairs.find((pair) => { const [baseAsset, quoteAsset] = pair.asset_infos; @@ -97,33 +59,57 @@ function getPairByAssetInfos(assetInfos: [AssetInfo, AssetInfo]): PairMapping { // get price ORAI in USDT base on ORAI/USDT pool. async function getOraiPrice(): Promise { const usdtInfo = { token: { contract_addr: usdtCw20Address } }; - const oraiInfo = { native_token: { denom: ORAI } }; const oraiUsdtPair = getPairByAssetInfos([oraiInfo, usdtInfo]); const ratioDirection: RatioDirection = parseAssetInfoOnlyDenom(oraiUsdtPair.asset_infos[0]) === ORAI ? "base_in_quote" : "quote_in_base"; return getPriceByAsset([oraiInfo, usdtInfo], ratioDirection); } -async function getPriceByAsset( - [baseAsset, quoteAsset]: [AssetInfo, AssetInfo], - ratioDirection: RatioDirection -): Promise { - // TODO: currently we get all pairinfo then find orai/usdt pair, but it slow, so need to refactor this ops - const allPairInfos = await getPairInfos(); - const pool = allPairInfos.find( - (pair) => - parseAssetInfo(pair.asset_infos[0]) === parseAssetInfo(baseAsset) && - parseAssetInfo(pair.asset_infos[1]) === parseAssetInfo(quoteAsset) - ); - if (!pool) return 0; +// get pair of assets then query info from contract to calculate price asset. +async function getPriceByAsset(assetInfos: [AssetInfo, AssetInfo], ratioDirection: RatioDirection): Promise { + const pairInfo = await getPairInfoFromAssets(assetInfos); // offer: orai, ask: usdt -> price offer in ask = calculatePriceByPool([ask, offer]) // offer: orai, ask: atom -> price ask in offer = calculatePriceByPool([offer, ask]) - const { offerPoolAmount, askPoolAmount } = await fetchPoolInfoAmount(baseAsset, quoteAsset, pool.contract_addr); - const assetPrice = calculatePriceByPool(askPoolAmount, offerPoolAmount, +pool.commission_rate); + const { offerPoolAmount, askPoolAmount } = await fetchPoolInfoAmount(...assetInfos, pairInfo.contract_addr); + const assetPrice = calculatePriceByPool(askPoolAmount, offerPoolAmount, +pairInfo.commission_rate); return ratioDirection === "base_in_quote" ? assetPrice : 1 / assetPrice; } +// find pool match this asset with orai => calculate price this asset token in ORAI. +// then, calculate price of this asset token in USDT based on price ORAI in USDT. +async function getPriceAssetByUsdt(asset: AssetInfo): Promise { + const foundPair = getPairByAssetInfos([asset, oraiInfo]); + if (!foundPair) return 0; + + const ratioDirection: RatioDirection = + parseAssetInfoOnlyDenom(foundPair.asset_infos[0]) === ORAI ? "quote_in_base" : "base_in_quote"; + const priceInOrai = await getPriceByAsset(foundPair.asset_infos, ratioDirection); + const priceOraiInUsdt = await getOraiPrice(); + console.log({ asset, priceInOrai, priceOraiInUsdt }); + return priceInOrai * priceOraiInUsdt; +} + +async function calculateFeeByUsdt(fee: Asset): Promise { + if (!fee) return 0; + const priceInUsdt = await getPriceAssetByUsdt(fee.info); + return priceInUsdt * +fee.amount; +} + +function calculateFeeByAsset(asset: Asset, shareRatio: number): Asset { + const TAX_CAP = 10 ** 6; + const TAX_RATE = 0.3; + // just native_token not ORAI has fee + if (!("native_token" in asset.info)) return null; + const amount = +asset.amount; + const refundAmount = amount * shareRatio; + const fee = Math.min(refundAmount - (refundAmount * 1) / (TAX_RATE + 1), TAX_CAP); + return { + amount: fee.toString(), + info: asset.info + }; +} + /** * First, calculate fee by offer asset & askAsset * then, calculate fee of those asset to ORAI @@ -131,7 +117,7 @@ async function getPriceByAsset( * @param pair * @param txHeight * @param withdrawnShare - * @returns + * @returns fee in USDT */ async function calculateLiquidityFee(pair: PairInfoData, txHeight: number, withdrawnShare: number): Promise { const cosmwasmClient = await getCosmwasmClient(); @@ -151,4 +137,13 @@ async function calculateLiquidityFee(pair: PairInfoData, txHeight: number, withd return BigInt(Math.round(feeByUsdt)); } -export { getPoolInfos, calculateLiquidityFee, getOraiPrice }; +export { + calculateFeeByAsset, + calculateLiquidityFee, + getOraiPrice, + getPairByAssetInfos, + getPoolInfos, + getPriceAssetByUsdt, + getPriceByAsset, + isPoolHasFee +}; diff --git a/packages/oraidex-sync/src/query.ts b/packages/oraidex-sync/src/query.ts index 4ba8ec2a..caa45e3b 100644 --- a/packages/oraidex-sync/src/query.ts +++ b/packages/oraidex-sync/src/query.ts @@ -11,7 +11,7 @@ import { pairs } from "./pairs"; import { findAssetInfoPathToUsdt, generateSwapOperations, parseAssetInfoOnlyDenom, toDisplay } from "./helper"; import { tenAmountInDecimalSix, usdtCw20Address } from "./constants"; -async function getPoolInfos(pairAddrs: string[], multicall: MulticallReadOnlyInterface): Promise { +async function queryPoolInfos(pairAddrs: string[], multicall: MulticallReadOnlyInterface): Promise { // adjust the query height to get data from the past const res = await multicall.tryAggregate({ queries: pairAddrs.map((pair) => { @@ -75,4 +75,4 @@ async function simulateSwapPrice(pairPath: AssetInfo[], router: OraiswapRouterRe } } -export { getAllPairInfos, getPoolInfos, simulateSwapPriceWithUsdt, simulateSwapPrice }; +export { getAllPairInfos, queryPoolInfos, simulateSwapPriceWithUsdt, simulateSwapPrice }; diff --git a/packages/oraidex-sync/src/tx-parsing.ts b/packages/oraidex-sync/src/tx-parsing.ts index cb820973..b6a2a11a 100644 --- a/packages/oraidex-sync/src/tx-parsing.ts +++ b/packages/oraidex-sync/src/tx-parsing.ts @@ -31,9 +31,8 @@ import { removeOpsDuplication } from "./helper"; import { pairs } from "./pairs"; -import { isPoolHasFee } from "./parse"; import { DuckDb } from "./db"; -import { calculateLiquidityFee } from "./poolHelper"; +import { calculateLiquidityFee, isPoolHasFee } from "./poolHelper"; function parseWasmEvents(events: readonly Event[]): (readonly Attribute[])[] { return events.filter((event) => event.type === "wasm").map((event) => event.attributes); diff --git a/packages/oraidex-sync/src/types.ts b/packages/oraidex-sync/src/types.ts index 16689aed..6c46a0af 100644 --- a/packages/oraidex-sync/src/types.ts +++ b/packages/oraidex-sync/src/types.ts @@ -132,6 +132,7 @@ export type OraiswapPairCw20HookMsg = { export type PairMapping = { asset_infos: [AssetInfo, AssetInfo]; symbols: [string, string]; + factoryV1?: boolean; }; export type InitialData = { diff --git a/packages/oraidex-sync/tests/parse.spec.ts b/packages/oraidex-sync/tests/parse.spec.ts deleted file mode 100644 index d2cb9bd6..00000000 --- a/packages/oraidex-sync/tests/parse.spec.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { AssetInfo } from "@oraichain/oraidex-contracts-sdk"; -import { isPoolHasFee, parseDenomToAssetLiquidity } from "../src/parse"; -import { - ORAI, - airiCw20Adress, - atomIbcDenom, - milkyCw20Address, - scAtomCw20Address, - usdtCw20Address -} from "../src/constants"; - -describe("test-parse", () => { - it.each<[[string, string], [AssetInfo, AssetInfo] | null]>([ - [["invalidDenom", "invalidDenom"], null], - [[ORAI, "invalidDenom"], null], - [ - [ORAI, airiCw20Adress], - [{ token: { contract_addr: airiCw20Adress } }, { native_token: { denom: ORAI } }] - ] - ])( - "test-parseDenomToAssetLiquidity-with-denom-%s-should-return-correctly-asset-%o", - (denoms: [string, string], expectedResult: [AssetInfo, AssetInfo] | null) => { - const result = parseDenomToAssetLiquidity(denoms); - // ✅ PASS - expect(result).toStrictEqual(expectedResult); - } - ); - - it.each<[string, [AssetInfo, AssetInfo], boolean]>([ - [ - "has-both-native-token-that-contain-ORAI-should-return: false", - [{ native_token: { denom: ORAI } }, { native_token: { denom: atomIbcDenom } }], - false - ], - // [ - // // NOTE: currently this case not exist, but in future maybe - // "has-both-native-token-that-NOT-contain-ORAI-should-return: true", - // [osmosisIbcDenom, atomIbcDenom], - // true - // ], - [ - "has-one-native-token-that-NOT-contain-ORAI-should-return: true", - [ - { native_token: { denom: atomIbcDenom } }, - { - token: { - contract_addr: scAtomCw20Address - } - } - ], - true - ], - [ - "NOT-has-native-token-should-return-is-has-fee: false", - [ - { - token: { - contract_addr: milkyCw20Address - } - }, - { - token: { - contract_addr: usdtCw20Address - } - } - ], - false - ] - ])("test-isPoolHasFee-with-pool-%s", (_caseName, assetInfos, expectIsHasFee) => { - const result = isPoolHasFee(assetInfos); - expect(result).toBe(expectIsHasFee); - }); -}); diff --git a/packages/oraidex-sync/tests/pool-helper.spec.ts b/packages/oraidex-sync/tests/pool-helper.spec.ts new file mode 100644 index 00000000..8f3e3235 --- /dev/null +++ b/packages/oraidex-sync/tests/pool-helper.spec.ts @@ -0,0 +1,274 @@ +import { Asset, AssetInfo, PairInfo } from "@oraichain/oraidex-contracts-sdk"; +import { + ORAI, + airiCw20Adress, + atomIbcDenom, + milkyCw20Address, + scAtomCw20Address, + usdtCw20Address +} from "../src/constants"; + +import { PairMapping } from "../src/types"; +import * as helper from "../src/helper"; +import * as poolHelper from "../src/poolHelper"; +import { mock } from "node:test"; +describe("test-pool-helper", () => { + afterEach(() => { + jest.clearAllMocks(); + // jest.resetAllMocks(); + // jest.restoreAllMocks(); + }); + + it.each<[string, [AssetInfo, AssetInfo], boolean]>([ + [ + "has-both-native-token-that-contain-ORAI-should-return: false", + [{ native_token: { denom: ORAI } }, { native_token: { denom: atomIbcDenom } }], + false + ], + // [ + // // NOTE: currently this case not exist, but in future maybe + // "has-both-native-token-that-NOT-contain-ORAI-should-return: true", + // [osmosisIbcDenom, atomIbcDenom], + // true + // ], + [ + "has-one-native-token-that-NOT-contain-ORAI-should-return: true", + [ + { native_token: { denom: atomIbcDenom } }, + { + token: { + contract_addr: scAtomCw20Address + } + } + ], + true + ], + [ + "NOT-has-native-token-should-return-is-has-fee: false", + [ + { + token: { + contract_addr: milkyCw20Address + } + }, + { + token: { + contract_addr: usdtCw20Address + } + } + ], + false + ] + ])("test-isPoolHasFee-with-pool-%s", (_caseName, assetInfos, expectIsHasFee) => { + const result = poolHelper.isPoolHasFee(assetInfos); + expect(result).toBe(expectIsHasFee); + }); + + it.each([ + [ + "test-calculateFeeByAsset-with-case-asset-is-cw20-token-should-return-null", + { + info: { + token: { + contract_addr: airiCw20Adress + } + }, + amount: "100" + }, + null + ], + [ + "test-calculateFeeByAsset-with-case-asset-is-native-token-should-return-correctly-fee", + { + info: { + native_token: { + denom: atomIbcDenom + } + }, + amount: "100" + }, + { + amount: "1000000", + info: { native_token: { denom: atomIbcDenom } } + } + ] + ])("%s", (_caseName: string, inputAsset: Asset, expectedFee: Asset | null) => { + const shareRatio = 100000; + const result = poolHelper.calculateFeeByAsset(inputAsset, shareRatio); + expect(result).toStrictEqual(expectedFee); + }); + + it.each<[string, [AssetInfo, AssetInfo], PairMapping | undefined]>([ + [ + "assetInfos-valid-in-list-pairs", + [ + { + token: { + contract_addr: usdtCw20Address + } + }, + { + native_token: { + denom: ORAI + } + } + ], + { + asset_infos: [{ native_token: { denom: ORAI } }, { token: { contract_addr: usdtCw20Address } }], + symbols: ["ORAI", "USDT"], + factoryV1: true + } + ], + [ + "assetInfos-invalid-in-list-pairs", + [ + { + token: { + contract_addr: "invalid" + } + }, + { + native_token: { + denom: atomIbcDenom + } + } + ], + undefined + ] + ])( + "test-getPairByAssetInfos-with-%s-should-return-correctly-pair", + (_caseName: string, assetInfos: [AssetInfo, AssetInfo], expectedPair: PairMapping | undefined) => { + const result = poolHelper.getPairByAssetInfos(assetInfos); + expect(result).toStrictEqual(expectedPair); + } + ); + + // it.each<[[AssetInfo, AssetInfo], Pick]>([ + // [ + // [ + // { + // native_token: { + // denom: ORAI + // } + // }, + // { + // token: { + // contract_addr: usdtCw20Address + // } + // } + // ], + // { + // contract_addr: "orai123", + // commission_rate: "0.003" + // } + // ] + // ])("test-getPairInfoFromAssets-should-return-correctly-pair-info", async (assetInfos, expectedPairInfo) => { + // const getPairInfoFromAssetsSpy = jest.spyOn(helper, "getPairInfoFromAssets"); + // const mockPairInfoFromAssets = { + // commission_rate: "0.003", + // contract_addr: "orai1234" + // }; + // getPairInfoFromAssetsSpy.mockResolvedValue(mockPairInfoFromAssets); + // const result = await helper.getPairInfoFromAssets(assetInfos); + // console.log({ result }); + // // expect(result).toStrictEqual(expectedPairInfo); + // }); + + describe("test-calculate-price-group-funcs", () => { + afterEach(jest.clearAllMocks); + it.each<[[AssetInfo, AssetInfo], poolHelper.RatioDirection, number]>([ + [ + [ + { + native_token: { + denom: ORAI + } + }, + { + token: { + contract_addr: usdtCw20Address + } + } + ], + "base_in_quote", + 0.5 + ], + [ + [ + { + native_token: { + denom: ORAI + } + }, + { + token: { + contract_addr: usdtCw20Address + } + } + ], + "quote_in_base", + 2 + ] + ])("test-getPriceByAsset-should-return-correctly-price", async (assetInfos, ratioDirection, expectedPrice) => { + // Mock the return values for fetchPoolInfoAmount and calculatePriceByPool + const mockAskPoolAmount = 1000n; + const mockOfferPoolAmount = 2000n; + const mockAssetPrice = 0.5; + const mockPairInfoFromAssets = { + commission_rate: "0.003", + contract_addr: "orai1234" + }; + + const getPairInfoFromAssetsSpy = jest.spyOn(helper, "getPairInfoFromAssets"); + getPairInfoFromAssetsSpy.mockResolvedValue(mockPairInfoFromAssets); + + const fetchPoolInfoAmountSpy = jest.spyOn(helper, "fetchPoolInfoAmount"); + const calculatePriceByPoolSpy = jest.spyOn(helper, "calculatePriceByPool"); + // Mock the fetchPoolInfoAmount function + fetchPoolInfoAmountSpy.mockResolvedValue({ + askPoolAmount: mockAskPoolAmount, + offerPoolAmount: mockOfferPoolAmount + }); + + // Mock the calculatePriceByPool function + calculatePriceByPoolSpy.mockReturnValue(mockAssetPrice); + + const result = await poolHelper.getPriceByAsset(assetInfos, ratioDirection); + expect(result).toEqual(expectedPrice); + }); + + it.each([ + [ + { + native_token: { + denom: "not-pair-with-orai" + } + }, + 0 + ], + [ + { + native_token: { + denom: atomIbcDenom + } + }, + 1 + ] + ])( + "test-getPriceAssetByUsdt-should-return-price-of-asset-in-USDT", + async (assetInfo: AssetInfo, expectedPrice: number) => { + const mockPriceByAssetValue = 4; + const mockOraiPrice = 2; + + const getPriceByAssetSpy = jest.spyOn(poolHelper, "getPriceByAsset"); + getPriceByAssetSpy.mockResolvedValue(mockPriceByAssetValue); + + const getOraiPriceSpy = jest.spyOn(poolHelper, "getOraiPrice"); + getOraiPriceSpy.mockResolvedValue(mockOraiPrice); + + const result = await poolHelper.getPriceAssetByUsdt(assetInfo); + expect(result).toEqual(expectedPrice); + } + ); + }); +}); From 22ee64bb95fd16d610a191b7e93744abcbda5418 Mon Sep 17 00:00:00 2001 From: trungbach Date: Fri, 25 Aug 2023 11:40:34 +0700 Subject: [PATCH 09/46] update: calculate volume swap & liquidity of pair without test --- packages/oraidex-sync/src/constants.ts | 1 + packages/oraidex-sync/src/db.ts | 82 +++++++++++++++- packages/oraidex-sync/src/helper.ts | 2 +- packages/oraidex-sync/src/index.ts | 109 +++++++++++++++++---- packages/oraidex-sync/src/poolHelper.ts | 22 +++-- packages/oraidex-sync/src/types.ts | 2 + packages/oraidex-sync/tests/helper.spec.ts | 25 +++-- 7 files changed, 203 insertions(+), 40 deletions(-) diff --git a/packages/oraidex-sync/src/constants.ts b/packages/oraidex-sync/src/constants.ts index be57ac7a..a2235b9d 100644 --- a/packages/oraidex-sync/src/constants.ts +++ b/packages/oraidex-sync/src/constants.ts @@ -14,3 +14,4 @@ export const tenAmountInDecimalSix = 10000000; export const truncDecimals = 6; export const atomic = 10 ** truncDecimals; export const oraiInfo = { native_token: { denom: ORAI } }; +export const usdtInfo = { token: { contract_addr: usdcCw20Address } }; diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index 775eb982..1d037353 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -10,7 +10,8 @@ import { VolumeRange, WithdrawLiquidityOperationData, GetCandlesQuery, - GetFeeSwap + GetFeeSwap, + GetVolumeQuery } from "./types"; import fs, { rename } from "fs"; import { isoToTimestampNumber, parseAssetInfo, renameKey, replaceAllNonAlphaBetChar, toObject } from "./helper"; @@ -382,8 +383,9 @@ export class DuckDb { async getFeeSwap(payload: GetFeeSwap): Promise { const { offerDenom, askDenom, startTime, endTime } = payload; - const result = await this.conn.all( - ` + const [feeRightDirection, feeReverseDirection] = await Promise.all([ + this.conn.all( + ` SELECT sum(commissionAmount + taxAmount) as totalFee, FROM swap_ops_data @@ -391,6 +393,42 @@ export class DuckDb { AND timestamp <= ? AND offerDenom = ? AND askDenom = ? + `, + startTime, + endTime, + offerDenom, + askDenom + ), + this.conn.all( + ` + SELECT + sum(commissionAmount + taxAmount) as totalFee, + FROM swap_ops_data + WHERE timestamp >= ? + AND timestamp <= ? + AND offerDenom = ? + AND askDenom = ? + `, + startTime, + endTime, + askDenom, + offerDenom + ) + ]); + return BigInt(feeRightDirection[0].totalFee ?? 0 + feeReverseDirection[0].totalFee ?? 0); + } + + async getFeeLiquidity(payload: GetFeeSwap): Promise { + const { offerDenom, askDenom, startTime, endTime } = payload; + const result = await this.conn.all( + ` + SELECT + sum(taxRate) as totalFee, + FROM lp_ops_data + WHERE timestamp >= ? + AND timestamp <= ? + AND baseTokenDenom = ? + AND quoteTokenDenom = ? `, startTime, endTime, @@ -399,4 +437,42 @@ export class DuckDb { ); return BigInt(result[0].totalFee ?? 0); } + + async getVolumeSwap(payload: GetVolumeQuery): Promise { + const { pair, startTime, endTime } = payload; + const result = await this.conn.all( + ` + SELECT + cast(sum(volume) as UBIGINT) as totalVolume, + FROM swap_ohlcv + WHERE timestamp >= ? + AND timestamp <= ? + AND pair = ? + `, + startTime, + endTime, + pair + ); + return result[0]?.totalVolume ?? 0; + } + + async getVolumeLiquidity(payload: GetFeeSwap): Promise { + const { offerDenom, askDenom, startTime, endTime } = payload; + const result = await this.conn.all( + ` + SELECT + sum(baseTokenAmount) as totalVolume, + FROM lp_ops_data + WHERE timestamp >= ? + AND timestamp <= ? + AND baseTokenDenom = ? + AND quoteTokenDenom = ? + `, + startTime, + endTime, + offerDenom, + askDenom + ); + return BigInt(result[0].totalVolume ?? 0); + } } diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index b5d75522..cd8012d1 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -463,7 +463,7 @@ async function getPairInfoFromAssets( const pairInfo = await factoryClient.pair({ assetInfos }); return { contract_addr: pairInfo.contract_addr, - commission_rate: pairInfo.contract_addr + commission_rate: pairInfo.commission_rate }; } diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index f7919f1a..d84c8dc1 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -10,7 +10,7 @@ import { getSymbolFromAsset, parseAssetInfoOnlyDenom } from "./helper"; -import { getPoolInfos } from "./poolHelper"; +import { getPoolInfos, getPriceAssetByUsdt, getPriceByAsset } from "./poolHelper"; import { getAllPairInfos } from "./query"; import { parseAssetInfo, parseTxs } from "./tx-parsing"; import { @@ -21,6 +21,7 @@ import { TxAnlysisResult, WithdrawLiquidityOperationData } from "./types"; +import { pairs } from "./pairs"; class WriteOrders extends WriteData { private firstWrite: boolean; @@ -103,41 +104,110 @@ class OraiDexSync { return getAllPairInfos(firstFactoryClient, secondFactoryClient); } - async getSwapFeePair(asset_infos: [AssetInfo, AssetInfo], startTime: Date, endTime: Date): Promise { - const [swapFee, swapFeeReverse] = await Promise.all([ + async getFeePair(asset_infos: [AssetInfo, AssetInfo], startTime: Date, endTime: Date): Promise { + const [swapFee, liquidityFee] = await Promise.all([ this.duckDb.getFeeSwap({ offerDenom: parseAssetInfoOnlyDenom(asset_infos[0]), askDenom: parseAssetInfoOnlyDenom(asset_infos[1]), startTime: convertDateToSecond(startTime), endTime: convertDateToSecond(endTime) }), - this.duckDb.getFeeSwap({ - offerDenom: parseAssetInfoOnlyDenom(asset_infos[1]), - askDenom: parseAssetInfoOnlyDenom(asset_infos[0]), + this.duckDb.getFeeLiquidity({ + offerDenom: parseAssetInfoOnlyDenom(asset_infos[0]), + askDenom: parseAssetInfoOnlyDenom(asset_infos[1]), startTime: convertDateToSecond(startTime), endTime: convertDateToSecond(endTime) }) ]); - return swapFee + swapFeeReverse; + return swapFee + liquidityFee; } - async getAllFees(pairInfos: PairInfo[]): Promise { + async getVolumeSwap( + [baseAssetInfo, quoteAssetInfo]: [AssetInfo, AssetInfo], + startTime: Date, + endTime: Date + ): Promise { + const pair = `${parseAssetInfoOnlyDenom(baseAssetInfo)}-${parseAssetInfoOnlyDenom(quoteAssetInfo)}`; + const volumePairInBaseAsset = await this.duckDb.getVolumeSwap({ + pair, + startTime: convertDateToSecond(startTime), + endTime: convertDateToSecond(endTime) + }); + let priceBaseAssetInUsdt = await getPriceAssetByUsdt(baseAssetInfo); + + // it means this asset not pair with ORAI + // in our pairs, if base asset not pair with ORAI, surely quote asset will pair with ORAI + if (priceBaseAssetInUsdt === 0) { + const priceQuoteAssetInUsdt = await getPriceAssetByUsdt(quoteAssetInfo); + const priceBaseInQuote = await getPriceByAsset([baseAssetInfo, quoteAssetInfo], "base_in_quote"); + priceBaseAssetInUsdt = priceBaseInQuote * priceQuoteAssetInUsdt; + } + const volumeInUsdt = priceBaseAssetInUsdt * Number(volumePairInBaseAsset); + return BigInt(Math.round(volumeInUsdt)); + } + + async getVolumeLiquidity( + [baseAssetInfo, quoteAssetInfo]: [AssetInfo, AssetInfo], + startTime: Date, + endTime: Date + ): Promise { + const volumePairInBaseAsset = await this.duckDb.getVolumeLiquidity({ + offerDenom: parseAssetInfoOnlyDenom(baseAssetInfo), + askDenom: parseAssetInfoOnlyDenom(quoteAssetInfo), + startTime: convertDateToSecond(startTime), + endTime: convertDateToSecond(endTime) + }); + let priceBaseAssetInUsdt = await getPriceAssetByUsdt(baseAssetInfo); + + // it means this asset not pair with ORAI + // in our pairs, if base asset not pair with ORAI, surely quote asset will pair with ORAI + if (priceBaseAssetInUsdt === 0) { + const priceQuoteAssetInUsdt = await getPriceAssetByUsdt(quoteAssetInfo); + const priceBaseInQuote = await getPriceByAsset([baseAssetInfo, quoteAssetInfo], "base_in_quote"); + priceBaseAssetInUsdt = priceBaseInQuote * priceQuoteAssetInUsdt; + } + const volumeInUsdt = priceBaseAssetInUsdt * Number(volumePairInBaseAsset); + console.log({ + volumeInUsdt, + priceBaseAssetInUsdt, + volumePairInBaseAsset, + pair: `${parseAssetInfoOnlyDenom(baseAssetInfo)}-${parseAssetInfoOnlyDenom(quoteAssetInfo)}` + }); + return BigInt(Math.round(volumeInUsdt)); + } + + async getVolumePair( + [baseAssetInfo, quoteAssetInfo]: [AssetInfo, AssetInfo], + startTime: Date, + endTime: Date + ): Promise { + const [volumeSwap, volumeLiquidity] = await Promise.all([ + this.getVolumeSwap([baseAssetInfo, quoteAssetInfo], startTime, endTime), + this.getVolumeLiquidity([baseAssetInfo, quoteAssetInfo], startTime, endTime) + ]); + return volumeSwap + volumeLiquidity; + } + + async getAllFees(): Promise { const tf = 7 * 24 * 60 * 60; // second of 7 days const currentDate = new Date(); const oneWeekBeforeNow = getSpecificDateBeforeNow(new Date(), tf); - const swapFees = await Promise.all( - pairInfos.map((pair) => this.getSwapFeePair(pair.asset_infos, oneWeekBeforeNow, currentDate)) + const allFees = await Promise.all( + pairs.map((pair) => this.getFeePair(pair.asset_infos, oneWeekBeforeNow, currentDate)) ); + return allFees; + } - // const lpFees = - return swapFees; + async getAllVolume24h(): Promise { + const tf = 100 * 24 * 60 * 60; // second of 24h * 100 + const currentDate = new Date(); + const oneDayBeforeNow = getSpecificDateBeforeNow(new Date(), tf); + const allVolumes = await Promise.all( + pairs.map((pair) => this.getVolumePair(pair.asset_infos, oneDayBeforeNow, currentDate)) + ); + return allVolumes; } - // fromIconUrl, toIconUrl: upload to other server - // volume24Hour: volume ohlcv + volume liquidity (?) - // apr: oraidex - // totalLiquidity: get liquidity in lp_ops_data with last block of pair - // fee7Days: sum of fee in swap_ops ( taxAmount + commissionAmount ) + & lp_ops fee of scatom/atom private async updateLatestPairInfos() { try { console.time("timer-updateLatestPairInfos"); @@ -147,7 +217,8 @@ class OraiDexSync { return getPairLiquidity(pair.asset_infos, pair.contract_addr); }) ); - const allFee7Days = await this.getAllFees(pairInfos); + const allFee7Days = await this.getAllFees(); + const allVolume24h = await this.getAllVolume24h(); await this.duckDb.insertPairInfos( pairInfos.map((pair, index) => { @@ -162,7 +233,7 @@ class OraiDexSync { symbols, fromIconUrl: "url1", toIconUrl: "url2", - volume24Hour: 1n, + volume24Hour: allVolume24h[index], apr: 2, totalLiquidity: allLiquidities[index], fee7Days: allFee7Days[index] diff --git a/packages/oraidex-sync/src/poolHelper.ts b/packages/oraidex-sync/src/poolHelper.ts index a177ab96..366669d1 100644 --- a/packages/oraidex-sync/src/poolHelper.ts +++ b/packages/oraidex-sync/src/poolHelper.ts @@ -1,7 +1,7 @@ import { MulticallQueryClient } from "@oraichain/common-contracts-sdk"; import { Asset, AssetInfo, OraiswapPairQueryClient } from "@oraichain/oraidex-contracts-sdk"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; -import { ORAI, oraiInfo, usdtCw20Address } from "./constants"; +import { ORAI, oraiInfo, usdtCw20Address, usdtInfo } from "./constants"; import { calculatePriceByPool, fetchPoolInfoAmount, @@ -58,7 +58,6 @@ function getPairByAssetInfos(assetInfos: [AssetInfo, AssetInfo]): PairMapping { // get price ORAI in USDT base on ORAI/USDT pool. async function getOraiPrice(): Promise { - const usdtInfo = { token: { contract_addr: usdtCw20Address } }; const oraiUsdtPair = getPairByAssetInfos([oraiInfo, usdtInfo]); const ratioDirection: RatioDirection = parseAssetInfoOnlyDenom(oraiUsdtPair.asset_infos[0]) === ORAI ? "base_in_quote" : "quote_in_base"; @@ -72,21 +71,26 @@ async function getPriceByAsset(assetInfos: [AssetInfo, AssetInfo], ratioDirectio // offer: orai, ask: atom -> price ask in offer = calculatePriceByPool([offer, ask]) const { offerPoolAmount, askPoolAmount } = await fetchPoolInfoAmount(...assetInfos, pairInfo.contract_addr); const assetPrice = calculatePriceByPool(askPoolAmount, offerPoolAmount, +pairInfo.commission_rate); - return ratioDirection === "base_in_quote" ? assetPrice : 1 / assetPrice; } // find pool match this asset with orai => calculate price this asset token in ORAI. // then, calculate price of this asset token in USDT based on price ORAI in USDT. async function getPriceAssetByUsdt(asset: AssetInfo): Promise { - const foundPair = getPairByAssetInfos([asset, oraiInfo]); - if (!foundPair) return 0; + if (parseAssetInfoOnlyDenom(asset) === parseAssetInfoOnlyDenom(usdtInfo)) return 1; + + let priceInOrai = 1; + if (parseAssetInfoOnlyDenom(asset) !== parseAssetInfoOnlyDenom(oraiInfo)) { + const foundPair = getPairByAssetInfos([asset, oraiInfo]); + if (!foundPair) return 0; + + const ratioDirection: RatioDirection = + parseAssetInfoOnlyDenom(foundPair.asset_infos[0]) === ORAI ? "quote_in_base" : "base_in_quote"; + priceInOrai = await getPriceByAsset(foundPair.asset_infos, ratioDirection); + } - const ratioDirection: RatioDirection = - parseAssetInfoOnlyDenom(foundPair.asset_infos[0]) === ORAI ? "quote_in_base" : "base_in_quote"; - const priceInOrai = await getPriceByAsset(foundPair.asset_infos, ratioDirection); const priceOraiInUsdt = await getOraiPrice(); - console.log({ asset, priceInOrai, priceOraiInUsdt }); + return priceInOrai * priceOraiInUsdt; } diff --git a/packages/oraidex-sync/src/types.ts b/packages/oraidex-sync/src/types.ts index 6c46a0af..b61c2905 100644 --- a/packages/oraidex-sync/src/types.ts +++ b/packages/oraidex-sync/src/types.ts @@ -204,3 +204,5 @@ export type GetFeeSwap = { startTime: number; endTime: number; }; + +export type GetVolumeQuery = Omit; diff --git a/packages/oraidex-sync/tests/helper.spec.ts b/packages/oraidex-sync/tests/helper.spec.ts index 3b00ab00..c128fce1 100644 --- a/packages/oraidex-sync/tests/helper.spec.ts +++ b/packages/oraidex-sync/tests/helper.spec.ts @@ -213,11 +213,13 @@ describe("test-helper", () => { expect(pairs).toEqual([ { asset_infos: [{ token: { contract_addr: airiCw20Adress } }, { native_token: { denom: ORAI } }], - symbols: ["AIRI", "ORAI"] + symbols: ["AIRI", "ORAI"], + factoryV1: true }, { asset_infos: [{ token: { contract_addr: oraixCw20Address } }, { native_token: { denom: ORAI } }], - symbols: ["ORAIX", "ORAI"] + symbols: ["ORAIX", "ORAI"], + factoryV1: true }, { asset_infos: [{ token: { contract_addr: scOraiCw20Address } }, { native_token: { denom: ORAI } }], @@ -225,15 +227,18 @@ describe("test-helper", () => { }, { asset_infos: [{ native_token: { denom: ORAI } }, { native_token: { denom: atomIbcDenom } }], - symbols: ["ORAI", "ATOM"] + symbols: ["ORAI", "ATOM"], + factoryV1: true }, { asset_infos: [{ native_token: { denom: ORAI } }, { token: { contract_addr: usdtCw20Address } }], - symbols: ["ORAI", "USDT"] + symbols: ["ORAI", "USDT"], + factoryV1: true }, { asset_infos: [{ token: { contract_addr: kwtCw20Address } }, { native_token: { denom: ORAI } }], - symbols: ["KWT", "ORAI"] + symbols: ["KWT", "ORAI"], + factoryV1: true }, { asset_infos: [ @@ -242,11 +247,13 @@ describe("test-helper", () => { native_token: { denom: osmosisIbcDenom } } ], - symbols: ["ORAI", "OSMO"] + symbols: ["ORAI", "OSMO"], + factoryV1: true }, { asset_infos: [{ token: { contract_addr: milkyCw20Address } }, { token: { contract_addr: usdtCw20Address } }], - symbols: ["MILKY", "USDT"] + symbols: ["MILKY", "USDT"], + factoryV1: true }, { asset_infos: [{ native_token: { denom: ORAI } }, { token: { contract_addr: usdcCw20Address } }], @@ -360,7 +367,9 @@ describe("test-helper", () => { "test-calculatePriceByPool-ORAI/USDT-pool-with-commision-rate=%s-should-return-price-%s-USDT", (commisionRate, expectedPrice) => { // base denom is ORAI, quote denom is USDT => base pool is ORAI, quote pool is USDT. - const result = calculatePriceByPool(BigInt(639997269712), BigInt(232967274783), commisionRate, 10 ** 6); + // const result = calculatePriceByPool(BigInt(639997269712), BigInt(232967274783), commisionRate, 10 ** 6); + const result = calculatePriceByPool(BigInt(397832351391), BigInt(193971155696), commisionRate); + console.log({ result }); expect(result.toString()).toEqual(expectedPrice); } ); From 79f5d9f8dc4fd6c08b1f04792b634a1ac6a95a3a Mon Sep 17 00:00:00 2001 From: trungbach Date: Fri, 25 Aug 2023 16:53:47 +0700 Subject: [PATCH 10/46] update: calculated APR pool without test --- packages/oraidex-sync/src/constants.ts | 14 +++ packages/oraidex-sync/src/helper.ts | 5 +- packages/oraidex-sync/src/index.ts | 10 +- packages/oraidex-sync/src/poolHelper.ts | 136 +++++++++++++++++++++++- packages/oraidex-sync/src/types.ts | 5 + 5 files changed, 159 insertions(+), 11 deletions(-) diff --git a/packages/oraidex-sync/src/constants.ts b/packages/oraidex-sync/src/constants.ts index a2235b9d..33798232 100644 --- a/packages/oraidex-sync/src/constants.ts +++ b/packages/oraidex-sync/src/constants.ts @@ -15,3 +15,17 @@ export const truncDecimals = 6; export const atomic = 10 ** truncDecimals; export const oraiInfo = { native_token: { denom: ORAI } }; export const usdtInfo = { token: { contract_addr: usdcCw20Address } }; +export const ORAIXOCH_INFO = { + token: { + contract_addr: "orai1lplapmgqnelqn253stz6kmvm3ulgdaytn89a8mz9y85xq8wd684s6xl3lt" + } +}; + +export const SEC_PER_YEAR = 60 * 60 * 24 * 365; +export const network = { + factory: process.env.FACTORY_CONTACT_ADDRESS_V1, + factory_v2: process.env.FACTORY_CONTACT_ADDRESS_V2, + router: process.env.ROUTER_CONTRACT_ADDRESS, + staking: process.env.STAKING_CONTRACT, + multicall: process.env.MULTICALL_CONTRACT_ADDRESS +}; diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index cd8012d1..51e456f0 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -17,6 +17,7 @@ import { Ohlcv, OraiDexType, PairInfoData, + PoolInfo, ProvideLiquidityOperationData, SwapDirection, SwapOperationData, @@ -411,10 +412,6 @@ function parsePoolAmount(poolInfo: OraiswapPairTypes.PoolResponse, trueAsset: As return BigInt(poolInfo.assets.find((asset) => isEqual(asset.info, trueAsset))?.amount || "0"); } -type PoolInfo = { - offerPoolAmount: bigint; - askPoolAmount: bigint; -}; async function fetchPoolInfoAmount(fromInfo: AssetInfo, toInfo: AssetInfo, pairAddr: string): Promise { const client = await getCosmwasmClient(); const pairContract = new OraiswapPairQueryClient(client, pairAddr); diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index d84c8dc1..eaee94d0 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -10,7 +10,7 @@ import { getSymbolFromAsset, parseAssetInfoOnlyDenom } from "./helper"; -import { getPoolInfos, getPriceAssetByUsdt, getPriceByAsset } from "./poolHelper"; +import { fetchAprResult, getPoolInfos, getPriceAssetByUsdt, getPriceByAsset } from "./poolHelper"; import { getAllPairInfos } from "./query"; import { parseAssetInfo, parseTxs } from "./tx-parsing"; import { @@ -208,6 +208,11 @@ class OraiDexSync { return allVolumes; } + async getAllApr(pairInfos: PairInfo[], allLiquidities: number[]): Promise { + const allApr = await fetchAprResult(pairInfos, allLiquidities); + return allApr; + } + private async updateLatestPairInfos() { try { console.time("timer-updateLatestPairInfos"); @@ -219,6 +224,7 @@ class OraiDexSync { ); const allFee7Days = await this.getAllFees(); const allVolume24h = await this.getAllVolume24h(); + const allAPr = await this.getAllApr(pairInfos, allLiquidities); await this.duckDb.insertPairInfos( pairInfos.map((pair, index) => { @@ -234,7 +240,7 @@ class OraiDexSync { fromIconUrl: "url1", toIconUrl: "url2", volume24Hour: allVolume24h[index], - apr: 2, + apr: allAPr[index], totalLiquidity: allLiquidities[index], fee7Days: allFee7Days[index] } as PairInfoData; diff --git a/packages/oraidex-sync/src/poolHelper.ts b/packages/oraidex-sync/src/poolHelper.ts index 366669d1..7311ed2f 100644 --- a/packages/oraidex-sync/src/poolHelper.ts +++ b/packages/oraidex-sync/src/poolHelper.ts @@ -1,17 +1,28 @@ import { MulticallQueryClient } from "@oraichain/common-contracts-sdk"; -import { Asset, AssetInfo, OraiswapPairQueryClient } from "@oraichain/oraidex-contracts-sdk"; +import { + Asset, + AssetInfo, + OraiswapPairQueryClient, + OraiswapStakingTypes, + OraiswapTokenTypes, + PairInfo +} from "@oraichain/oraidex-contracts-sdk"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; -import { ORAI, oraiInfo, usdtCw20Address, usdtInfo } from "./constants"; +import { ORAI, ORAIXOCH_INFO, SEC_PER_YEAR, atomic, network, oraiInfo, usdtInfo } from "./constants"; import { calculatePriceByPool, fetchPoolInfoAmount, getCosmwasmClient, getPairInfoFromAssets, - parseAssetInfoOnlyDenom + parseAssetInfoOnlyDenom, + validateNumber } from "./helper"; import { pairs } from "./pairs"; import { queryPoolInfos } from "./query"; -import { PairInfoData, PairMapping } from "./types"; +import { PairInfoData, PairMapping, PoolInfo } from "./types"; +import { isEqual } from "lodash"; +import { fromBinary, toBinary } from "@cosmjs/cosmwasm-stargate"; +import { TokenInfoResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapToken.types"; // use this type to determine the ratio of price of base to the quote or vice versa export type RatioDirection = "base_in_quote" | "quote_in_base"; @@ -141,6 +152,120 @@ async function calculateLiquidityFee(pair: PairInfoData, txHeight: number, withd return BigInt(Math.round(feeByUsdt)); } +// ==== calculate APR ==== +export const calculateAprResult = async ( + pairs: PairInfo[], + allLiquidities: number[], + allTokenInfo: TokenInfoResponse[], + allLpTokenAsset: OraiswapStakingTypes.PoolInfoResponse[], + allRewardPerSec: OraiswapStakingTypes.RewardsPerSecResponse[] +): Promise => { + let aprResult = []; + let ind = 0; + for (const pair of pairs) { + console.time(`apr/${parseAssetInfoOnlyDenom(pair.asset_infos[0])}-${parseAssetInfoOnlyDenom(pair.asset_infos[1])}`); + const liquidityAmount = allLiquidities[ind] * Math.pow(10, -6); + const lpToken = allLpTokenAsset[ind]; + const tokenSupply = allTokenInfo[ind]; + const rewardsPerSecData = allRewardPerSec[ind]; + if (!lpToken || !tokenSupply || !rewardsPerSecData) continue; + + const bondValue = + (validateNumber(lpToken.total_bond_amount) * liquidityAmount) / validateNumber(tokenSupply.total_supply); + + let rewardsPerYearValue = 0; + for (const { amount, info } of rewardsPerSecData.assets) { + // NOTE: current hardcode price token xOCH: $0.4 + const priceAssetInUsdt = isEqual(info, ORAIXOCH_INFO) ? 0.4 : await getPriceAssetByUsdt(info); + rewardsPerYearValue += (SEC_PER_YEAR * validateNumber(amount) * priceAssetInUsdt) / atomic; + } + aprResult[ind] = (100 * rewardsPerYearValue) / bondValue || 0; + ind++; + console.timeEnd( + `apr/${parseAssetInfoOnlyDenom(pair.asset_infos[0])}-${parseAssetInfoOnlyDenom(pair.asset_infos[1])}` + ); + } + return aprResult; +}; + +async function fetchTokenInfos(pairInfos: PairInfo[]): Promise { + const queries = pairInfos.map((pair) => ({ + address: pair.liquidity_token, + data: toBinary({ + token_info: {} + } as OraiswapTokenTypes.QueryMsg) + })); + const client = await getCosmwasmClient(); + const multicall = new MulticallQueryClient(client, network.multicall); + const res = await multicall.aggregate({ + queries + }); + return pairInfos.map((item, ind) => { + const info: TokenInfoResponse = fromBinary(res.return_data[ind++].data); + return info; + }); +} + +async function fetchAllTokenAssetPools(assetInfos: AssetInfo[]): Promise { + const queries = assetInfos.map((assetInfo) => { + return { + address: network.staking, + data: toBinary({ + pool_info: { + asset_info: assetInfo + } + } as OraiswapStakingTypes.QueryMsg) + }; + }); + + const client = await getCosmwasmClient(); + const multicall = new MulticallQueryClient(client, network.multicall); + const res = await multicall.tryAggregate({ + queries + }); + return res.return_data.map((data) => (data.success ? fromBinary(data.data) : undefined)); +} + +async function fetchAllRewardPerSecInfos( + assetInfos: AssetInfo[] +): Promise { + const queries = assetInfos.map((assetInfo) => { + return { + address: network.staking, + data: toBinary({ + rewards_per_sec: { + asset_info: assetInfo + } + } as OraiswapStakingTypes.QueryMsg) + }; + }); + const client = await getCosmwasmClient(); + const multicall = new MulticallQueryClient(client, network.multicall); + const res = await multicall.tryAggregate({ + queries + }); + return res.return_data.map((data) => (data.success ? fromBinary(data.data) : undefined)); +} + +function getStakingAssetInfo(assetInfos: AssetInfo[]): AssetInfo { + return parseAssetInfoOnlyDenom(assetInfos[0]) === ORAI ? assetInfos[1] : assetInfos[0]; +} + +// Fetch APR +const fetchAprResult = async (pairInfos: PairInfo[], allLiquidities: number[]): Promise => { + const assetTokens = pairs.map((pair) => getStakingAssetInfo(pair.asset_infos)); + try { + const [allTokenInfo, allLpTokenAsset, allRewardPerSec] = await Promise.all([ + fetchTokenInfos(pairInfos), + fetchAllTokenAssetPools(assetTokens), + fetchAllRewardPerSecInfos(assetTokens) + ]); + return calculateAprResult(pairInfos, allLiquidities, allTokenInfo, allLpTokenAsset, allRewardPerSec); + } catch (error) { + console.log({ errorFetchAprResult: error }); + } +}; + export { calculateFeeByAsset, calculateLiquidityFee, @@ -149,5 +274,6 @@ export { getPoolInfos, getPriceAssetByUsdt, getPriceByAsset, - isPoolHasFee + isPoolHasFee, + fetchAprResult }; diff --git a/packages/oraidex-sync/src/types.ts b/packages/oraidex-sync/src/types.ts index b61c2905..5b710a59 100644 --- a/packages/oraidex-sync/src/types.ts +++ b/packages/oraidex-sync/src/types.ts @@ -206,3 +206,8 @@ export type GetFeeSwap = { }; export type GetVolumeQuery = Omit; + +export type PoolInfo = { + offerPoolAmount: bigint; + askPoolAmount: bigint; +}; From 446b02ce7eb0eb88bc9e8c9be6614af79ff9d347 Mon Sep 17 00:00:00 2001 From: trungbach Date: Fri, 25 Aug 2023 16:56:12 +0700 Subject: [PATCH 11/46] refactor: use network config instead process.env --- packages/oraidex-sync/src/helper.ts | 9 +++------ packages/oraidex-sync/src/poolHelper.ts | 5 +---- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index 51e456f0..f05df8b8 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -10,7 +10,7 @@ import { } from "@oraichain/oraidex-contracts-sdk"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; import { isEqual, maxBy, minBy } from "lodash"; -import { ORAI, atomic, tenAmountInDecimalSix, truncDecimals, usdtCw20Address } from "./constants"; +import { ORAI, atomic, network, tenAmountInDecimalSix, truncDecimals, usdtCw20Address } from "./constants"; import { pairs, pairsOnlyDenom } from "./pairs"; import { simulateSwapPriceWithUsdt } from "./query"; import { @@ -424,10 +424,7 @@ async function fetchPoolInfoAmount(fromInfo: AssetInfo, toInfo: AssetInfo, pairA async function getPairLiquidity([fromInfo, toInfo]: [AssetInfo, AssetInfo], pairAddr: string): Promise { const { offerPoolAmount, askPoolAmount } = await fetchPoolInfoAmount(fromInfo, toInfo, pairAddr); - const routerContract = new OraiswapRouterQueryClient( - await getCosmwasmClient(), - process.env.ROUTER_CONTRACT_ADDRESS || "orai1j0r67r9k8t34pnhy00x3ftuxuwg0r6r4p8p6rrc8az0ednzr8y9s3sj2sf" - ); + const routerContract = new OraiswapRouterQueryClient(await getCosmwasmClient(), network.router); const { amount } = await simulateSwapPriceWithUsdt(fromInfo, routerContract); const totalLiquid = Number(amount) * Number(offerPoolAmount) * 2; return totalLiquid; @@ -455,7 +452,7 @@ async function getPairInfoFromAssets( const pair = getPairByAssetInfos(assetInfos); const factoryClient = new OraiswapFactoryQueryClient( await getCosmwasmClient(), - pair.factoryV1 ? process.env.FACTORY_CONTACT_ADDRESS_V1 : process.env.FACTORY_CONTACT_ADDRESS_V2 + pair.factoryV1 ? network.factory : network.factory_v2 ); const pairInfo = await factoryClient.pair({ assetInfos }); return { diff --git a/packages/oraidex-sync/src/poolHelper.ts b/packages/oraidex-sync/src/poolHelper.ts index 7311ed2f..bf9ebb09 100644 --- a/packages/oraidex-sync/src/poolHelper.ts +++ b/packages/oraidex-sync/src/poolHelper.ts @@ -49,10 +49,7 @@ async function getPoolInfos(pairAddrs: string[], wantedHeight?: number): Promise // adjust the query height to get data from the past const cosmwasmClient = await getCosmwasmClient(); cosmwasmClient.setQueryClientWithHeight(wantedHeight); - const multicall = new MulticallQueryClient( - cosmwasmClient, - process.env.MULTICALL_CONTRACT_ADDRESS || "orai1q7x644gmf7h8u8y6y8t9z9nnwl8djkmspypr6mxavsk9ual7dj0sxpmgwd" - ); + const multicall = new MulticallQueryClient(cosmwasmClient, network.multicall); const res = await queryPoolInfos(pairAddrs, multicall); return res; } From d1aa45401f6367a8f0a78f70c87b3a2783060a75 Mon Sep 17 00:00:00 2001 From: trungbach Date: Sun, 27 Aug 2023 15:12:57 +0700 Subject: [PATCH 12/46] refactor: restructure helper function & fixed error in collectAccumulateLpData --- packages/oraidex-sync/src/helper.ts | 20 +-- packages/oraidex-sync/src/index.ts | 7 +- packages/oraidex-sync/src/poolHelper.ts | 88 ++------------ packages/oraidex-sync/src/query.ts | 74 +++++++++++- packages/oraidex-sync/src/tx-parsing.ts | 6 +- .../oraidex-sync/tests/pool-helper.spec.ts | 114 +++--------------- 6 files changed, 115 insertions(+), 194 deletions(-) diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index f05df8b8..c793f8e9 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -247,29 +247,33 @@ export function collectAccumulateLpData( info.assets.some((assetInfo) => parseAssetInfoOnlyDenom(assetInfo.info) === op.quoteTokenDenom) ); if (!pool) continue; + + let baseAmount = BigInt(op.baseTokenAmount); + let quoteAmount = BigInt(op.quoteTokenAmount); if (op.opType === "withdraw") { // reverse sign since withdraw means lp decreases - op.baseTokenReserve = -BigInt(op.baseTokenReserve); - op.quoteTokenReserve = -BigInt(op.quoteTokenReserve); + baseAmount = -BigInt(op.baseTokenAmount); + quoteAmount = -BigInt(op.quoteTokenAmount); + console.log({ op }); } const denom = `${op.baseTokenDenom}-${op.quoteTokenDenom}`; if (!accumulateData[denom]) { const initialFirstTokenAmount = parseInt( - pool.assets.find((info) => parseAssetInfoOnlyDenom(info.info) === op.baseTokenDenom).amount + pool.assets.find((asset) => parseAssetInfoOnlyDenom(asset.info) === op.baseTokenDenom).amount ); const initialSecondTokenAmount = parseInt( - pool.assets.find((info) => parseAssetInfoOnlyDenom(info.info) === op.quoteTokenDenom).amount + pool.assets.find((asset) => parseAssetInfoOnlyDenom(asset.info) === op.quoteTokenDenom).amount ); accumulateData[denom] = { - baseTokenAmount: BigInt(initialFirstTokenAmount) + BigInt(op.baseTokenReserve), - quoteTokenAmount: BigInt(initialSecondTokenAmount) + BigInt(op.quoteTokenReserve) + baseTokenAmount: BigInt(initialFirstTokenAmount) + baseAmount, + quoteTokenAmount: BigInt(initialSecondTokenAmount) + quoteAmount }; op.baseTokenReserve = accumulateData[denom].baseTokenAmount; op.quoteTokenReserve = accumulateData[denom].quoteTokenAmount; continue; } - accumulateData[denom].baseTokenAmount += BigInt(op.baseTokenReserve); - accumulateData[denom].quoteTokenAmount += BigInt(op.quoteTokenReserve); + accumulateData[denom].baseTokenAmount += baseAmount; + accumulateData[denom].quoteTokenAmount += quoteAmount; op.baseTokenReserve = accumulateData[denom].baseTokenAmount; op.quoteTokenReserve = accumulateData[denom].quoteTokenAmount; } diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index eaee94d0..4143ec8d 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -208,11 +208,6 @@ class OraiDexSync { return allVolumes; } - async getAllApr(pairInfos: PairInfo[], allLiquidities: number[]): Promise { - const allApr = await fetchAprResult(pairInfos, allLiquidities); - return allApr; - } - private async updateLatestPairInfos() { try { console.time("timer-updateLatestPairInfos"); @@ -224,7 +219,7 @@ class OraiDexSync { ); const allFee7Days = await this.getAllFees(); const allVolume24h = await this.getAllVolume24h(); - const allAPr = await this.getAllApr(pairInfos, allLiquidities); + const allAPr = await fetchAprResult(pairInfos, allLiquidities); await this.duckDb.insertPairInfos( pairInfos.map((pair, index) => { diff --git a/packages/oraidex-sync/src/poolHelper.ts b/packages/oraidex-sync/src/poolHelper.ts index bf9ebb09..0a0e2866 100644 --- a/packages/oraidex-sync/src/poolHelper.ts +++ b/packages/oraidex-sync/src/poolHelper.ts @@ -4,10 +4,11 @@ import { AssetInfo, OraiswapPairQueryClient, OraiswapStakingTypes, - OraiswapTokenTypes, PairInfo } from "@oraichain/oraidex-contracts-sdk"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; +import { TokenInfoResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapToken.types"; +import { isEqual } from "lodash"; import { ORAI, ORAIXOCH_INFO, SEC_PER_YEAR, atomic, network, oraiInfo, usdtInfo } from "./constants"; import { calculatePriceByPool, @@ -18,11 +19,8 @@ import { validateNumber } from "./helper"; import { pairs } from "./pairs"; -import { queryPoolInfos } from "./query"; -import { PairInfoData, PairMapping, PoolInfo } from "./types"; -import { isEqual } from "lodash"; -import { fromBinary, toBinary } from "@cosmjs/cosmwasm-stargate"; -import { TokenInfoResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapToken.types"; +import { fetchAllRewardPerSecInfos, fetchAllTokenAssetPools, fetchTokenInfos, queryPoolInfos } from "./query"; +import { PairInfoData, PairMapping } from "./types"; // use this type to determine the ratio of price of base to the quote or vice versa export type RatioDirection = "base_in_quote" | "quote_in_base"; @@ -86,17 +84,14 @@ async function getPriceByAsset(assetInfos: [AssetInfo, AssetInfo], ratioDirectio // then, calculate price of this asset token in USDT based on price ORAI in USDT. async function getPriceAssetByUsdt(asset: AssetInfo): Promise { if (parseAssetInfoOnlyDenom(asset) === parseAssetInfoOnlyDenom(usdtInfo)) return 1; + if (parseAssetInfoOnlyDenom(asset) === parseAssetInfoOnlyDenom(oraiInfo)) return await getOraiPrice(); - let priceInOrai = 1; - if (parseAssetInfoOnlyDenom(asset) !== parseAssetInfoOnlyDenom(oraiInfo)) { - const foundPair = getPairByAssetInfos([asset, oraiInfo]); - if (!foundPair) return 0; - - const ratioDirection: RatioDirection = - parseAssetInfoOnlyDenom(foundPair.asset_infos[0]) === ORAI ? "quote_in_base" : "base_in_quote"; - priceInOrai = await getPriceByAsset(foundPair.asset_infos, ratioDirection); - } + const foundPair = getPairByAssetInfos([asset, oraiInfo]); + if (!foundPair) return 0; + const ratioDirection: RatioDirection = + parseAssetInfoOnlyDenom(foundPair.asset_infos[0]) === ORAI ? "quote_in_base" : "base_in_quote"; + const priceInOrai = await getPriceByAsset(foundPair.asset_infos, ratioDirection); const priceOraiInUsdt = await getOraiPrice(); return priceInOrai * priceOraiInUsdt; @@ -185,65 +180,6 @@ export const calculateAprResult = async ( return aprResult; }; -async function fetchTokenInfos(pairInfos: PairInfo[]): Promise { - const queries = pairInfos.map((pair) => ({ - address: pair.liquidity_token, - data: toBinary({ - token_info: {} - } as OraiswapTokenTypes.QueryMsg) - })); - const client = await getCosmwasmClient(); - const multicall = new MulticallQueryClient(client, network.multicall); - const res = await multicall.aggregate({ - queries - }); - return pairInfos.map((item, ind) => { - const info: TokenInfoResponse = fromBinary(res.return_data[ind++].data); - return info; - }); -} - -async function fetchAllTokenAssetPools(assetInfos: AssetInfo[]): Promise { - const queries = assetInfos.map((assetInfo) => { - return { - address: network.staking, - data: toBinary({ - pool_info: { - asset_info: assetInfo - } - } as OraiswapStakingTypes.QueryMsg) - }; - }); - - const client = await getCosmwasmClient(); - const multicall = new MulticallQueryClient(client, network.multicall); - const res = await multicall.tryAggregate({ - queries - }); - return res.return_data.map((data) => (data.success ? fromBinary(data.data) : undefined)); -} - -async function fetchAllRewardPerSecInfos( - assetInfos: AssetInfo[] -): Promise { - const queries = assetInfos.map((assetInfo) => { - return { - address: network.staking, - data: toBinary({ - rewards_per_sec: { - asset_info: assetInfo - } - } as OraiswapStakingTypes.QueryMsg) - }; - }); - const client = await getCosmwasmClient(); - const multicall = new MulticallQueryClient(client, network.multicall); - const res = await multicall.tryAggregate({ - queries - }); - return res.return_data.map((data) => (data.success ? fromBinary(data.data) : undefined)); -} - function getStakingAssetInfo(assetInfos: AssetInfo[]): AssetInfo { return parseAssetInfoOnlyDenom(assetInfos[0]) === ORAI ? assetInfos[1] : assetInfos[0]; } @@ -266,11 +202,11 @@ const fetchAprResult = async (pairInfos: PairInfo[], allLiquidities: number[]): export { calculateFeeByAsset, calculateLiquidityFee, + fetchAprResult, getOraiPrice, getPairByAssetInfos, getPoolInfos, getPriceAssetByUsdt, getPriceByAsset, - isPoolHasFee, - fetchAprResult + isPoolHasFee }; diff --git a/packages/oraidex-sync/src/query.ts b/packages/oraidex-sync/src/query.ts index caa45e3b..d7e01f63 100644 --- a/packages/oraidex-sync/src/query.ts +++ b/packages/oraidex-sync/src/query.ts @@ -1,15 +1,24 @@ import { OraiswapFactoryReadOnlyInterface, OraiswapRouterReadOnlyInterface, + OraiswapStakingTypes, + OraiswapTokenTypes, PairInfo } from "@oraichain/oraidex-contracts-sdk"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; import { Asset, AssetInfo } from "@oraichain/oraidex-contracts-sdk"; -import { MulticallReadOnlyInterface } from "@oraichain/common-contracts-sdk"; +import { Call, MulticallQueryClient, MulticallReadOnlyInterface } from "@oraichain/common-contracts-sdk"; import { fromBinary, toBinary } from "@cosmjs/cosmwasm-stargate"; import { pairs } from "./pairs"; -import { findAssetInfoPathToUsdt, generateSwapOperations, parseAssetInfoOnlyDenom, toDisplay } from "./helper"; -import { tenAmountInDecimalSix, usdtCw20Address } from "./constants"; +import { + findAssetInfoPathToUsdt, + generateSwapOperations, + getCosmwasmClient, + parseAssetInfoOnlyDenom, + toDisplay +} from "./helper"; +import { network, tenAmountInDecimalSix, usdtCw20Address } from "./constants"; +import { TokenInfoResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapToken.types"; async function queryPoolInfos(pairAddrs: string[], multicall: MulticallReadOnlyInterface): Promise { // adjust the query height to get data from the past @@ -75,4 +84,61 @@ async function simulateSwapPrice(pairPath: AssetInfo[], router: OraiswapRouterRe } } -export { getAllPairInfos, queryPoolInfos, simulateSwapPriceWithUsdt, simulateSwapPrice }; +async function aggregateMulticall(queries: Call[]) { + const client = await getCosmwasmClient(); + const multicall = new MulticallQueryClient(client, network.multicall); + const res = await multicall.aggregate({ queries }); + return res.return_data.map((data) => (data.success ? fromBinary(data.data) : undefined)); +} + +async function fetchTokenInfos(pairInfos: PairInfo[]): Promise { + const queries = pairInfos.map((pair) => ({ + address: pair.liquidity_token, + data: toBinary({ + token_info: {} + } as OraiswapTokenTypes.QueryMsg) + })); + return await aggregateMulticall(queries); +} + +async function fetchAllTokenAssetPools(assetInfos: AssetInfo[]): Promise { + const queries = assetInfos.map((assetInfo) => { + return { + address: network.staking, + data: toBinary({ + pool_info: { + asset_info: assetInfo + } + } as OraiswapStakingTypes.QueryMsg) + }; + }); + + return await aggregateMulticall(queries); +} + +async function fetchAllRewardPerSecInfos( + assetInfos: AssetInfo[] +): Promise { + const queries = assetInfos.map((assetInfo) => { + return { + address: network.staking, + data: toBinary({ + rewards_per_sec: { + asset_info: assetInfo + } + } as OraiswapStakingTypes.QueryMsg) + }; + }); + return await aggregateMulticall(queries); +} + +export { + getAllPairInfos, + queryPoolInfos, + simulateSwapPriceWithUsdt, + simulateSwapPrice, + aggregateMulticall, + fetchTokenInfos, + fetchAllTokenAssetPools, + fetchAllRewardPerSecInfos +}; diff --git a/packages/oraidex-sync/src/tx-parsing.ts b/packages/oraidex-sync/src/tx-parsing.ts index b6a2a11a..a76f755e 100644 --- a/packages/oraidex-sync/src/tx-parsing.ts +++ b/packages/oraidex-sync/src/tx-parsing.ts @@ -249,8 +249,8 @@ async function extractMsgWithdrawLiquidity( ) ); if (findedPair) { - baseAsset = assets[3]; - quoteAsset = assets[1]; + [baseAsset, quoteAsset] = [quoteAsset, baseAsset]; + [baseAssetAmount, quoteAssetAmount] = [quoteAssetAmount, baseAssetAmount]; } if (assets.length !== 4) continue; @@ -259,7 +259,7 @@ async function extractMsgWithdrawLiquidity( withdrawData.push({ basePrice: calculatePriceByPool(BigInt(baseAssetAmount), BigInt(quoteAssetAmount)), baseTokenAmount: baseAssetAmount, - baseTokenDenom: assets[1], + baseTokenDenom: baseAsset, baseTokenReserve: baseAssetAmount, opType: "withdraw", uniqueKey: concatDataToUniqueKey({ diff --git a/packages/oraidex-sync/tests/pool-helper.spec.ts b/packages/oraidex-sync/tests/pool-helper.spec.ts index 8f3e3235..3df87a19 100644 --- a/packages/oraidex-sync/tests/pool-helper.spec.ts +++ b/packages/oraidex-sync/tests/pool-helper.spec.ts @@ -1,22 +1,22 @@ -import { Asset, AssetInfo, PairInfo } from "@oraichain/oraidex-contracts-sdk"; +import { Asset, AssetInfo } from "@oraichain/oraidex-contracts-sdk"; import { ORAI, airiCw20Adress, atomIbcDenom, milkyCw20Address, + oraiInfo, scAtomCw20Address, - usdtCw20Address + usdtCw20Address, + usdtInfo } from "../src/constants"; import { PairMapping } from "../src/types"; import * as helper from "../src/helper"; import * as poolHelper from "../src/poolHelper"; -import { mock } from "node:test"; + describe("test-pool-helper", () => { afterEach(() => { jest.clearAllMocks(); - // jest.resetAllMocks(); - // jest.restoreAllMocks(); }); it.each<[string, [AssetInfo, AssetInfo], boolean]>([ @@ -101,20 +101,9 @@ describe("test-pool-helper", () => { it.each<[string, [AssetInfo, AssetInfo], PairMapping | undefined]>([ [ "assetInfos-valid-in-list-pairs", - [ - { - token: { - contract_addr: usdtCw20Address - } - }, - { - native_token: { - denom: ORAI - } - } - ], + [usdtInfo, oraiInfo], { - asset_infos: [{ native_token: { denom: ORAI } }, { token: { contract_addr: usdtCw20Address } }], + asset_infos: [oraiInfo, usdtInfo], symbols: ["ORAI", "USDT"], factoryV1: true } @@ -143,72 +132,11 @@ describe("test-pool-helper", () => { } ); - // it.each<[[AssetInfo, AssetInfo], Pick]>([ - // [ - // [ - // { - // native_token: { - // denom: ORAI - // } - // }, - // { - // token: { - // contract_addr: usdtCw20Address - // } - // } - // ], - // { - // contract_addr: "orai123", - // commission_rate: "0.003" - // } - // ] - // ])("test-getPairInfoFromAssets-should-return-correctly-pair-info", async (assetInfos, expectedPairInfo) => { - // const getPairInfoFromAssetsSpy = jest.spyOn(helper, "getPairInfoFromAssets"); - // const mockPairInfoFromAssets = { - // commission_rate: "0.003", - // contract_addr: "orai1234" - // }; - // getPairInfoFromAssetsSpy.mockResolvedValue(mockPairInfoFromAssets); - // const result = await helper.getPairInfoFromAssets(assetInfos); - // console.log({ result }); - // // expect(result).toStrictEqual(expectedPairInfo); - // }); - describe("test-calculate-price-group-funcs", () => { afterEach(jest.clearAllMocks); it.each<[[AssetInfo, AssetInfo], poolHelper.RatioDirection, number]>([ - [ - [ - { - native_token: { - denom: ORAI - } - }, - { - token: { - contract_addr: usdtCw20Address - } - } - ], - "base_in_quote", - 0.5 - ], - [ - [ - { - native_token: { - denom: ORAI - } - }, - { - token: { - contract_addr: usdtCw20Address - } - } - ], - "quote_in_base", - 2 - ] + [[oraiInfo, usdtInfo], "base_in_quote", 0.5], + [[oraiInfo, usdtInfo], "quote_in_base", 2] ])("test-getPriceByAsset-should-return-correctly-price", async (assetInfos, ratioDirection, expectedPrice) => { // Mock the return values for fetchPoolInfoAmount and calculatePriceByPool const mockAskPoolAmount = 1000n; @@ -238,15 +166,10 @@ describe("test-pool-helper", () => { }); it.each([ + ["asset-is-cw20-USDT", usdtInfo, 1], + ["asset-is-ORAI", oraiInfo, 2], [ - { - native_token: { - denom: "not-pair-with-orai" - } - }, - 0 - ], - [ + "asset-is-not-pair-with-ORAI", { native_token: { denom: atomIbcDenom @@ -255,16 +178,13 @@ describe("test-pool-helper", () => { 1 ] ])( - "test-getPriceAssetByUsdt-should-return-price-of-asset-in-USDT", - async (assetInfo: AssetInfo, expectedPrice: number) => { + "test-getPriceAssetByUsdt-with-%p-should-return-price-of-asset-in-USDT", + async (_caseName: string, assetInfo: AssetInfo, expectedPrice: number) => { const mockPriceByAssetValue = 4; - const mockOraiPrice = 2; - - const getPriceByAssetSpy = jest.spyOn(poolHelper, "getPriceByAsset"); - getPriceByAssetSpy.mockResolvedValue(mockPriceByAssetValue); + jest.spyOn(poolHelper, "getPriceByAsset").mockResolvedValue(mockPriceByAssetValue); - const getOraiPriceSpy = jest.spyOn(poolHelper, "getOraiPrice"); - getOraiPriceSpy.mockResolvedValue(mockOraiPrice); + const mockOraiPrice = 2; + jest.spyOn(poolHelper, "getOraiPrice").mockResolvedValue(mockOraiPrice); const result = await poolHelper.getPriceAssetByUsdt(assetInfo); expect(result).toEqual(expectedPrice); From a12a00ef4f310bfaed8127822cf9d8f6a1bebd91 Mon Sep 17 00:00:00 2001 From: trungbach Date: Sun, 27 Aug 2023 16:30:56 +0700 Subject: [PATCH 13/46] refactor: move util function calculate fee pair to helper file --- packages/oraidex-server/src/index.ts | 15 +++- packages/oraidex-sync/src/helper.ts | 128 ++++++++++++++++++++++++++- packages/oraidex-sync/src/index.ts | 126 ++------------------------ 3 files changed, 144 insertions(+), 125 deletions(-) diff --git a/packages/oraidex-server/src/index.ts b/packages/oraidex-server/src/index.ts index 9667d781..af8513dc 100644 --- a/packages/oraidex-server/src/index.ts +++ b/packages/oraidex-server/src/index.ts @@ -14,7 +14,8 @@ import { pairsOnlyDenom, VolumeRange, oraiUsdtPairOnlyDenom, - ORAI + ORAI, + getAllVolume24h } from "@oraichain/oraidex-sync"; import cors from "cors"; import { CosmWasmClient } from "@cosmjs/cosmwasm-stargate"; @@ -215,7 +216,7 @@ app.get("/volume/v2/historical/chart", async (req, res) => { // console.log("prefix sum: ", prefixSum); // res.status(200).send("hello world"); // } catch (error) { -// console.log("server error /liquidity/v2/historical/chart: ", error); +// console.log("server error /liquidity/v2/historical/chart: ", error); // res.status(500).send(JSON.stringify(error)); // } finally { // return; @@ -233,8 +234,16 @@ app.get("/v1/candles/", async (req: Request<{}, {}, {}, GetCandlesQuery>, res) = app.get("/v1/pools/", async (req, res) => { try { + const volumes = await getAllVolume24h(duckDb); const pools = await duckDb.getPools(); - res.status(200).send(pools); + res.status(200).send( + pools.map((pool, index) => { + return { + ...pool, + volume24Hour: volumes[index].toString() + }; + }) + ); } catch (error) { res.status(500).send(error.message); } diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index c793f8e9..5b6d77d5 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -11,7 +11,9 @@ import { import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; import { isEqual, maxBy, minBy } from "lodash"; import { ORAI, atomic, network, tenAmountInDecimalSix, truncDecimals, usdtCw20Address } from "./constants"; +import { DuckDb } from "./db"; import { pairs, pairsOnlyDenom } from "./pairs"; +import { getPairByAssetInfos, getPriceAssetByUsdt, getPriceByAsset } from "./poolHelper"; import { simulateSwapPriceWithUsdt } from "./query"; import { Ohlcv, @@ -23,7 +25,6 @@ import { SwapOperationData, WithdrawLiquidityOperationData } from "./types"; -import { getPairByAssetInfos } from "./poolHelper"; export function toObject(data: any) { return JSON.parse( @@ -465,6 +466,121 @@ async function getPairInfoFromAssets( }; } +// ====== get volume pairs ====== +async function getVolumeSwap( + [baseAssetInfo, quoteAssetInfo]: [AssetInfo, AssetInfo], + startTime: Date, + endTime: Date, + duckDb: DuckDb +): Promise { + const pair = `${parseAssetInfoOnlyDenom(baseAssetInfo)}-${parseAssetInfoOnlyDenom(quoteAssetInfo)}`; + const volumePairInBaseAsset = await duckDb.getVolumeSwap({ + pair, + startTime: convertDateToSecond(startTime), + endTime: convertDateToSecond(endTime) + }); + let priceBaseAssetInUsdt = await getPriceAssetByUsdt(baseAssetInfo); + + // it means this asset not pair with ORAI + // in our pairs, if base asset not pair with ORAI, surely quote asset will pair with ORAI + if (priceBaseAssetInUsdt === 0) { + const priceQuoteAssetInUsdt = await getPriceAssetByUsdt(quoteAssetInfo); + const priceBaseInQuote = await getPriceByAsset([baseAssetInfo, quoteAssetInfo], "base_in_quote"); + priceBaseAssetInUsdt = priceBaseInQuote * priceQuoteAssetInUsdt; + } + + const volumeInUsdt = priceBaseAssetInUsdt * Number(volumePairInBaseAsset); + return BigInt(Math.round(volumeInUsdt)); +} + +async function getVolumeLiquidity( + [baseAssetInfo, quoteAssetInfo]: [AssetInfo, AssetInfo], + startTime: Date, + endTime: Date, + duckDb: DuckDb +): Promise { + const volumePairInBaseAsset = await duckDb.getVolumeLiquidity({ + offerDenom: parseAssetInfoOnlyDenom(baseAssetInfo), + askDenom: parseAssetInfoOnlyDenom(quoteAssetInfo), + startTime: convertDateToSecond(startTime), + endTime: convertDateToSecond(endTime) + }); + let priceBaseAssetInUsdt = await getPriceAssetByUsdt(baseAssetInfo); + + // it means this asset not pair with ORAI + // in our pairs, if base asset not pair with ORAI, surely quote asset will pair with ORAI + if (priceBaseAssetInUsdt === 0) { + const priceQuoteAssetInUsdt = await getPriceAssetByUsdt(quoteAssetInfo); + const priceBaseInQuote = await getPriceByAsset([baseAssetInfo, quoteAssetInfo], "base_in_quote"); + priceBaseAssetInUsdt = priceBaseInQuote * priceQuoteAssetInUsdt; + } + const volumeInUsdt = priceBaseAssetInUsdt * Number(volumePairInBaseAsset); + console.log({ + volumeInUsdt, + priceBaseAssetInUsdt, + volumePairInBaseAsset, + pair: `${parseAssetInfoOnlyDenom(baseAssetInfo)}-${parseAssetInfoOnlyDenom(quoteAssetInfo)}` + }); + return BigInt(Math.round(volumeInUsdt)); +} + +async function getVolumePair( + [baseAssetInfo, quoteAssetInfo]: [AssetInfo, AssetInfo], + startTime: Date, + endTime: Date, + duckDb: DuckDb +): Promise { + const [volumeSwap, volumeLiquidity] = await Promise.all([ + getVolumeSwap([baseAssetInfo, quoteAssetInfo], startTime, endTime, duckDb), + getVolumeLiquidity([baseAssetInfo, quoteAssetInfo], startTime, endTime, duckDb) + ]); + return volumeSwap + volumeLiquidity; +} + +async function getAllVolume24h(duckDb: DuckDb): Promise { + const tf = 100 * 24 * 60 * 60; // second of 24h * 100 + const currentDate = new Date(); + const oneDayBeforeNow = getSpecificDateBeforeNow(new Date(), tf); + const allVolumes = await Promise.all( + pairs.map((pair) => getVolumePair(pair.asset_infos, oneDayBeforeNow, currentDate, duckDb)) + ); + return allVolumes; +} + +// ==== get fee pair ==== +async function getFeePair( + asset_infos: [AssetInfo, AssetInfo], + startTime: Date, + endTime: Date, + duckDb: DuckDb +): Promise { + const [swapFee, liquidityFee] = await Promise.all([ + duckDb.getFeeSwap({ + offerDenom: parseAssetInfoOnlyDenom(asset_infos[0]), + askDenom: parseAssetInfoOnlyDenom(asset_infos[1]), + startTime: convertDateToSecond(startTime), + endTime: convertDateToSecond(endTime) + }), + duckDb.getFeeLiquidity({ + offerDenom: parseAssetInfoOnlyDenom(asset_infos[0]), + askDenom: parseAssetInfoOnlyDenom(asset_infos[1]), + startTime: convertDateToSecond(startTime), + endTime: convertDateToSecond(endTime) + }) + ]); + return swapFee + liquidityFee; +} + +async function getAllFees(duckDb: DuckDb): Promise { + const tf = 7 * 24 * 60 * 60; // second of 7 days + const currentDate = new Date(); + const oneWeekBeforeNow = getSpecificDateBeforeNow(new Date(), tf); + const allFees = await Promise.all( + pairs.map((pair) => getFeePair(pair.asset_infos, oneWeekBeforeNow, currentDate, duckDb)) + ); + return allFees; +} + export { calculatePriceByPool, convertDateToSecond, @@ -474,11 +590,17 @@ export { findMappedTargetedAssetInfo, findPairAddress, generateSwapOperations, + getAllFees, + getAllVolume24h, getCosmwasmClient, + getFeePair, + getPairInfoFromAssets, getPairLiquidity, getSpecificDateBeforeNow, getSymbolFromAsset, + getVolumeLiquidity, + getVolumePair, + getVolumeSwap, parseAssetInfo, - parseAssetInfoOnlyDenom, - getPairInfoFromAssets + parseAssetInfoOnlyDenom }; diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index 4143ec8d..fc9d1706 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -1,16 +1,9 @@ import { SyncData, Txs, WriteData } from "@oraichain/cosmos-rpc-sync"; -import { AssetInfo, CosmWasmClient, OraiswapFactoryQueryClient, PairInfo } from "@oraichain/oraidex-contracts-sdk"; +import { CosmWasmClient, OraiswapFactoryQueryClient, PairInfo } from "@oraichain/oraidex-contracts-sdk"; import "dotenv/config"; import { DuckDb } from "./db"; -import { - collectAccumulateLpData, - convertDateToSecond, - getPairLiquidity, - getSpecificDateBeforeNow, - getSymbolFromAsset, - parseAssetInfoOnlyDenom -} from "./helper"; -import { fetchAprResult, getPoolInfos, getPriceAssetByUsdt, getPriceByAsset } from "./poolHelper"; +import { collectAccumulateLpData, getAllFees, getAllVolume24h, getPairLiquidity, getSymbolFromAsset } from "./helper"; +import { fetchAprResult, getPoolInfos } from "./poolHelper"; import { getAllPairInfos } from "./query"; import { parseAssetInfo, parseTxs } from "./tx-parsing"; import { @@ -21,7 +14,6 @@ import { TxAnlysisResult, WithdrawLiquidityOperationData } from "./types"; -import { pairs } from "./pairs"; class WriteOrders extends WriteData { private firstWrite: boolean; @@ -104,110 +96,6 @@ class OraiDexSync { return getAllPairInfos(firstFactoryClient, secondFactoryClient); } - async getFeePair(asset_infos: [AssetInfo, AssetInfo], startTime: Date, endTime: Date): Promise { - const [swapFee, liquidityFee] = await Promise.all([ - this.duckDb.getFeeSwap({ - offerDenom: parseAssetInfoOnlyDenom(asset_infos[0]), - askDenom: parseAssetInfoOnlyDenom(asset_infos[1]), - startTime: convertDateToSecond(startTime), - endTime: convertDateToSecond(endTime) - }), - this.duckDb.getFeeLiquidity({ - offerDenom: parseAssetInfoOnlyDenom(asset_infos[0]), - askDenom: parseAssetInfoOnlyDenom(asset_infos[1]), - startTime: convertDateToSecond(startTime), - endTime: convertDateToSecond(endTime) - }) - ]); - return swapFee + liquidityFee; - } - - async getVolumeSwap( - [baseAssetInfo, quoteAssetInfo]: [AssetInfo, AssetInfo], - startTime: Date, - endTime: Date - ): Promise { - const pair = `${parseAssetInfoOnlyDenom(baseAssetInfo)}-${parseAssetInfoOnlyDenom(quoteAssetInfo)}`; - const volumePairInBaseAsset = await this.duckDb.getVolumeSwap({ - pair, - startTime: convertDateToSecond(startTime), - endTime: convertDateToSecond(endTime) - }); - let priceBaseAssetInUsdt = await getPriceAssetByUsdt(baseAssetInfo); - - // it means this asset not pair with ORAI - // in our pairs, if base asset not pair with ORAI, surely quote asset will pair with ORAI - if (priceBaseAssetInUsdt === 0) { - const priceQuoteAssetInUsdt = await getPriceAssetByUsdt(quoteAssetInfo); - const priceBaseInQuote = await getPriceByAsset([baseAssetInfo, quoteAssetInfo], "base_in_quote"); - priceBaseAssetInUsdt = priceBaseInQuote * priceQuoteAssetInUsdt; - } - const volumeInUsdt = priceBaseAssetInUsdt * Number(volumePairInBaseAsset); - return BigInt(Math.round(volumeInUsdt)); - } - - async getVolumeLiquidity( - [baseAssetInfo, quoteAssetInfo]: [AssetInfo, AssetInfo], - startTime: Date, - endTime: Date - ): Promise { - const volumePairInBaseAsset = await this.duckDb.getVolumeLiquidity({ - offerDenom: parseAssetInfoOnlyDenom(baseAssetInfo), - askDenom: parseAssetInfoOnlyDenom(quoteAssetInfo), - startTime: convertDateToSecond(startTime), - endTime: convertDateToSecond(endTime) - }); - let priceBaseAssetInUsdt = await getPriceAssetByUsdt(baseAssetInfo); - - // it means this asset not pair with ORAI - // in our pairs, if base asset not pair with ORAI, surely quote asset will pair with ORAI - if (priceBaseAssetInUsdt === 0) { - const priceQuoteAssetInUsdt = await getPriceAssetByUsdt(quoteAssetInfo); - const priceBaseInQuote = await getPriceByAsset([baseAssetInfo, quoteAssetInfo], "base_in_quote"); - priceBaseAssetInUsdt = priceBaseInQuote * priceQuoteAssetInUsdt; - } - const volumeInUsdt = priceBaseAssetInUsdt * Number(volumePairInBaseAsset); - console.log({ - volumeInUsdt, - priceBaseAssetInUsdt, - volumePairInBaseAsset, - pair: `${parseAssetInfoOnlyDenom(baseAssetInfo)}-${parseAssetInfoOnlyDenom(quoteAssetInfo)}` - }); - return BigInt(Math.round(volumeInUsdt)); - } - - async getVolumePair( - [baseAssetInfo, quoteAssetInfo]: [AssetInfo, AssetInfo], - startTime: Date, - endTime: Date - ): Promise { - const [volumeSwap, volumeLiquidity] = await Promise.all([ - this.getVolumeSwap([baseAssetInfo, quoteAssetInfo], startTime, endTime), - this.getVolumeLiquidity([baseAssetInfo, quoteAssetInfo], startTime, endTime) - ]); - return volumeSwap + volumeLiquidity; - } - - async getAllFees(): Promise { - const tf = 7 * 24 * 60 * 60; // second of 7 days - const currentDate = new Date(); - const oneWeekBeforeNow = getSpecificDateBeforeNow(new Date(), tf); - const allFees = await Promise.all( - pairs.map((pair) => this.getFeePair(pair.asset_infos, oneWeekBeforeNow, currentDate)) - ); - return allFees; - } - - async getAllVolume24h(): Promise { - const tf = 100 * 24 * 60 * 60; // second of 24h * 100 - const currentDate = new Date(); - const oneDayBeforeNow = getSpecificDateBeforeNow(new Date(), tf); - const allVolumes = await Promise.all( - pairs.map((pair) => this.getVolumePair(pair.asset_infos, oneDayBeforeNow, currentDate)) - ); - return allVolumes; - } - private async updateLatestPairInfos() { try { console.time("timer-updateLatestPairInfos"); @@ -217,8 +105,8 @@ class OraiDexSync { return getPairLiquidity(pair.asset_infos, pair.contract_addr); }) ); - const allFee7Days = await this.getAllFees(); - const allVolume24h = await this.getAllVolume24h(); + const allFee7Days = await getAllFees(this.duckDb); + const allVolume24h = await getAllVolume24h(this.duckDb); const allAPr = await fetchAprResult(pairInfos, allLiquidities); await this.duckDb.insertPairInfos( @@ -285,13 +173,13 @@ async function initSync() { oraidexSync.sync(); } -initSync(); +// initSync(); export { OraiDexSync }; export * from "./constants"; export * from "./db"; export * from "./helper"; export * from "./pairs"; +export * from "./poolHelper"; export * from "./query"; export * from "./types"; -export * from "./poolHelper"; From dd02dbf240f8aa4ccecae4f170b83860eb408598 Mon Sep 17 00:00:00 2001 From: trungbach Date: Sun, 27 Aug 2023 20:18:06 +0700 Subject: [PATCH 14/46] test: add testcase for helper & update api get pools --- packages/oraidex-server/src/index.ts | 24 +++++++-- packages/oraidex-sync/src/helper.ts | 4 +- packages/oraidex-sync/src/index.ts | 39 ++++---------- packages/oraidex-sync/src/poolHelper.ts | 42 +++++++++++++-- packages/oraidex-sync/src/query.ts | 4 +- packages/oraidex-sync/tests/helper.spec.ts | 20 ++++++-- .../oraidex-sync/tests/pool-helper.spec.ts | 51 +++++++++++++++++++ 7 files changed, 139 insertions(+), 45 deletions(-) diff --git a/packages/oraidex-server/src/index.ts b/packages/oraidex-server/src/index.ts index af8513dc..364638b2 100644 --- a/packages/oraidex-server/src/index.ts +++ b/packages/oraidex-server/src/index.ts @@ -15,7 +15,11 @@ import { VolumeRange, oraiUsdtPairOnlyDenom, ORAI, - getAllVolume24h + getAllVolume24h, + getAllFees, + getAllPairInfos, + getPairLiquidity, + fetchAprResult } from "@oraichain/oraidex-sync"; import cors from "cors"; import { CosmWasmClient } from "@cosmjs/cosmwasm-stargate"; @@ -234,13 +238,27 @@ app.get("/v1/candles/", async (req: Request<{}, {}, {}, GetCandlesQuery>, res) = app.get("/v1/pools/", async (req, res) => { try { - const volumes = await getAllVolume24h(duckDb); + const [volumes, allFee7Days, pairInfos] = await Promise.all([ + getAllVolume24h(duckDb), + getAllFees(duckDb), + getAllPairInfos() + ]); + + const allLiquidities = await Promise.all( + pairInfos.map((pair) => { + return getPairLiquidity(pair.asset_infos, pair.contract_addr); + }) + ); + const allApr = await fetchAprResult(pairInfos, allLiquidities); + const pools = await duckDb.getPools(); res.status(200).send( pools.map((pool, index) => { return { ...pool, - volume24Hour: volumes[index].toString() + volume24Hour: volumes[index].toString(), + fee7Days: allFee7Days[index].toString(), + apr: allApr[index] }; }) ); diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index 5b6d77d5..197700d6 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -221,8 +221,8 @@ export function roundTime(timeIn: number, timeframe: number): number { } export function isAssetInfoPairReverse(assetInfos: AssetInfo[]): boolean { - if (pairs.find((pair) => JSON.stringify(pair.asset_infos) === JSON.stringify(assetInfos.reverse()))) return true; - return false; + if (pairs.find((pair) => JSON.stringify(pair.asset_infos) === JSON.stringify(assetInfos))) return false; + return true; } /** diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index fc9d1706..d2fe968b 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -1,10 +1,9 @@ import { SyncData, Txs, WriteData } from "@oraichain/cosmos-rpc-sync"; -import { CosmWasmClient, OraiswapFactoryQueryClient, PairInfo } from "@oraichain/oraidex-contracts-sdk"; +import { CosmWasmClient } from "@oraichain/oraidex-contracts-sdk"; import "dotenv/config"; import { DuckDb } from "./db"; -import { collectAccumulateLpData, getAllFees, getAllVolume24h, getPairLiquidity, getSymbolFromAsset } from "./helper"; -import { fetchAprResult, getPoolInfos } from "./poolHelper"; -import { getAllPairInfos } from "./query"; +import { collectAccumulateLpData, getSymbolFromAsset } from "./helper"; +import { getAllPairInfos, getPoolInfos } from "./poolHelper"; import { parseAssetInfo, parseTxs } from "./tx-parsing"; import { Env, @@ -84,30 +83,10 @@ class OraiDexSync { return new OraiDexSync(duckDb, rpcUrl, cosmwasmClient, env); } - private async getAllPairInfos(): Promise { - const firstFactoryClient = new OraiswapFactoryQueryClient( - this.cosmwasmClient, - this.env.FACTORY_CONTACT_ADDRESS_V1 || "orai1hemdkz4xx9kukgrunxu3yw0nvpyxf34v82d2c8" - ); - const secondFactoryClient = new OraiswapFactoryQueryClient( - this.cosmwasmClient, - this.env.FACTORY_CONTACT_ADDRESS_V2 || "orai167r4ut7avvgpp3rlzksz6vw5spmykluzagvmj3ht845fjschwugqjsqhst" - ); - return getAllPairInfos(firstFactoryClient, secondFactoryClient); - } - private async updateLatestPairInfos() { try { console.time("timer-updateLatestPairInfos"); - const pairInfos = await this.getAllPairInfos(); - const allLiquidities = await Promise.all( - pairInfos.map((pair) => { - return getPairLiquidity(pair.asset_infos, pair.contract_addr); - }) - ); - const allFee7Days = await getAllFees(this.duckDb); - const allVolume24h = await getAllVolume24h(this.duckDb); - const allAPr = await fetchAprResult(pairInfos, allLiquidities); + const pairInfos = await getAllPairInfos(); await this.duckDb.insertPairInfos( pairInfos.map((pair, index) => { @@ -122,10 +101,10 @@ class OraiDexSync { symbols, fromIconUrl: "url1", toIconUrl: "url2", - volume24Hour: allVolume24h[index], - apr: allAPr[index], - totalLiquidity: allLiquidities[index], - fee7Days: allFee7Days[index] + volume24Hour: 0n, + apr: 0, + totalLiquidity: 0, + fee7Days: 0n } as PairInfoData; }) ); @@ -173,7 +152,7 @@ async function initSync() { oraidexSync.sync(); } -// initSync(); +initSync(); export { OraiDexSync }; export * from "./constants"; diff --git a/packages/oraidex-sync/src/poolHelper.ts b/packages/oraidex-sync/src/poolHelper.ts index 0a0e2866..4b69c7dd 100644 --- a/packages/oraidex-sync/src/poolHelper.ts +++ b/packages/oraidex-sync/src/poolHelper.ts @@ -2,6 +2,7 @@ import { MulticallQueryClient } from "@oraichain/common-contracts-sdk"; import { Asset, AssetInfo, + OraiswapFactoryQueryClient, OraiswapPairQueryClient, OraiswapStakingTypes, PairInfo @@ -15,11 +16,18 @@ import { fetchPoolInfoAmount, getCosmwasmClient, getPairInfoFromAssets, + isAssetInfoPairReverse, parseAssetInfoOnlyDenom, validateNumber } from "./helper"; import { pairs } from "./pairs"; -import { fetchAllRewardPerSecInfos, fetchAllTokenAssetPools, fetchTokenInfos, queryPoolInfos } from "./query"; +import { + fetchAllRewardPerSecInfos, + fetchAllTokenAssetPools, + fetchTokenInfos, + queryAllPairInfos, + queryPoolInfos +} from "./query"; import { PairInfoData, PairMapping } from "./types"; // use this type to determine the ratio of price of base to the quote or vice versa @@ -93,7 +101,6 @@ async function getPriceAssetByUsdt(asset: AssetInfo): Promise { parseAssetInfoOnlyDenom(foundPair.asset_infos[0]) === ORAI ? "quote_in_base" : "base_in_quote"; const priceInOrai = await getPriceByAsset(foundPair.asset_infos, ratioDirection); const priceOraiInUsdt = await getOraiPrice(); - return priceInOrai * priceOraiInUsdt; } @@ -172,7 +179,20 @@ export const calculateAprResult = async ( rewardsPerYearValue += (SEC_PER_YEAR * validateNumber(amount) * priceAssetInUsdt) / atomic; } aprResult[ind] = (100 * rewardsPerYearValue) / bondValue || 0; - ind++; + ind += 1; + console.dir( + { + aprResult, + bondValue, + rewardsPerYearValue, + rewardsPerSecData, + lpToken, + tokenSupply, + liquidityAmount, + pair: `${parseAssetInfoOnlyDenom(pair.asset_infos[0])}-${parseAssetInfoOnlyDenom(pair.asset_infos[1])}` + }, + { depth: null } + ); console.timeEnd( `apr/${parseAssetInfoOnlyDenom(pair.asset_infos[0])}-${parseAssetInfoOnlyDenom(pair.asset_infos[1])}` ); @@ -181,12 +201,15 @@ export const calculateAprResult = async ( }; function getStakingAssetInfo(assetInfos: AssetInfo[]): AssetInfo { + if (isAssetInfoPairReverse(assetInfos)) { + assetInfos = assetInfos.reverse(); + } return parseAssetInfoOnlyDenom(assetInfos[0]) === ORAI ? assetInfos[1] : assetInfos[0]; } // Fetch APR const fetchAprResult = async (pairInfos: PairInfo[], allLiquidities: number[]): Promise => { - const assetTokens = pairs.map((pair) => getStakingAssetInfo(pair.asset_infos)); + const assetTokens = pairInfos.map((pair) => getStakingAssetInfo(JSON.parse(JSON.stringify(pair.asset_infos)))); try { const [allTokenInfo, allLpTokenAsset, allRewardPerSec] = await Promise.all([ fetchTokenInfos(pairInfos), @@ -199,14 +222,23 @@ const fetchAprResult = async (pairInfos: PairInfo[], allLiquidities: number[]): } }; +async function getAllPairInfos(): Promise { + const cosmwasmClient = await getCosmwasmClient(); + const firstFactoryClient = new OraiswapFactoryQueryClient(cosmwasmClient, network.factory); + const secondFactoryClient = new OraiswapFactoryQueryClient(cosmwasmClient, network.factory_v2); + return queryAllPairInfos(firstFactoryClient, secondFactoryClient); +} + export { calculateFeeByAsset, calculateLiquidityFee, fetchAprResult, + getAllPairInfos, getOraiPrice, getPairByAssetInfos, getPoolInfos, getPriceAssetByUsdt, getPriceByAsset, - isPoolHasFee + isPoolHasFee, + getStakingAssetInfo }; diff --git a/packages/oraidex-sync/src/query.ts b/packages/oraidex-sync/src/query.ts index d7e01f63..30f85cbf 100644 --- a/packages/oraidex-sync/src/query.ts +++ b/packages/oraidex-sync/src/query.ts @@ -36,7 +36,7 @@ async function queryPoolInfos(pairAddrs: string[], multicall: MulticallReadOnlyI return res.return_data.map((data) => (data.success ? fromBinary(data.data) : undefined)).filter((data) => data); // remove undefined items } -async function getAllPairInfos( +async function queryAllPairInfos( factoryV1: OraiswapFactoryReadOnlyInterface, factoryV2: OraiswapFactoryReadOnlyInterface ): Promise { @@ -133,7 +133,7 @@ async function fetchAllRewardPerSecInfos( } export { - getAllPairInfos, + queryAllPairInfos, queryPoolInfos, simulateSwapPriceWithUsdt, simulateSwapPrice, diff --git a/packages/oraidex-sync/tests/helper.spec.ts b/packages/oraidex-sync/tests/helper.spec.ts index c128fce1..e1e2135d 100644 --- a/packages/oraidex-sync/tests/helper.spec.ts +++ b/packages/oraidex-sync/tests/helper.spec.ts @@ -1,4 +1,4 @@ -import { AssetInfo } from "@oraichain/oraidex-contracts-sdk"; +import { Asset, AssetInfo } from "@oraichain/oraidex-contracts-sdk"; import { findAssetInfoPathToUsdt, findMappedTargetedAssetInfo, @@ -16,7 +16,8 @@ import { getSwapDirection, findPairIndexFromDenoms, toObject, - calculateSwapOhlcv + calculateSwapOhlcv, + isAssetInfoPairReverse } from "../src/helper"; import { extractUniqueAndFlatten, pairs } from "../src/pairs"; import { @@ -25,13 +26,15 @@ import { atomIbcDenom, kwtCw20Address, milkyCw20Address, + oraiInfo, oraixCw20Address, osmosisIbcDenom, scAtomCw20Address, scOraiCw20Address, tronCw20Address, usdcCw20Address, - usdtCw20Address + usdtCw20Address, + usdtInfo } from "../src/constants"; import { PairInfoData, ProvideLiquidityOperationData, SwapDirection, SwapOperationData } from "../src/types"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; @@ -635,4 +638,15 @@ describe("test-helper", () => { expect(result).toEqual(expectedIndex); } ); + + it.each([ + ["case-asset-info-pairs-is-NOT-reversed", [oraiInfo, usdtInfo], false], + ["case-asset-info-pairs-is-reversed", [usdtInfo, oraiInfo], true] + ])( + "test-isAssetInfoPairReverse-should-return-correctly", + (_caseName: string, assetInfos: AssetInfo[], expectedResult: boolean) => { + const result = isAssetInfoPairReverse(assetInfos); + expect(result).toBe(expectedResult); + } + ); }); diff --git a/packages/oraidex-sync/tests/pool-helper.spec.ts b/packages/oraidex-sync/tests/pool-helper.spec.ts index 3df87a19..9223858d 100644 --- a/packages/oraidex-sync/tests/pool-helper.spec.ts +++ b/packages/oraidex-sync/tests/pool-helper.spec.ts @@ -191,4 +191,55 @@ describe("test-pool-helper", () => { } ); }); + + it.each<[string, AssetInfo[], AssetInfo]>([ + [ + "case-asset-info-pairs-is-NOT-reversed-and-base-asset-NOT-ORAI", + [ + { + token: { + contract_addr: scAtomCw20Address + } + }, + { + native_token: { + denom: atomIbcDenom + } + } + ], + { + token: { + contract_addr: scAtomCw20Address + } + } + ], + ["case-asset-info-pairs-is-NOT-reversed-and-base-asset-is-ORAI", [oraiInfo, usdtInfo], usdtInfo], + [ + "case-asset-info-pairs-is-reversed-and-base-asset-NOT-ORAI", + [ + { + native_token: { + denom: atomIbcDenom + } + }, + { + token: { + contract_addr: scAtomCw20Address + } + } + ], + { + token: { + contract_addr: scAtomCw20Address + } + } + ], + ["case-asset-info-pairs-is-reversed-and-base-asset-is-ORAI", [usdtInfo, oraiInfo], usdtInfo] + ])( + "test-getStakingAssetInfo-with-%p-should-return-correctly-staking-asset-info", + (_caseName: string, assetInfos: AssetInfo[], expectedStakingAssetInfo: AssetInfo) => { + const result = poolHelper.getStakingAssetInfo(assetInfos); + expect(result).toStrictEqual(expectedStakingAssetInfo); + } + ); }); From 0b8d56c9673bd29f66e6c22d5b761cf092927ce2 Mon Sep 17 00:00:00 2001 From: trungbach Date: Mon, 28 Aug 2023 17:29:07 +0700 Subject: [PATCH 15/46] update: calculted price asset via pool amount --- packages/oraidex-server/src/index.ts | 19 +++++++---- packages/oraidex-sync/src/db.ts | 38 +++++++++++++++++++--- packages/oraidex-sync/src/helper.ts | 30 ++++++++--------- packages/oraidex-sync/src/index.ts | 7 ++++ packages/oraidex-sync/src/poolHelper.ts | 43 ++++++++++++------------- 5 files changed, 89 insertions(+), 48 deletions(-) diff --git a/packages/oraidex-server/src/index.ts b/packages/oraidex-server/src/index.ts index 364638b2..a85915b2 100644 --- a/packages/oraidex-server/src/index.ts +++ b/packages/oraidex-server/src/index.ts @@ -238,27 +238,35 @@ app.get("/v1/candles/", async (req: Request<{}, {}, {}, GetCandlesQuery>, res) = app.get("/v1/pools/", async (req, res) => { try { + console.time("get util"); const [volumes, allFee7Days, pairInfos] = await Promise.all([ getAllVolume24h(duckDb), getAllFees(duckDb), getAllPairInfos() ]); + console.timeEnd("get util"); + + console.time("allLiquidities"); const allLiquidities = await Promise.all( pairInfos.map((pair) => { - return getPairLiquidity(pair.asset_infos, pair.contract_addr); + return getPairLiquidity(pair.asset_infos, duckDb); }) ); - const allApr = await fetchAprResult(pairInfos, allLiquidities); + console.timeEnd("allLiquidities"); + console.time("getApr"); + const allApr = await fetchAprResult(pairInfos, allLiquidities); + console.timeEnd("getApr"); const pools = await duckDb.getPools(); res.status(200).send( pools.map((pool, index) => { return { ...pool, - volume24Hour: volumes[index].toString(), - fee7Days: allFee7Days[index].toString(), - apr: allApr[index] + volume24Hour: volumes[index]?.toString() ?? "0", + fee7Days: allFee7Days[index]?.toString() ?? "0", + apr: allApr[index], + totalLiquidity: allLiquidities[index] }; }) ); @@ -269,7 +277,6 @@ app.get("/v1/pools/", async (req, res) => { app.listen(port, hostname, async () => { // sync data for the service to read - // console.dir(pairInfos, { depth: null }); duckDb = await DuckDb.create(process.env.DUCKDB_PROD_FILENAME || "oraidex-sync-data"); const oraidexSync = await OraiDexSync.create( duckDb, diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index 1d037353..e95e48fc 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -11,10 +11,18 @@ import { WithdrawLiquidityOperationData, GetCandlesQuery, GetFeeSwap, - GetVolumeQuery + GetVolumeQuery, + PoolInfo } from "./types"; import fs, { rename } from "fs"; -import { isoToTimestampNumber, parseAssetInfo, renameKey, replaceAllNonAlphaBetChar, toObject } from "./helper"; +import { + isoToTimestampNumber, + parseAssetInfo, + parseAssetInfoOnlyDenom, + renameKey, + replaceAllNonAlphaBetChar, + toObject +} from "./helper"; import { AssetInfo } from "@oraichain/oraidex-contracts-sdk"; export class DuckDb { @@ -415,7 +423,7 @@ export class DuckDb { offerDenom ) ]); - return BigInt(feeRightDirection[0].totalFee ?? 0 + feeReverseDirection[0].totalFee ?? 0); + return BigInt(feeRightDirection[0]?.totalFee ?? 0 + feeReverseDirection[0]?.totalFee ?? 0); } async getFeeLiquidity(payload: GetFeeSwap): Promise { @@ -435,7 +443,7 @@ export class DuckDb { offerDenom, askDenom ); - return BigInt(result[0].totalFee ?? 0); + return BigInt(result[0]?.totalFee ?? 0); } async getVolumeSwap(payload: GetVolumeQuery): Promise { @@ -473,6 +481,26 @@ export class DuckDb { offerDenom, askDenom ); - return BigInt(result[0].totalVolume ?? 0); + return BigInt(result[0]?.totalVolume ?? 0); + } + + async getPoolAmountFromAssetInfos(assetInfos: [AssetInfo, AssetInfo]): Promise { + const baseTokenDenom = parseAssetInfoOnlyDenom(assetInfos[0]); + const quoteTokenDenom = parseAssetInfoOnlyDenom(assetInfos[1]); + const result = await this.conn.all( + `SELECT * from lp_ops_data WHERE baseTokenDenom = ? AND quoteTokenDenom = ? ORDER BY txheight LIMIT 1`, + baseTokenDenom, + quoteTokenDenom + ); + + if (result.length === 0) { + console.dir({ nullForPair: assetInfos }, { depth: null }); + return null; + } + + return { + offerPoolAmount: BigInt(result[0].baseTokenReserve), + askPoolAmount: BigInt(result[0].quoteTokenReserve) + }; } } diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index 197700d6..8259145d 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -4,7 +4,6 @@ import { OraiswapFactoryQueryClient, OraiswapPairQueryClient, OraiswapPairTypes, - OraiswapRouterQueryClient, PairInfo, SwapOperation } from "@oraichain/oraidex-contracts-sdk"; @@ -14,7 +13,6 @@ import { ORAI, atomic, network, tenAmountInDecimalSix, truncDecimals, usdtCw20Ad import { DuckDb } from "./db"; import { pairs, pairsOnlyDenom } from "./pairs"; import { getPairByAssetInfos, getPriceAssetByUsdt, getPriceByAsset } from "./poolHelper"; -import { simulateSwapPriceWithUsdt } from "./query"; import { Ohlcv, OraiDexType, @@ -255,7 +253,6 @@ export function collectAccumulateLpData( // reverse sign since withdraw means lp decreases baseAmount = -BigInt(op.baseTokenAmount); quoteAmount = -BigInt(op.quoteTokenAmount); - console.log({ op }); } const denom = `${op.baseTokenDenom}-${op.quoteTokenDenom}`; if (!accumulateData[denom]) { @@ -426,13 +423,22 @@ async function fetchPoolInfoAmount(fromInfo: AssetInfo, toInfo: AssetInfo, pairA return { offerPoolAmount, askPoolAmount }; } -async function getPairLiquidity([fromInfo, toInfo]: [AssetInfo, AssetInfo], pairAddr: string): Promise { - const { offerPoolAmount, askPoolAmount } = await fetchPoolInfoAmount(fromInfo, toInfo, pairAddr); +async function getPairLiquidity(rawAssetInfos: [AssetInfo, AssetInfo], duckDb: DuckDb): Promise { + let assetInfos: [AssetInfo, AssetInfo] = JSON.parse(JSON.stringify(rawAssetInfos)); + if (isAssetInfoPairReverse(assetInfos)) { + assetInfos = assetInfos.reverse() as [AssetInfo, AssetInfo]; + } + const poolInfo = await duckDb.getPoolByAssetInfos(assetInfos); + if (!poolInfo) throw new Error(`Cannot found pool info when get pair liquidity: ${JSON.stringify(assetInfos)}`); + + // get info of last tx in lp_ops_data, if not have data => get info from contract + let poolAmounts = + (await duckDb.getPoolAmountFromAssetInfos(assetInfos)) ?? + (await fetchPoolInfoAmount(...assetInfos, poolInfo.pairAddr)); + if (!poolAmounts) throw new Error(` Cannot found pool amount: ${JSON.stringify(assetInfos)}`); - const routerContract = new OraiswapRouterQueryClient(await getCosmwasmClient(), network.router); - const { amount } = await simulateSwapPriceWithUsdt(fromInfo, routerContract); - const totalLiquid = Number(amount) * Number(offerPoolAmount) * 2; - return totalLiquid; + const priceBaseAssetInUsdt = await getPriceAssetByUsdt(assetInfos[0]); + return priceBaseAssetInUsdt * Number(poolAmounts.offerPoolAmount) * 2; } /** @@ -515,12 +521,6 @@ async function getVolumeLiquidity( priceBaseAssetInUsdt = priceBaseInQuote * priceQuoteAssetInUsdt; } const volumeInUsdt = priceBaseAssetInUsdt * Number(volumePairInBaseAsset); - console.log({ - volumeInUsdt, - priceBaseAssetInUsdt, - volumePairInBaseAsset, - pair: `${parseAssetInfoOnlyDenom(baseAssetInfo)}-${parseAssetInfoOnlyDenom(quoteAssetInfo)}` - }); return BigInt(Math.round(volumeInUsdt)); } diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index d2fe968b..a58fdd19 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -71,6 +71,7 @@ class WriteOrders extends WriteData { } class OraiDexSync { + public static duckDbInstance: DuckDb | null = null; protected constructor( private readonly duckDb: DuckDb, private readonly rpcUrl: string, @@ -80,9 +81,14 @@ class OraiDexSync { public static async create(duckDb: DuckDb, rpcUrl: string, env: Env): Promise { const cosmwasmClient = await CosmWasmClient.connect(rpcUrl); + OraiDexSync.duckDbInstance = duckDb; // Store the provided duckDb instance return new OraiDexSync(duckDb, rpcUrl, cosmwasmClient, env); } + public static getDuckDbInstance() { + return OraiDexSync.duckDbInstance; + } + private async updateLatestPairInfos() { try { console.time("timer-updateLatestPairInfos"); @@ -153,6 +159,7 @@ async function initSync() { } initSync(); + export { OraiDexSync }; export * from "./constants"; diff --git a/packages/oraidex-sync/src/poolHelper.ts b/packages/oraidex-sync/src/poolHelper.ts index 4b69c7dd..65503759 100644 --- a/packages/oraidex-sync/src/poolHelper.ts +++ b/packages/oraidex-sync/src/poolHelper.ts @@ -29,6 +29,7 @@ import { queryPoolInfos } from "./query"; import { PairInfoData, PairMapping } from "./types"; +import { OraiDexSync, DuckDb } from "./index"; // use this type to determine the ratio of price of base to the quote or vice versa export type RatioDirection = "base_in_quote" | "quote_in_base"; @@ -78,14 +79,25 @@ async function getOraiPrice(): Promise { return getPriceByAsset([oraiInfo, usdtInfo], ratioDirection); } -// get pair of assets then query info from contract to calculate price asset. async function getPriceByAsset(assetInfos: [AssetInfo, AssetInfo], ratioDirection: RatioDirection): Promise { - const pairInfo = await getPairInfoFromAssets(assetInfos); + const duckDb: DuckDb = OraiDexSync.getDuckDbInstance(); + const poolInfo = await duckDb.getPoolByAssetInfos(assetInfos); + if (!poolInfo) throw new Error(`Cannot found pool info: ${JSON.stringify(assetInfos)}`); + + // get info of last tx in lp_ops_data, if not have data => get info from contract + let poolAmounts = + (await duckDb.getPoolAmountFromAssetInfos(assetInfos)) ?? + (await fetchPoolInfoAmount(...assetInfos, poolInfo.pairAddr)); + if (!poolAmounts) throw new Error(` Cannot found pool amount: ${JSON.stringify(assetInfos)}`); + // offer: orai, ask: usdt -> price offer in ask = calculatePriceByPool([ask, offer]) // offer: orai, ask: atom -> price ask in offer = calculatePriceByPool([offer, ask]) - const { offerPoolAmount, askPoolAmount } = await fetchPoolInfoAmount(...assetInfos, pairInfo.contract_addr); - const assetPrice = calculatePriceByPool(askPoolAmount, offerPoolAmount, +pairInfo.commission_rate); - return ratioDirection === "base_in_quote" ? assetPrice : 1 / assetPrice; + const basePrice = calculatePriceByPool( + BigInt(poolAmounts.askPoolAmount), + BigInt(poolAmounts.offerPoolAmount), + +poolInfo.commissionRate + ); + return ratioDirection === "base_in_quote" ? basePrice : 1 / basePrice; } // find pool match this asset with orai => calculate price this asset token in ORAI. @@ -101,6 +113,10 @@ async function getPriceAssetByUsdt(asset: AssetInfo): Promise { parseAssetInfoOnlyDenom(foundPair.asset_infos[0]) === ORAI ? "quote_in_base" : "base_in_quote"; const priceInOrai = await getPriceByAsset(foundPair.asset_infos, ratioDirection); const priceOraiInUsdt = await getOraiPrice(); + // console.dir( + // { price: priceInOrai * priceOraiInUsdt, priceOraiInUsdt, asset: parseAssetInfoOnlyDenom(asset) }, + // { depth: null } + // ); return priceInOrai * priceOraiInUsdt; } @@ -162,7 +178,6 @@ export const calculateAprResult = async ( let aprResult = []; let ind = 0; for (const pair of pairs) { - console.time(`apr/${parseAssetInfoOnlyDenom(pair.asset_infos[0])}-${parseAssetInfoOnlyDenom(pair.asset_infos[1])}`); const liquidityAmount = allLiquidities[ind] * Math.pow(10, -6); const lpToken = allLpTokenAsset[ind]; const tokenSupply = allTokenInfo[ind]; @@ -180,22 +195,6 @@ export const calculateAprResult = async ( } aprResult[ind] = (100 * rewardsPerYearValue) / bondValue || 0; ind += 1; - console.dir( - { - aprResult, - bondValue, - rewardsPerYearValue, - rewardsPerSecData, - lpToken, - tokenSupply, - liquidityAmount, - pair: `${parseAssetInfoOnlyDenom(pair.asset_infos[0])}-${parseAssetInfoOnlyDenom(pair.asset_infos[1])}` - }, - { depth: null } - ); - console.timeEnd( - `apr/${parseAssetInfoOnlyDenom(pair.asset_infos[0])}-${parseAssetInfoOnlyDenom(pair.asset_infos[1])}` - ); } return aprResult; }; From 03217b310b5a10e45efe780418f33775a134a1f2 Mon Sep 17 00:00:00 2001 From: trungbach Date: Mon, 28 Aug 2023 18:08:08 +0700 Subject: [PATCH 16/46] api: finished api get pools info --- packages/oraidex-server/src/index.ts | 12 ++--- packages/oraidex-sync/src/helper.ts | 70 ++++++++----------------- packages/oraidex-sync/src/poolHelper.ts | 8 +-- packages/oraidex-sync/src/query.ts | 5 +- 4 files changed, 32 insertions(+), 63 deletions(-) diff --git a/packages/oraidex-server/src/index.ts b/packages/oraidex-server/src/index.ts index a85915b2..116145dd 100644 --- a/packages/oraidex-server/src/index.ts +++ b/packages/oraidex-server/src/index.ts @@ -239,26 +239,22 @@ app.get("/v1/candles/", async (req: Request<{}, {}, {}, GetCandlesQuery>, res) = app.get("/v1/pools/", async (req, res) => { try { console.time("get util"); - const [volumes, allFee7Days, pairInfos] = await Promise.all([ - getAllVolume24h(duckDb), - getAllFees(duckDb), - getAllPairInfos() - ]); + const [volumes, allFee7Days] = await Promise.all([getAllVolume24h(duckDb), getAllFees(duckDb)]); console.timeEnd("get util"); console.time("allLiquidities"); const allLiquidities = await Promise.all( - pairInfos.map((pair) => { + pairs.map((pair) => { return getPairLiquidity(pair.asset_infos, duckDb); }) ); console.timeEnd("allLiquidities"); + const pools = await duckDb.getPools(); console.time("getApr"); - const allApr = await fetchAprResult(pairInfos, allLiquidities); + const allApr = await fetchAprResult(pools, allLiquidities); console.timeEnd("getApr"); - const pools = await duckDb.getPools(); res.status(200).send( pools.map((pool, index) => { return { diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index 8259145d..123c93db 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -473,44 +473,26 @@ async function getPairInfoFromAssets( } // ====== get volume pairs ====== -async function getVolumeSwap( +async function getVolumePair( [baseAssetInfo, quoteAssetInfo]: [AssetInfo, AssetInfo], startTime: Date, endTime: Date, duckDb: DuckDb ): Promise { const pair = `${parseAssetInfoOnlyDenom(baseAssetInfo)}-${parseAssetInfoOnlyDenom(quoteAssetInfo)}`; - const volumePairInBaseAsset = await duckDb.getVolumeSwap({ - pair, - startTime: convertDateToSecond(startTime), - endTime: convertDateToSecond(endTime) - }); - let priceBaseAssetInUsdt = await getPriceAssetByUsdt(baseAssetInfo); - - // it means this asset not pair with ORAI - // in our pairs, if base asset not pair with ORAI, surely quote asset will pair with ORAI - if (priceBaseAssetInUsdt === 0) { - const priceQuoteAssetInUsdt = await getPriceAssetByUsdt(quoteAssetInfo); - const priceBaseInQuote = await getPriceByAsset([baseAssetInfo, quoteAssetInfo], "base_in_quote"); - priceBaseAssetInUsdt = priceBaseInQuote * priceQuoteAssetInUsdt; - } - - const volumeInUsdt = priceBaseAssetInUsdt * Number(volumePairInBaseAsset); - return BigInt(Math.round(volumeInUsdt)); -} - -async function getVolumeLiquidity( - [baseAssetInfo, quoteAssetInfo]: [AssetInfo, AssetInfo], - startTime: Date, - endTime: Date, - duckDb: DuckDb -): Promise { - const volumePairInBaseAsset = await duckDb.getVolumeLiquidity({ - offerDenom: parseAssetInfoOnlyDenom(baseAssetInfo), - askDenom: parseAssetInfoOnlyDenom(quoteAssetInfo), - startTime: convertDateToSecond(startTime), - endTime: convertDateToSecond(endTime) - }); + const [volumeSwapPairInBaseAsset, volumeLiquidityPairInBaseAsset] = await Promise.all([ + duckDb.getVolumeSwap({ + pair, + startTime: convertDateToSecond(startTime), + endTime: convertDateToSecond(endTime) + }), + duckDb.getVolumeLiquidity({ + offerDenom: parseAssetInfoOnlyDenom(baseAssetInfo), + askDenom: parseAssetInfoOnlyDenom(quoteAssetInfo), + startTime: convertDateToSecond(startTime), + endTime: convertDateToSecond(endTime) + }) + ]); let priceBaseAssetInUsdt = await getPriceAssetByUsdt(baseAssetInfo); // it means this asset not pair with ORAI @@ -520,30 +502,20 @@ async function getVolumeLiquidity( const priceBaseInQuote = await getPriceByAsset([baseAssetInfo, quoteAssetInfo], "base_in_quote"); priceBaseAssetInUsdt = priceBaseInQuote * priceQuoteAssetInUsdt; } - const volumeInUsdt = priceBaseAssetInUsdt * Number(volumePairInBaseAsset); + const volumeInUsdt = + priceBaseAssetInUsdt * (Number(volumeSwapPairInBaseAsset) + Number(volumeLiquidityPairInBaseAsset)); return BigInt(Math.round(volumeInUsdt)); } -async function getVolumePair( - [baseAssetInfo, quoteAssetInfo]: [AssetInfo, AssetInfo], - startTime: Date, - endTime: Date, - duckDb: DuckDb -): Promise { - const [volumeSwap, volumeLiquidity] = await Promise.all([ - getVolumeSwap([baseAssetInfo, quoteAssetInfo], startTime, endTime, duckDb), - getVolumeLiquidity([baseAssetInfo, quoteAssetInfo], startTime, endTime, duckDb) - ]); - return volumeSwap + volumeLiquidity; -} - async function getAllVolume24h(duckDb: DuckDb): Promise { - const tf = 100 * 24 * 60 * 60; // second of 24h * 100 + console.time("getAllVolume24h"); + const tf = 24 * 60 * 60; // second of 24h const currentDate = new Date(); const oneDayBeforeNow = getSpecificDateBeforeNow(new Date(), tf); const allVolumes = await Promise.all( pairs.map((pair) => getVolumePair(pair.asset_infos, oneDayBeforeNow, currentDate, duckDb)) ); + console.timeEnd("getAllVolume24h"); return allVolumes; } @@ -572,12 +544,14 @@ async function getFeePair( } async function getAllFees(duckDb: DuckDb): Promise { + console.time("getAllFee"); const tf = 7 * 24 * 60 * 60; // second of 7 days const currentDate = new Date(); const oneWeekBeforeNow = getSpecificDateBeforeNow(new Date(), tf); const allFees = await Promise.all( pairs.map((pair) => getFeePair(pair.asset_infos, oneWeekBeforeNow, currentDate, duckDb)) ); + console.timeEnd("getAllFee"); return allFees; } @@ -598,9 +572,7 @@ export { getPairLiquidity, getSpecificDateBeforeNow, getSymbolFromAsset, - getVolumeLiquidity, getVolumePair, - getVolumeSwap, parseAssetInfo, parseAssetInfoOnlyDenom }; diff --git a/packages/oraidex-sync/src/poolHelper.ts b/packages/oraidex-sync/src/poolHelper.ts index 65503759..8e6e53c4 100644 --- a/packages/oraidex-sync/src/poolHelper.ts +++ b/packages/oraidex-sync/src/poolHelper.ts @@ -169,7 +169,7 @@ async function calculateLiquidityFee(pair: PairInfoData, txHeight: number, withd // ==== calculate APR ==== export const calculateAprResult = async ( - pairs: PairInfo[], + pairs: PairMapping[], allLiquidities: number[], allTokenInfo: TokenInfoResponse[], allLpTokenAsset: OraiswapStakingTypes.PoolInfoResponse[], @@ -207,15 +207,15 @@ function getStakingAssetInfo(assetInfos: AssetInfo[]): AssetInfo { } // Fetch APR -const fetchAprResult = async (pairInfos: PairInfo[], allLiquidities: number[]): Promise => { - const assetTokens = pairInfos.map((pair) => getStakingAssetInfo(JSON.parse(JSON.stringify(pair.asset_infos)))); +const fetchAprResult = async (pairInfos: PairInfoData[], allLiquidities: number[]): Promise => { + const assetTokens = pairs.map((pair) => getStakingAssetInfo(JSON.parse(JSON.stringify(pair.asset_infos)))); try { const [allTokenInfo, allLpTokenAsset, allRewardPerSec] = await Promise.all([ fetchTokenInfos(pairInfos), fetchAllTokenAssetPools(assetTokens), fetchAllRewardPerSecInfos(assetTokens) ]); - return calculateAprResult(pairInfos, allLiquidities, allTokenInfo, allLpTokenAsset, allRewardPerSec); + return calculateAprResult(pairs, allLiquidities, allTokenInfo, allLpTokenAsset, allRewardPerSec); } catch (error) { console.log({ errorFetchAprResult: error }); } diff --git a/packages/oraidex-sync/src/query.ts b/packages/oraidex-sync/src/query.ts index 30f85cbf..bb913588 100644 --- a/packages/oraidex-sync/src/query.ts +++ b/packages/oraidex-sync/src/query.ts @@ -19,6 +19,7 @@ import { } from "./helper"; import { network, tenAmountInDecimalSix, usdtCw20Address } from "./constants"; import { TokenInfoResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapToken.types"; +import { PairInfoData } from "./types"; async function queryPoolInfos(pairAddrs: string[], multicall: MulticallReadOnlyInterface): Promise { // adjust the query height to get data from the past @@ -91,9 +92,9 @@ async function aggregateMulticall(queries: Call[]) { return res.return_data.map((data) => (data.success ? fromBinary(data.data) : undefined)); } -async function fetchTokenInfos(pairInfos: PairInfo[]): Promise { +async function fetchTokenInfos(pairInfos: PairInfoData[]): Promise { const queries = pairInfos.map((pair) => ({ - address: pair.liquidity_token, + address: pair.liquidityAddr, data: toBinary({ token_info: {} } as OraiswapTokenTypes.QueryMsg) From 3175650d2d299e2d21276667eae413abe8b7c931 Mon Sep 17 00:00:00 2001 From: trungbach Date: Tue, 29 Aug 2023 16:56:06 +0700 Subject: [PATCH 17/46] fix: fixed error calculated wrong apr --- packages/oraidex-server/src/index.ts | 8 +++--- packages/oraidex-sync/src/helper.ts | 36 +++++++------------------ packages/oraidex-sync/src/poolHelper.ts | 15 ++++------- 3 files changed, 18 insertions(+), 41 deletions(-) diff --git a/packages/oraidex-server/src/index.ts b/packages/oraidex-server/src/index.ts index 116145dd..331422b1 100644 --- a/packages/oraidex-server/src/index.ts +++ b/packages/oraidex-server/src/index.ts @@ -17,7 +17,6 @@ import { ORAI, getAllVolume24h, getAllFees, - getAllPairInfos, getPairLiquidity, fetchAprResult } from "@oraichain/oraidex-sync"; @@ -243,15 +242,14 @@ app.get("/v1/pools/", async (req, res) => { console.timeEnd("get util"); + const pools = await duckDb.getPools(); console.time("allLiquidities"); const allLiquidities = await Promise.all( - pairs.map((pair) => { - return getPairLiquidity(pair.asset_infos, duckDb); + pools.map((pair) => { + return getPairLiquidity([JSON.parse(pair.firstAssetInfo), JSON.parse(pair.secondAssetInfo)], duckDb); }) ); console.timeEnd("allLiquidities"); - - const pools = await duckDb.getPools(); console.time("getApr"); const allApr = await fetchAprResult(pools, allLiquidities); console.timeEnd("getApr"); diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index 123c93db..991bda3b 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -423,11 +423,7 @@ async function fetchPoolInfoAmount(fromInfo: AssetInfo, toInfo: AssetInfo, pairA return { offerPoolAmount, askPoolAmount }; } -async function getPairLiquidity(rawAssetInfos: [AssetInfo, AssetInfo], duckDb: DuckDb): Promise { - let assetInfos: [AssetInfo, AssetInfo] = JSON.parse(JSON.stringify(rawAssetInfos)); - if (isAssetInfoPairReverse(assetInfos)) { - assetInfos = assetInfos.reverse() as [AssetInfo, AssetInfo]; - } +async function getPairLiquidity(assetInfos: [AssetInfo, AssetInfo], duckDb: DuckDb): Promise { const poolInfo = await duckDb.getPoolByAssetInfos(assetInfos); if (!poolInfo) throw new Error(`Cannot found pool info when get pair liquidity: ${JSON.stringify(assetInfos)}`); @@ -437,7 +433,15 @@ async function getPairLiquidity(rawAssetInfos: [AssetInfo, AssetInfo], duckDb: D (await fetchPoolInfoAmount(...assetInfos, poolInfo.pairAddr)); if (!poolAmounts) throw new Error(` Cannot found pool amount: ${JSON.stringify(assetInfos)}`); - const priceBaseAssetInUsdt = await getPriceAssetByUsdt(assetInfos[0]); + let priceBaseAssetInUsdt = await getPriceAssetByUsdt(assetInfos[0]); + // it means this asset not pair with ORAI + // in our pairs, if base asset not pair with ORAI, surely quote asset will pair with ORAI + if (priceBaseAssetInUsdt === 0) { + const priceQuoteAssetInUsdt = await getPriceAssetByUsdt(assetInfos[1]); + const priceBaseInQuote = await getPriceByAsset(assetInfos, "base_in_quote"); + priceBaseAssetInUsdt = priceBaseInQuote * priceQuoteAssetInUsdt; + } + return priceBaseAssetInUsdt * Number(poolAmounts.offerPoolAmount) * 2; } @@ -457,21 +461,6 @@ function convertDateToSecond(date: Date): number { return Math.round(date.valueOf() / 1000); } -async function getPairInfoFromAssets( - assetInfos: [AssetInfo, AssetInfo] -): Promise> { - const pair = getPairByAssetInfos(assetInfos); - const factoryClient = new OraiswapFactoryQueryClient( - await getCosmwasmClient(), - pair.factoryV1 ? network.factory : network.factory_v2 - ); - const pairInfo = await factoryClient.pair({ assetInfos }); - return { - contract_addr: pairInfo.contract_addr, - commission_rate: pairInfo.commission_rate - }; -} - // ====== get volume pairs ====== async function getVolumePair( [baseAssetInfo, quoteAssetInfo]: [AssetInfo, AssetInfo], @@ -508,14 +497,12 @@ async function getVolumePair( } async function getAllVolume24h(duckDb: DuckDb): Promise { - console.time("getAllVolume24h"); const tf = 24 * 60 * 60; // second of 24h const currentDate = new Date(); const oneDayBeforeNow = getSpecificDateBeforeNow(new Date(), tf); const allVolumes = await Promise.all( pairs.map((pair) => getVolumePair(pair.asset_infos, oneDayBeforeNow, currentDate, duckDb)) ); - console.timeEnd("getAllVolume24h"); return allVolumes; } @@ -544,14 +531,12 @@ async function getFeePair( } async function getAllFees(duckDb: DuckDb): Promise { - console.time("getAllFee"); const tf = 7 * 24 * 60 * 60; // second of 7 days const currentDate = new Date(); const oneWeekBeforeNow = getSpecificDateBeforeNow(new Date(), tf); const allFees = await Promise.all( pairs.map((pair) => getFeePair(pair.asset_infos, oneWeekBeforeNow, currentDate, duckDb)) ); - console.timeEnd("getAllFee"); return allFees; } @@ -568,7 +553,6 @@ export { getAllVolume24h, getCosmwasmClient, getFeePair, - getPairInfoFromAssets, getPairLiquidity, getSpecificDateBeforeNow, getSymbolFromAsset, diff --git a/packages/oraidex-sync/src/poolHelper.ts b/packages/oraidex-sync/src/poolHelper.ts index 8e6e53c4..491a263b 100644 --- a/packages/oraidex-sync/src/poolHelper.ts +++ b/packages/oraidex-sync/src/poolHelper.ts @@ -15,7 +15,6 @@ import { calculatePriceByPool, fetchPoolInfoAmount, getCosmwasmClient, - getPairInfoFromAssets, isAssetInfoPairReverse, parseAssetInfoOnlyDenom, validateNumber @@ -113,10 +112,6 @@ async function getPriceAssetByUsdt(asset: AssetInfo): Promise { parseAssetInfoOnlyDenom(foundPair.asset_infos[0]) === ORAI ? "quote_in_base" : "base_in_quote"; const priceInOrai = await getPriceByAsset(foundPair.asset_infos, ratioDirection); const priceOraiInUsdt = await getOraiPrice(); - // console.dir( - // { price: priceInOrai * priceOraiInUsdt, priceOraiInUsdt, asset: parseAssetInfoOnlyDenom(asset) }, - // { depth: null } - // ); return priceInOrai * priceOraiInUsdt; } @@ -177,7 +172,7 @@ export const calculateAprResult = async ( ): Promise => { let aprResult = []; let ind = 0; - for (const pair of pairs) { + for (const _pair of pairs) { const liquidityAmount = allLiquidities[ind] * Math.pow(10, -6); const lpToken = allLpTokenAsset[ind]; const tokenSupply = allTokenInfo[ind]; @@ -200,15 +195,15 @@ export const calculateAprResult = async ( }; function getStakingAssetInfo(assetInfos: AssetInfo[]): AssetInfo { - if (isAssetInfoPairReverse(assetInfos)) { - assetInfos = assetInfos.reverse(); - } + if (isAssetInfoPairReverse(assetInfos)) assetInfos.reverse(); return parseAssetInfoOnlyDenom(assetInfos[0]) === ORAI ? assetInfos[1] : assetInfos[0]; } // Fetch APR const fetchAprResult = async (pairInfos: PairInfoData[], allLiquidities: number[]): Promise => { - const assetTokens = pairs.map((pair) => getStakingAssetInfo(JSON.parse(JSON.stringify(pair.asset_infos)))); + const assetTokens = pairInfos.map((pair) => + getStakingAssetInfo([JSON.parse(pair.firstAssetInfo), JSON.parse(pair.secondAssetInfo)]) + ); try { const [allTokenInfo, allLpTokenAsset, allRewardPerSec] = await Promise.all([ fetchTokenInfos(pairInfos), From 39d55578e1ad81602124a11a140df81124903af9 Mon Sep 17 00:00:00 2001 From: trungbach Date: Tue, 29 Aug 2023 18:39:51 +0700 Subject: [PATCH 18/46] test: fixed error test with duckdb --- packages/oraidex-server/src/index.ts | 2 +- packages/oraidex-sync/src/db.ts | 24 ++++++++++++++----- packages/oraidex-sync/src/index.ts | 8 +------ packages/oraidex-sync/src/poolHelper.ts | 4 ++-- packages/oraidex-sync/tests/db.spec.ts | 9 ++++--- .../oraidex-sync/tests/pool-helper.spec.ts | 18 +++++++------- 6 files changed, 37 insertions(+), 28 deletions(-) diff --git a/packages/oraidex-server/src/index.ts b/packages/oraidex-server/src/index.ts index 331422b1..504008d7 100644 --- a/packages/oraidex-server/src/index.ts +++ b/packages/oraidex-server/src/index.ts @@ -271,7 +271,7 @@ app.get("/v1/pools/", async (req, res) => { app.listen(port, hostname, async () => { // sync data for the service to read - duckDb = await DuckDb.create(process.env.DUCKDB_PROD_FILENAME || "oraidex-sync-data"); + duckDb = await DuckDb.create(process.env.DUCKDB_PROD_FILENAME); const oraidexSync = await OraiDexSync.create( duckDb, process.env.RPC_URL || "https://rpc.orai.io", diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index e95e48fc..3cb5d652 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -26,14 +26,26 @@ import { import { AssetInfo } from "@oraichain/oraidex-contracts-sdk"; export class DuckDb { + static instances: DuckDb; protected constructor(public readonly conn: Connection, private db: Database) {} - static async create(fileName?: string): Promise { - let db = await Database.create(fileName ?? "data"); - await db.close(); // close to flush WAL file - db = await Database.create(fileName ?? "data"); - const conn = await db.connect(); - return new DuckDb(conn, db); + static async create(fileName: string): Promise { + if (!fileName) throw new Error("Filename is not provided!"); + // let db = await Database.create(fileName); + // await db.close(); // close to flush WAL file + // db = await Database.create(fileName); + // const conn = await db.connect(); + // return new DuckDb(conn, db); + + if (!DuckDb.instances) { + let db = await Database.create(fileName); + await db.close(); // close to flush WAL file + db = await Database.create(fileName); + const conn = await db.connect(); + DuckDb.instances = new DuckDb(conn, db); + } + + return DuckDb.instances; } async closeDb() { diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index a58fdd19..d87f41cd 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -71,7 +71,6 @@ class WriteOrders extends WriteData { } class OraiDexSync { - public static duckDbInstance: DuckDb | null = null; protected constructor( private readonly duckDb: DuckDb, private readonly rpcUrl: string, @@ -81,14 +80,9 @@ class OraiDexSync { public static async create(duckDb: DuckDb, rpcUrl: string, env: Env): Promise { const cosmwasmClient = await CosmWasmClient.connect(rpcUrl); - OraiDexSync.duckDbInstance = duckDb; // Store the provided duckDb instance return new OraiDexSync(duckDb, rpcUrl, cosmwasmClient, env); } - public static getDuckDbInstance() { - return OraiDexSync.duckDbInstance; - } - private async updateLatestPairInfos() { try { console.time("timer-updateLatestPairInfos"); @@ -158,7 +152,7 @@ async function initSync() { oraidexSync.sync(); } -initSync(); +// initSync(); export { OraiDexSync }; diff --git a/packages/oraidex-sync/src/poolHelper.ts b/packages/oraidex-sync/src/poolHelper.ts index 491a263b..265eaae1 100644 --- a/packages/oraidex-sync/src/poolHelper.ts +++ b/packages/oraidex-sync/src/poolHelper.ts @@ -28,7 +28,7 @@ import { queryPoolInfos } from "./query"; import { PairInfoData, PairMapping } from "./types"; -import { OraiDexSync, DuckDb } from "./index"; +import { DuckDb } from "./index"; // use this type to determine the ratio of price of base to the quote or vice versa export type RatioDirection = "base_in_quote" | "quote_in_base"; @@ -79,7 +79,7 @@ async function getOraiPrice(): Promise { } async function getPriceByAsset(assetInfos: [AssetInfo, AssetInfo], ratioDirection: RatioDirection): Promise { - const duckDb: DuckDb = OraiDexSync.getDuckDbInstance(); + const duckDb = DuckDb.instances; const poolInfo = await duckDb.getPoolByAssetInfos(assetInfos); if (!poolInfo) throw new Error(`Cannot found pool info: ${JSON.stringify(assetInfos)}`); diff --git a/packages/oraidex-sync/tests/db.spec.ts b/packages/oraidex-sync/tests/db.spec.ts index 3f14528d..307fade0 100644 --- a/packages/oraidex-sync/tests/db.spec.ts +++ b/packages/oraidex-sync/tests/db.spec.ts @@ -184,7 +184,8 @@ describe("test-duckdb", () => { quoteTokenDenom: "atom", txCreator: "foobar", opType: "provide", - txheight: 1 + txheight: 1, + taxRate: 1n } ]) ).rejects.toThrow(); @@ -210,7 +211,8 @@ describe("test-duckdb", () => { timestamp: newDate, txCreator: "foobar", txhash: "foo", - txheight: 1 + txheight: 1, + taxRate: 1n } ]; await duckDb.insertLpOps(data); @@ -238,7 +240,8 @@ describe("test-duckdb", () => { timestamp: currentTimeStamp, txCreator: "foobar", txhash: "foo", - txheight: 1 + txheight: 1, + taxRate: 1n } ]; await duckDb.insertLpOps(data); diff --git a/packages/oraidex-sync/tests/pool-helper.spec.ts b/packages/oraidex-sync/tests/pool-helper.spec.ts index 9223858d..d1552491 100644 --- a/packages/oraidex-sync/tests/pool-helper.spec.ts +++ b/packages/oraidex-sync/tests/pool-helper.spec.ts @@ -1,3 +1,4 @@ +import fs from "fs"; import { Asset, AssetInfo } from "@oraichain/oraidex-contracts-sdk"; import { ORAI, @@ -13,8 +14,16 @@ import { import { PairMapping } from "../src/types"; import * as helper from "../src/helper"; import * as poolHelper from "../src/poolHelper"; +import { DuckDb } from "../src/index"; describe("test-pool-helper", () => { + let duckDb: DuckDb; + beforeAll(async () => { + duckDb = await DuckDb.create(":memory:"); + }); + afterAll(() => { + fs.unlink(":memory:", () => {}); + }); afterEach(() => { jest.clearAllMocks(); }); @@ -142,23 +151,14 @@ describe("test-pool-helper", () => { const mockAskPoolAmount = 1000n; const mockOfferPoolAmount = 2000n; const mockAssetPrice = 0.5; - const mockPairInfoFromAssets = { - commission_rate: "0.003", - contract_addr: "orai1234" - }; - - const getPairInfoFromAssetsSpy = jest.spyOn(helper, "getPairInfoFromAssets"); - getPairInfoFromAssetsSpy.mockResolvedValue(mockPairInfoFromAssets); const fetchPoolInfoAmountSpy = jest.spyOn(helper, "fetchPoolInfoAmount"); const calculatePriceByPoolSpy = jest.spyOn(helper, "calculatePriceByPool"); - // Mock the fetchPoolInfoAmount function fetchPoolInfoAmountSpy.mockResolvedValue({ askPoolAmount: mockAskPoolAmount, offerPoolAmount: mockOfferPoolAmount }); - // Mock the calculatePriceByPool function calculatePriceByPoolSpy.mockReturnValue(mockAssetPrice); const result = await poolHelper.getPriceByAsset(assetInfos, ratioDirection); From 6f941d13e1d08500a5c5991e80ead94bdc35ccb2 Mon Sep 17 00:00:00 2001 From: trungbach Date: Wed, 30 Aug 2023 17:38:38 +0700 Subject: [PATCH 19/46] type: update type alias & remove redunant code --- packages/oraidex-sync/src/constants.ts | 8 +++++--- packages/oraidex-sync/src/db.ts | 6 ------ packages/oraidex-sync/src/tx-parsing.ts | 9 +-------- packages/oraidex-sync/src/types.ts | 2 +- 4 files changed, 7 insertions(+), 18 deletions(-) diff --git a/packages/oraidex-sync/src/constants.ts b/packages/oraidex-sync/src/constants.ts index 33798232..f92b0706 100644 --- a/packages/oraidex-sync/src/constants.ts +++ b/packages/oraidex-sync/src/constants.ts @@ -1,3 +1,5 @@ +import { AssetInfo } from "@oraichain/common-contracts-sdk"; + export const ORAI = "orai"; export const airiCw20Adress = "orai10ldgzued6zjp0mkqwsv2mux3ml50l97c74x8sg"; export const oraixCw20Address = "orai1lus0f0rhx8s03gdllx2n6vhkmf0536dv57wfge"; @@ -13,9 +15,9 @@ export const osmosisIbcDenom = "ibc/9C4DCD21B48231D0BC2AC3D1B74A864746B37E429269 export const tenAmountInDecimalSix = 10000000; export const truncDecimals = 6; export const atomic = 10 ** truncDecimals; -export const oraiInfo = { native_token: { denom: ORAI } }; -export const usdtInfo = { token: { contract_addr: usdcCw20Address } }; -export const ORAIXOCH_INFO = { +export const oraiInfo: AssetInfo = { native_token: { denom: ORAI } }; +export const usdtInfo: AssetInfo = { token: { contract_addr: usdtCw20Address } }; +export const ORAIXOCH_INFO: AssetInfo = { token: { contract_addr: "orai1lplapmgqnelqn253stz6kmvm3ulgdaytn89a8mz9y85xq8wd684s6xl3lt" } diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index 3cb5d652..737333e2 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -31,12 +31,6 @@ export class DuckDb { static async create(fileName: string): Promise { if (!fileName) throw new Error("Filename is not provided!"); - // let db = await Database.create(fileName); - // await db.close(); // close to flush WAL file - // db = await Database.create(fileName); - // const conn = await db.connect(); - // return new DuckDb(conn, db); - if (!DuckDb.instances) { let db = await Database.create(fileName); await db.close(); // close to flush WAL file diff --git a/packages/oraidex-sync/src/tx-parsing.ts b/packages/oraidex-sync/src/tx-parsing.ts index a76f755e..3921ddfd 100644 --- a/packages/oraidex-sync/src/tx-parsing.ts +++ b/packages/oraidex-sync/src/tx-parsing.ts @@ -362,11 +362,4 @@ async function parseTxs(txs: Tx[], duckDb: DuckDb): Promise { }; } -export { - parseAssetInfo, - parseWasmEvents, - parseTxs, - parseWithdrawLiquidityAssets, - parseTxToMsgExecuteContractMsgs, - calculateLiquidityFee -}; +export { parseAssetInfo, parseWasmEvents, parseTxs, parseWithdrawLiquidityAssets, parseTxToMsgExecuteContractMsgs }; diff --git a/packages/oraidex-sync/src/types.ts b/packages/oraidex-sync/src/types.ts index 5b710a59..b5c0c47c 100644 --- a/packages/oraidex-sync/src/types.ts +++ b/packages/oraidex-sync/src/types.ts @@ -75,7 +75,7 @@ export type ProvideLiquidityOperationData = { opType: LiquidityOpType; uniqueKey: string; // concat of first, second denom, amount, and timestamp => should be unique. unique key is used to override duplication only. txCreator: string; - taxRate: bigint; + taxRate: number | bigint; } & BasicTxData; export type WithdrawLiquidityOperationData = ProvideLiquidityOperationData; From bab86868108ab0954c90830f632278357cb4e799 Mon Sep 17 00:00:00 2001 From: trungbach Date: Wed, 30 Aug 2023 17:41:44 +0700 Subject: [PATCH 20/46] test: update testcase for pool info --- packages/oraidex-sync/src/poolHelper.ts | 5 +- packages/oraidex-sync/tests/db.spec.ts | 13 +- packages/oraidex-sync/tests/helper.spec.ts | 4 +- .../oraidex-sync/tests/pool-helper.spec.ts | 174 +++++++++++++++--- 4 files changed, 163 insertions(+), 33 deletions(-) diff --git a/packages/oraidex-sync/src/poolHelper.ts b/packages/oraidex-sync/src/poolHelper.ts index 265eaae1..2e6d9181 100644 --- a/packages/oraidex-sync/src/poolHelper.ts +++ b/packages/oraidex-sync/src/poolHelper.ts @@ -115,7 +115,7 @@ async function getPriceAssetByUsdt(asset: AssetInfo): Promise { return priceInOrai * priceOraiInUsdt; } -async function calculateFeeByUsdt(fee: Asset): Promise { +async function calculateFeeByUsdt(fee: Asset | null): Promise { if (!fee) return 0; const priceInUsdt = await getPriceAssetByUsdt(fee.info); return priceInUsdt * +fee.amount; @@ -234,5 +234,6 @@ export { getPriceAssetByUsdt, getPriceByAsset, isPoolHasFee, - getStakingAssetInfo + getStakingAssetInfo, + calculateFeeByUsdt }; diff --git a/packages/oraidex-sync/tests/db.spec.ts b/packages/oraidex-sync/tests/db.spec.ts index 307fade0..c5fcb57b 100644 --- a/packages/oraidex-sync/tests/db.spec.ts +++ b/packages/oraidex-sync/tests/db.spec.ts @@ -1,10 +1,14 @@ import { DuckDb } from "../src/db"; import { isoToTimestampNumber } from "../src/helper"; import { ProvideLiquidityOperationData } from "../src/types"; - +import fs from "fs"; describe("test-duckdb", () => { let duckDb: DuckDb; + afterEach(() => { + fs.unlink(":memory:", () => {}); + }); + it.each<[string[], number[]]>([ [ ["orai", "atom"], @@ -194,7 +198,7 @@ describe("test-duckdb", () => { it("test-duckdb-insert-bulk-should-pass-and-can-query", async () => { //setup duckDb = await DuckDb.create(":memory:"); - await Promise.all([duckDb.createHeightSnapshot(), duckDb.createLiquidityOpsTable(), duckDb.createSwapOpsTable()]); + await Promise.all([duckDb.createLiquidityOpsTable()]); // act & test const newDate = 1689610068000 / 1000; const data: ProvideLiquidityOperationData[] = [ @@ -212,9 +216,10 @@ describe("test-duckdb", () => { txCreator: "foobar", txhash: "foo", txheight: 1, - taxRate: 1n + taxRate: 1 } ]; + await duckDb.insertLpOps(data); let queryResult = await duckDb.queryLpOps(); queryResult[0].timestamp = queryResult[0].timestamp; @@ -241,7 +246,7 @@ describe("test-duckdb", () => { txCreator: "foobar", txhash: "foo", txheight: 1, - taxRate: 1n + taxRate: 1 } ]; await duckDb.insertLpOps(data); diff --git a/packages/oraidex-sync/tests/helper.spec.ts b/packages/oraidex-sync/tests/helper.spec.ts index e1e2135d..3338386e 100644 --- a/packages/oraidex-sync/tests/helper.spec.ts +++ b/packages/oraidex-sync/tests/helper.spec.ts @@ -370,9 +370,7 @@ describe("test-helper", () => { "test-calculatePriceByPool-ORAI/USDT-pool-with-commision-rate=%s-should-return-price-%s-USDT", (commisionRate, expectedPrice) => { // base denom is ORAI, quote denom is USDT => base pool is ORAI, quote pool is USDT. - // const result = calculatePriceByPool(BigInt(639997269712), BigInt(232967274783), commisionRate, 10 ** 6); - const result = calculatePriceByPool(BigInt(397832351391), BigInt(193971155696), commisionRate); - console.log({ result }); + const result = calculatePriceByPool(BigInt(639997269712), BigInt(232967274783), commisionRate, 10 ** 6); expect(result.toString()).toEqual(expectedPrice); } ); diff --git a/packages/oraidex-sync/tests/pool-helper.spec.ts b/packages/oraidex-sync/tests/pool-helper.spec.ts index d1552491..21b70cdc 100644 --- a/packages/oraidex-sync/tests/pool-helper.spec.ts +++ b/packages/oraidex-sync/tests/pool-helper.spec.ts @@ -1,5 +1,5 @@ -import fs from "fs"; import { Asset, AssetInfo } from "@oraichain/oraidex-contracts-sdk"; +import fs from "fs"; import { ORAI, airiCw20Adress, @@ -11,18 +11,22 @@ import { usdtInfo } from "../src/constants"; -import { PairMapping } from "../src/types"; import * as helper from "../src/helper"; -import * as poolHelper from "../src/poolHelper"; import { DuckDb } from "../src/index"; +import * as poolHelper from "../src/poolHelper"; +import { PairInfoData, PairMapping, ProvideLiquidityOperationData } from "../src/types"; describe("test-pool-helper", () => { let duckDb: DuckDb; beforeAll(async () => { duckDb = await DuckDb.create(":memory:"); - }); - afterAll(() => { - fs.unlink(":memory:", () => {}); + await Promise.all([ + duckDb.createHeightSnapshot(), + duckDb.createLiquidityOpsTable(), + duckDb.createSwapOpsTable(), + duckDb.createPairInfosTable(), + duckDb.createSwapOhlcv() + ]); }); afterEach(() => { jest.clearAllMocks(); @@ -142,24 +146,43 @@ describe("test-pool-helper", () => { ); describe("test-calculate-price-group-funcs", () => { + // use orai/usdt in this test suite afterEach(jest.clearAllMocks); + + it("test-getPriceByAsset-when-duckdb-empty-should-throw-error", async () => { + await expect(poolHelper.getPriceByAsset([oraiInfo, usdtInfo], "base_in_quote")).rejects.toBeInstanceOf(Error); + }); + it.each<[[AssetInfo, AssetInfo], poolHelper.RatioDirection, number]>([ [[oraiInfo, usdtInfo], "base_in_quote", 0.5], [[oraiInfo, usdtInfo], "quote_in_base", 2] ])("test-getPriceByAsset-should-return-correctly-price", async (assetInfos, ratioDirection, expectedPrice) => { - // Mock the return values for fetchPoolInfoAmount and calculatePriceByPool - const mockAskPoolAmount = 1000n; - const mockOfferPoolAmount = 2000n; - const mockAssetPrice = 0.5; - - const fetchPoolInfoAmountSpy = jest.spyOn(helper, "fetchPoolInfoAmount"); - const calculatePriceByPoolSpy = jest.spyOn(helper, "calculatePriceByPool"); - fetchPoolInfoAmountSpy.mockResolvedValue({ - askPoolAmount: mockAskPoolAmount, - offerPoolAmount: mockOfferPoolAmount - }); + // setup + let pairInfoData: PairInfoData[] = [ + { + firstAssetInfo: JSON.stringify({ native_token: { denom: ORAI } } as AssetInfo), + secondAssetInfo: JSON.stringify({ token: { contract_addr: usdtCw20Address } } as AssetInfo), + commissionRate: "", + pairAddr: "orai1c5s03c3l336dgesne7dylnmhszw8554tsyy9yt", + liquidityAddr: "", + oracleAddr: "", + symbols: "1", + fromIconUrl: "1", + toIconUrl: "1", + volume24Hour: 1n, + apr: 1, + totalLiquidity: 1, + fee7Days: 1n + } + ]; + await duckDb.insertPairInfos(pairInfoData); - calculatePriceByPoolSpy.mockReturnValue(mockAssetPrice); + // mock result of nested function implemented inside getPriceByAsset. + jest.spyOn(helper, "fetchPoolInfoAmount").mockResolvedValue({ + askPoolAmount: 1000n, + offerPoolAmount: 2000n + }); + jest.spyOn(helper, "calculatePriceByPool").mockReturnValue(0.5); const result = await poolHelper.getPriceByAsset(assetInfos, ratioDirection); expect(result).toEqual(expectedPrice); @@ -167,7 +190,7 @@ describe("test-pool-helper", () => { it.each([ ["asset-is-cw20-USDT", usdtInfo, 1], - ["asset-is-ORAI", oraiInfo, 2], + ["asset-is-ORAI", oraiInfo, 0.5], [ "asset-is-not-pair-with-ORAI", { @@ -180,18 +203,121 @@ describe("test-pool-helper", () => { ])( "test-getPriceAssetByUsdt-with-%p-should-return-price-of-asset-in-USDT", async (_caseName: string, assetInfo: AssetInfo, expectedPrice: number) => { - const mockPriceByAssetValue = 4; - jest.spyOn(poolHelper, "getPriceByAsset").mockResolvedValue(mockPriceByAssetValue); - - const mockOraiPrice = 2; - jest.spyOn(poolHelper, "getOraiPrice").mockResolvedValue(mockOraiPrice); + // setup + let pairInfoData: PairInfoData[] = [ + { + firstAssetInfo: JSON.stringify(oraiInfo as AssetInfo), + secondAssetInfo: JSON.stringify(usdtInfo as AssetInfo), + commissionRate: "", + pairAddr: "orai1c5s03c3l336dgesne7dylnmhszw8554tsyy9yt", + liquidityAddr: "", + oracleAddr: "", + symbols: "1", + fromIconUrl: "1", + toIconUrl: "1", + volume24Hour: 1n, + apr: 1, + totalLiquidity: 1, + fee7Days: 1n + }, + { + firstAssetInfo: JSON.stringify(oraiInfo as AssetInfo), + secondAssetInfo: JSON.stringify({ native_token: { denom: atomIbcDenom } } as AssetInfo), + commissionRate: "", + pairAddr: "orai/atom", + liquidityAddr: "", + oracleAddr: "", + symbols: "1", + fromIconUrl: "1", + toIconUrl: "1", + volume24Hour: 1n, + apr: 1, + totalLiquidity: 1, + fee7Days: 1n + } + ]; + await duckDb.insertPairInfos(pairInfoData); + const data: ProvideLiquidityOperationData[] = [ + { + basePrice: 1, + baseTokenAmount: 1, + baseTokenDenom: "orai", + baseTokenReserve: 1000000000 / 2, + opType: "withdraw", + uniqueKey: "2", + quoteTokenAmount: 2, + quoteTokenDenom: usdtCw20Address, + quoteTokenReserve: 1000000000, + timestamp: 1, + txCreator: "foobar", + txhash: "foo", + txheight: 1, + taxRate: 1n + } + ]; + await duckDb.insertLpOps(data); + jest.spyOn(poolHelper, "getPriceByAsset").mockResolvedValue(4); + jest.spyOn(poolHelper, "getOraiPrice").mockResolvedValue(2); + // act const result = await poolHelper.getPriceAssetByUsdt(assetInfo); + + // assertion expect(result).toEqual(expectedPrice); } ); }); + describe("test-calculate-fee-of-pools", () => { + // setup + const usdtFeeInfo = { + info: usdtInfo, + amount: "1" + }; + + const shareRatio = 0.5; + + it.each([ + ["test-calculateFeeByAsset-with-asset-info-is-NOT-native-token-should-return-null", usdtInfo, null], + [ + "test-calculateFeeByAsset-with-asset-info-is-native-token-should-return-correctly-fee", + oraiInfo, + { amount: "0.11538461538461542", info: { native_token: { denom: "orai" } } } + ] + ])("%s", (_caseName: string, assetInfo: AssetInfo, expectedResult: Asset | null) => { + // act + const result = poolHelper.calculateFeeByAsset( + { + info: assetInfo, + amount: "1" + }, + shareRatio + ); + // assert + expect(result).toEqual(expectedResult); + }); + + it.each([ + ["test-calculateFeeByUsdt-with-asset-NULL-should-return-fee-is-0", null, 0], + [ + "test-calculateFeeByUsdt-with-asset-native-should-return-correctly-fee", + { + info: oraiInfo, + amount: "1" + }, + 0.5 + ] + ])("%s", async (_caseName: string, assetFee: Asset | null, expectedResult: number) => { + // mock + jest.spyOn(poolHelper, "getPriceAssetByUsdt").mockResolvedValueOnce(2); + + // act + const result = await poolHelper.calculateFeeByUsdt(assetFee); + + // assert + expect(result).toEqual(expectedResult); + }); + }); it.each<[string, AssetInfo[], AssetInfo]>([ [ "case-asset-info-pairs-is-NOT-reversed-and-base-asset-NOT-ORAI", From 31cdc3a13a5e2c72549090b5a1880480fadfa687 Mon Sep 17 00:00:00 2001 From: trungbach Date: Wed, 30 Aug 2023 17:52:57 +0700 Subject: [PATCH 21/46] refactor: use duckdb instance instead pass into function parameter --- packages/oraidex-server/src/index.ts | 6 ++--- packages/oraidex-sync/src/helper.ts | 25 ++++++++----------- .../oraidex-sync/tests/pool-helper.spec.ts | 12 +++------ 3 files changed, 17 insertions(+), 26 deletions(-) diff --git a/packages/oraidex-server/src/index.ts b/packages/oraidex-server/src/index.ts index 504008d7..0717a652 100644 --- a/packages/oraidex-server/src/index.ts +++ b/packages/oraidex-server/src/index.ts @@ -235,10 +235,10 @@ app.get("/v1/candles/", async (req: Request<{}, {}, {}, GetCandlesQuery>, res) = } }); -app.get("/v1/pools/", async (req, res) => { +app.get("/v1/pools/", async (_req, res) => { try { console.time("get util"); - const [volumes, allFee7Days] = await Promise.all([getAllVolume24h(duckDb), getAllFees(duckDb)]); + const [volumes, allFee7Days] = await Promise.all([getAllVolume24h(), getAllFees()]); console.timeEnd("get util"); @@ -246,7 +246,7 @@ app.get("/v1/pools/", async (req, res) => { console.time("allLiquidities"); const allLiquidities = await Promise.all( pools.map((pair) => { - return getPairLiquidity([JSON.parse(pair.firstAssetInfo), JSON.parse(pair.secondAssetInfo)], duckDb); + return getPairLiquidity([JSON.parse(pair.firstAssetInfo), JSON.parse(pair.secondAssetInfo)]); }) ); console.timeEnd("allLiquidities"); diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index 991bda3b..8443b670 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -423,7 +423,8 @@ async function fetchPoolInfoAmount(fromInfo: AssetInfo, toInfo: AssetInfo, pairA return { offerPoolAmount, askPoolAmount }; } -async function getPairLiquidity(assetInfos: [AssetInfo, AssetInfo], duckDb: DuckDb): Promise { +async function getPairLiquidity(assetInfos: [AssetInfo, AssetInfo]): Promise { + const duckDb = DuckDb.instances; const poolInfo = await duckDb.getPoolByAssetInfos(assetInfos); if (!poolInfo) throw new Error(`Cannot found pool info when get pair liquidity: ${JSON.stringify(assetInfos)}`); @@ -465,9 +466,9 @@ function convertDateToSecond(date: Date): number { async function getVolumePair( [baseAssetInfo, quoteAssetInfo]: [AssetInfo, AssetInfo], startTime: Date, - endTime: Date, - duckDb: DuckDb + endTime: Date ): Promise { + const duckDb = DuckDb.instances; const pair = `${parseAssetInfoOnlyDenom(baseAssetInfo)}-${parseAssetInfoOnlyDenom(quoteAssetInfo)}`; const [volumeSwapPairInBaseAsset, volumeLiquidityPairInBaseAsset] = await Promise.all([ duckDb.getVolumeSwap({ @@ -496,23 +497,19 @@ async function getVolumePair( return BigInt(Math.round(volumeInUsdt)); } -async function getAllVolume24h(duckDb: DuckDb): Promise { +async function getAllVolume24h(): Promise { const tf = 24 * 60 * 60; // second of 24h const currentDate = new Date(); const oneDayBeforeNow = getSpecificDateBeforeNow(new Date(), tf); const allVolumes = await Promise.all( - pairs.map((pair) => getVolumePair(pair.asset_infos, oneDayBeforeNow, currentDate, duckDb)) + pairs.map((pair) => getVolumePair(pair.asset_infos, oneDayBeforeNow, currentDate)) ); return allVolumes; } // ==== get fee pair ==== -async function getFeePair( - asset_infos: [AssetInfo, AssetInfo], - startTime: Date, - endTime: Date, - duckDb: DuckDb -): Promise { +async function getFeePair(asset_infos: [AssetInfo, AssetInfo], startTime: Date, endTime: Date): Promise { + const duckDb = DuckDb.instances; const [swapFee, liquidityFee] = await Promise.all([ duckDb.getFeeSwap({ offerDenom: parseAssetInfoOnlyDenom(asset_infos[0]), @@ -530,13 +527,11 @@ async function getFeePair( return swapFee + liquidityFee; } -async function getAllFees(duckDb: DuckDb): Promise { +async function getAllFees(): Promise { const tf = 7 * 24 * 60 * 60; // second of 7 days const currentDate = new Date(); const oneWeekBeforeNow = getSpecificDateBeforeNow(new Date(), tf); - const allFees = await Promise.all( - pairs.map((pair) => getFeePair(pair.asset_infos, oneWeekBeforeNow, currentDate, duckDb)) - ); + const allFees = await Promise.all(pairs.map((pair) => getFeePair(pair.asset_infos, oneWeekBeforeNow, currentDate))); return allFees; } diff --git a/packages/oraidex-sync/tests/pool-helper.spec.ts b/packages/oraidex-sync/tests/pool-helper.spec.ts index 21b70cdc..a847951a 100644 --- a/packages/oraidex-sync/tests/pool-helper.spec.ts +++ b/packages/oraidex-sync/tests/pool-helper.spec.ts @@ -1,5 +1,4 @@ import { Asset, AssetInfo } from "@oraichain/oraidex-contracts-sdk"; -import fs from "fs"; import { ORAI, airiCw20Adress, @@ -35,7 +34,7 @@ describe("test-pool-helper", () => { it.each<[string, [AssetInfo, AssetInfo], boolean]>([ [ "has-both-native-token-that-contain-ORAI-should-return: false", - [{ native_token: { denom: ORAI } }, { native_token: { denom: atomIbcDenom } }], + [oraiInfo, { native_token: { denom: atomIbcDenom } }], false ], // [ @@ -64,11 +63,7 @@ describe("test-pool-helper", () => { contract_addr: milkyCw20Address } }, - { - token: { - contract_addr: usdtCw20Address - } - } + usdtInfo ], false ] @@ -184,6 +179,7 @@ describe("test-pool-helper", () => { }); jest.spyOn(helper, "calculatePriceByPool").mockReturnValue(0.5); + // assert const result = await poolHelper.getPriceByAsset(assetInfos, ratioDirection); expect(result).toEqual(expectedPrice); }); @@ -274,7 +270,6 @@ describe("test-pool-helper", () => { info: usdtInfo, amount: "1" }; - const shareRatio = 0.5; it.each([ @@ -318,6 +313,7 @@ describe("test-pool-helper", () => { expect(result).toEqual(expectedResult); }); }); + it.each<[string, AssetInfo[], AssetInfo]>([ [ "case-asset-info-pairs-is-NOT-reversed-and-base-asset-NOT-ORAI", From 9b4912fe505cd0af4c4a575dfa3b323e76794704 Mon Sep 17 00:00:00 2001 From: trungbach Date: Wed, 30 Aug 2023 17:56:34 +0700 Subject: [PATCH 22/46] refactor: remove redundant code --- packages/oraidex-server/src/index.ts | 1 - packages/oraidex-sync/src/helper.ts | 22 ----- packages/oraidex-sync/tests/helper.spec.ts | 104 +++++---------------- 3 files changed, 21 insertions(+), 106 deletions(-) diff --git a/packages/oraidex-server/src/index.ts b/packages/oraidex-server/src/index.ts index 0717a652..bfb32b27 100644 --- a/packages/oraidex-server/src/index.ts +++ b/packages/oraidex-server/src/index.ts @@ -83,7 +83,6 @@ app.get("/tickers", async (req, res) => { const symbols = pair.symbols; const pairAddr = findPairAddress(pairInfos, pair.asset_infos); const tickerId = parseSymbolsToTickerId(symbols); - // const { baseIndex, targetIndex, target } = findUsdOraiInPair(pair.asset_infos); const baseIndex = 0; const targetIndex = 1; console.log(latestTimestamp, then); diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index 8443b670..beaca2d4 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -357,7 +357,6 @@ export function getSwapDirection(offerDenom: string, askDenom: string): SwapDire if (!pair) { console.error("Cannot find asset infos in list of pairs"); return; - // throw new Error("Cannot find asset infos in list of pairs"); } const assetInfos = pair.asset_infos; // use quote denom as offer then its buy. Quote denom in pairs is the 2nd index in the array @@ -371,27 +370,6 @@ export function findPairIndexFromDenoms(offerDenom: string, askDenom: string): n ); } -// /** -// * -// * @param infos -// * @returns -// */ -// function findUsdOraiInPair(infos: [AssetInfo, AssetInfo]): { -// baseIndex: number; -// targetIndex: number; -// target: AssetInfo; -// } { -// const firstInfo = parseAssetInfoOnlyDenom(infos[0]); -// const secondInfo = parseAssetInfoOnlyDenom(infos[1]); -// if (firstInfo === usdtCw20Address || firstInfo === usdcCw20Address) -// return { baseIndex: 0, targetIndex: 1, target: infos[1] }; -// if (secondInfo === usdtCw20Address || secondInfo === usdcCw20Address) -// return { baseIndex: 1, targetIndex: 0, target: infos[0] }; -// if (firstInfo === ORAI) return { baseIndex: 0, targetIndex: 1, target: infos[1] }; -// if (secondInfo === ORAI) return { baseIndex: 1, targetIndex: 0, target: infos[0] }; -// return { baseIndex: 1, targetIndex: 0, target: infos[0] }; // default we calculate the first info in the asset info list -// } - function getSymbolFromAsset(asset_infos: [AssetInfo, AssetInfo]): string { const findedPair = pairs.find( (p) => diff --git a/packages/oraidex-sync/tests/helper.spec.ts b/packages/oraidex-sync/tests/helper.spec.ts index 3338386e..ca8ddaa9 100644 --- a/packages/oraidex-sync/tests/helper.spec.ts +++ b/packages/oraidex-sync/tests/helper.spec.ts @@ -1,25 +1,5 @@ -import { Asset, AssetInfo } from "@oraichain/oraidex-contracts-sdk"; -import { - findAssetInfoPathToUsdt, - findMappedTargetedAssetInfo, - findPairAddress, - calculatePriceByPool, - toAmount, - toDisplay, - toDecimal, - roundTime, - groupByTime, - collectAccumulateLpData, - concatDataToUniqueKey, - removeOpsDuplication, - calculateBasePriceFromSwapOp, - getSwapDirection, - findPairIndexFromDenoms, - toObject, - calculateSwapOhlcv, - isAssetInfoPairReverse -} from "../src/helper"; -import { extractUniqueAndFlatten, pairs } from "../src/pairs"; +import { AssetInfo } from "@oraichain/oraidex-contracts-sdk"; +import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; import { ORAI, airiCw20Adress, @@ -36,8 +16,26 @@ import { usdtCw20Address, usdtInfo } from "../src/constants"; +import { + calculateBasePriceFromSwapOp, + calculatePriceByPool, + collectAccumulateLpData, + concatDataToUniqueKey, + findAssetInfoPathToUsdt, + findMappedTargetedAssetInfo, + findPairAddress, + findPairIndexFromDenoms, + getSwapDirection, + groupByTime, + isAssetInfoPairReverse, + removeOpsDuplication, + roundTime, + toAmount, + toDecimal, + toDisplay +} from "../src/helper"; +import { extractUniqueAndFlatten, pairs } from "../src/pairs"; import { PairInfoData, ProvideLiquidityOperationData, SwapDirection, SwapOperationData } from "../src/types"; -import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; describe("test-helper", () => { describe("bigint", () => { @@ -523,66 +521,6 @@ describe("test-helper", () => { expect(newOps[1].uniqueKey).toEqual("2"); }); - // it.each<[[AssetInfo, AssetInfo], AssetInfo, number]>([ - // [ - // [{ native_token: { denom: ORAI } }, { native_token: { denom: atomIbcDenom } }], - // { native_token: { denom: atomIbcDenom } }, - // 0 - // ], - // [ - // [{ native_token: { denom: ORAI } }, { token: { contract_addr: usdtCw20Address } }], - // { native_token: { denom: ORAI } }, - // 1 - // ], - // [ - // [{ native_token: { denom: ORAI } }, { token: { contract_addr: usdcCw20Address } }], - // { native_token: { denom: ORAI } }, - // 1 - // ], - // [ - // [{ token: { contract_addr: tronCw20Address } }, { native_token: { denom: atomIbcDenom } }], - // { token: { contract_addr: tronCw20Address } }, - // 1 - // ] - // ])("test-findUsdOraiInPair", (infos, expectedInfo, expectedBase) => { - // // act - // const result = findUsdOraiInPair(infos); - // // assert - // expect(result.target).toEqual(expectedInfo); - // expect(result.baseIndex).toEqual(expectedBase); - // }); - - // it.each([ - // [ - // [ - // { - // timestamp: 60000, - // pair: "orai-usdt", - // price: 1, - // volume: 100n - // }, - // { - // timestamp: 60000, - // pair: "orai-usdt", - // price: 2, - // volume: 100n - // } - // ], - // { - // open: 1, - // close: 2, - // low: 1, - // high: 2, - // volume: 200n, - // timestamp: 60000, - // pair: "orai-usdt" - // } - // ] - // ])("test-calculateOhlcv", (ops, expectedOhlcv) => { - // const ohlcv = calculateSwapOhlcv(ops); - // expect(toObject(ohlcv)).toEqual(toObject(expectedOhlcv)); - // }); - it.each([ ["Buy" as SwapDirection, 2], ["Sell" as SwapDirection, 0.5] From 3cfe585a4617fc03b438f72bee7dc885dd3d77a0 Mon Sep 17 00:00:00 2001 From: trungbach Date: Wed, 30 Aug 2023 18:29:20 +0700 Subject: [PATCH 23/46] update: added some testcase for helper in oraidex-sync --- packages/oraidex-sync/src/helper.ts | 31 +++++++++++++------- packages/oraidex-sync/src/poolHelper.ts | 1 + packages/oraidex-sync/tests/helper.spec.ts | 34 ++++++++++++++++++++++ 3 files changed, 56 insertions(+), 10 deletions(-) diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index beaca2d4..37ba745c 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -441,13 +441,13 @@ function convertDateToSecond(date: Date): number { } // ====== get volume pairs ====== -async function getVolumePair( - [baseAssetInfo, quoteAssetInfo]: [AssetInfo, AssetInfo], +async function getVolumePairByAsset( + [baseDenom, quoteDenom]: [string, string], startTime: Date, endTime: Date ): Promise { const duckDb = DuckDb.instances; - const pair = `${parseAssetInfoOnlyDenom(baseAssetInfo)}-${parseAssetInfoOnlyDenom(quoteAssetInfo)}`; + const pair = `${baseDenom}-${quoteDenom}`; const [volumeSwapPairInBaseAsset, volumeLiquidityPairInBaseAsset] = await Promise.all([ duckDb.getVolumeSwap({ pair, @@ -455,23 +455,33 @@ async function getVolumePair( endTime: convertDateToSecond(endTime) }), duckDb.getVolumeLiquidity({ - offerDenom: parseAssetInfoOnlyDenom(baseAssetInfo), - askDenom: parseAssetInfoOnlyDenom(quoteAssetInfo), + offerDenom: baseDenom, + askDenom: quoteDenom, startTime: convertDateToSecond(startTime), endTime: convertDateToSecond(endTime) }) ]); + return volumeSwapPairInBaseAsset + volumeLiquidityPairInBaseAsset; +} + +async function getVolumePairByUsdt( + [baseAssetInfo, quoteAssetInfo]: [AssetInfo, AssetInfo], + startTime: Date, + endTime: Date +): Promise { + const [baseDenom, quoteDenom] = [parseAssetInfoOnlyDenom(baseAssetInfo), parseAssetInfoOnlyDenom(quoteAssetInfo)]; + const volumePairInBaseAsset = await getVolumePairByAsset([baseDenom, quoteDenom], startTime, endTime); let priceBaseAssetInUsdt = await getPriceAssetByUsdt(baseAssetInfo); // it means this asset not pair with ORAI // in our pairs, if base asset not pair with ORAI, surely quote asset will pair with ORAI + // TODO: should refactor this, not need to check 0 in this case, update later. if (priceBaseAssetInUsdt === 0) { const priceQuoteAssetInUsdt = await getPriceAssetByUsdt(quoteAssetInfo); const priceBaseInQuote = await getPriceByAsset([baseAssetInfo, quoteAssetInfo], "base_in_quote"); priceBaseAssetInUsdt = priceBaseInQuote * priceQuoteAssetInUsdt; } - const volumeInUsdt = - priceBaseAssetInUsdt * (Number(volumeSwapPairInBaseAsset) + Number(volumeLiquidityPairInBaseAsset)); + const volumeInUsdt = priceBaseAssetInUsdt * Number(volumePairInBaseAsset); return BigInt(Math.round(volumeInUsdt)); } @@ -480,7 +490,7 @@ async function getAllVolume24h(): Promise { const currentDate = new Date(); const oneDayBeforeNow = getSpecificDateBeforeNow(new Date(), tf); const allVolumes = await Promise.all( - pairs.map((pair) => getVolumePair(pair.asset_infos, oneDayBeforeNow, currentDate)) + pairs.map((pair) => getVolumePairByUsdt(pair.asset_infos, oneDayBeforeNow, currentDate)) ); return allVolumes; } @@ -529,7 +539,8 @@ export { getPairLiquidity, getSpecificDateBeforeNow, getSymbolFromAsset, - getVolumePair, + getVolumePairByUsdt, parseAssetInfo, - parseAssetInfoOnlyDenom + parseAssetInfoOnlyDenom, + getVolumePairByAsset }; diff --git a/packages/oraidex-sync/src/poolHelper.ts b/packages/oraidex-sync/src/poolHelper.ts index 2e6d9181..f1c41e5a 100644 --- a/packages/oraidex-sync/src/poolHelper.ts +++ b/packages/oraidex-sync/src/poolHelper.ts @@ -106,6 +106,7 @@ async function getPriceAssetByUsdt(asset: AssetInfo): Promise { if (parseAssetInfoOnlyDenom(asset) === parseAssetInfoOnlyDenom(oraiInfo)) return await getOraiPrice(); const foundPair = getPairByAssetInfos([asset, oraiInfo]); + // TODO: should refactor this to find asset that matched pair with input asset instead return 0 if (!foundPair) return 0; const ratioDirection: RatioDirection = diff --git a/packages/oraidex-sync/tests/helper.spec.ts b/packages/oraidex-sync/tests/helper.spec.ts index ca8ddaa9..0ee4dcf3 100644 --- a/packages/oraidex-sync/tests/helper.spec.ts +++ b/packages/oraidex-sync/tests/helper.spec.ts @@ -36,8 +36,11 @@ import { } from "../src/helper"; import { extractUniqueAndFlatten, pairs } from "../src/pairs"; import { PairInfoData, ProvideLiquidityOperationData, SwapDirection, SwapOperationData } from "../src/types"; +import { DuckDb, getVolumePairByAsset, getVolumePairByUsdt } from "../src"; describe("test-helper", () => { + let duckDb: DuckDb; + describe("bigint", () => { describe("toAmount", () => { it("toAmount-percent", () => { @@ -585,4 +588,35 @@ describe("test-helper", () => { expect(result).toBe(expectedResult); } ); + + describe("test-get-volume-pairs", () => { + it("test-getVolumePairByAsset-should-return-correctly-sum-volume-swap-&-liquidity", async () => { + //setup mock + duckDb = await DuckDb.create(":memory:"); + jest.spyOn(duckDb, "getVolumeSwap").mockResolvedValue(1n); + jest.spyOn(duckDb, "getVolumeLiquidity").mockResolvedValue(1n); + + // act + const result = await getVolumePairByAsset(["orai", "usdt"], new Date(1693394183), new Date(1693394183)); + + // assert + expect(result).toEqual(2n); + }); + + // it("test-getVolumePairByUsdt-should-return-correctly-volume-pair-in-USDT", async () => { + // //setup + // duckDb = await DuckDb.create(":memory:"); + // const [baseAssetInfo, quoteAssetInfo] = [oraiInfo, usdtInfo]; + + // // act + // const result = await getVolumePairByUsdt( + // [baseAssetInfo, quoteAssetInfo], + // new Date(1693394183), + // new Date(1693394183) + // ); + + // // assert + // expect(result).toEqual(2n); + // }); + }); }); From 4760e8629bae45e8132f61c83c91f2f30eae7146 Mon Sep 17 00:00:00 2001 From: trungbach Date: Wed, 6 Sep 2023 09:45:59 +0700 Subject: [PATCH 24/46] style: renamed function to make more sense --- packages/oraidex-sync/src/poolHelper.ts | 6 +++--- packages/oraidex-sync/tests/pool-helper.spec.ts | 14 +++++--------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/packages/oraidex-sync/src/poolHelper.ts b/packages/oraidex-sync/src/poolHelper.ts index f1c41e5a..e8aaa784 100644 --- a/packages/oraidex-sync/src/poolHelper.ts +++ b/packages/oraidex-sync/src/poolHelper.ts @@ -116,7 +116,7 @@ async function getPriceAssetByUsdt(asset: AssetInfo): Promise { return priceInOrai * priceOraiInUsdt; } -async function calculateFeeByUsdt(fee: Asset | null): Promise { +async function convertFeeAssetToUsdt(fee: Asset | null): Promise { if (!fee) return 0; const priceInUsdt = await getPriceAssetByUsdt(fee.info); return priceInUsdt * +fee.amount; @@ -159,7 +159,7 @@ async function calculateLiquidityFee(pair: PairInfoData, txHeight: number, withd calculateFeeByAsset(poolInfo.assets[1], shareRatio) ]; - const feeByUsdt = (await calculateFeeByUsdt(feeByAssetFrom)) + (await calculateFeeByUsdt(feeByAssetTo)); + const feeByUsdt = (await convertFeeAssetToUsdt(feeByAssetFrom)) + (await convertFeeAssetToUsdt(feeByAssetTo)); return BigInt(Math.round(feeByUsdt)); } @@ -236,5 +236,5 @@ export { getPriceByAsset, isPoolHasFee, getStakingAssetInfo, - calculateFeeByUsdt + convertFeeAssetToUsdt }; diff --git a/packages/oraidex-sync/tests/pool-helper.spec.ts b/packages/oraidex-sync/tests/pool-helper.spec.ts index a847951a..f499fbec 100644 --- a/packages/oraidex-sync/tests/pool-helper.spec.ts +++ b/packages/oraidex-sync/tests/pool-helper.spec.ts @@ -233,7 +233,7 @@ describe("test-pool-helper", () => { } ]; await duckDb.insertPairInfos(pairInfoData); - const data: ProvideLiquidityOperationData[] = [ + const lpOpsData: ProvideLiquidityOperationData[] = [ { basePrice: 1, baseTokenAmount: 1, @@ -251,7 +251,7 @@ describe("test-pool-helper", () => { taxRate: 1n } ]; - await duckDb.insertLpOps(data); + await duckDb.insertLpOps(lpOpsData); jest.spyOn(poolHelper, "getPriceByAsset").mockResolvedValue(4); jest.spyOn(poolHelper, "getOraiPrice").mockResolvedValue(2); @@ -266,10 +266,6 @@ describe("test-pool-helper", () => { describe("test-calculate-fee-of-pools", () => { // setup - const usdtFeeInfo = { - info: usdtInfo, - amount: "1" - }; const shareRatio = 0.5; it.each([ @@ -293,9 +289,9 @@ describe("test-pool-helper", () => { }); it.each([ - ["test-calculateFeeByUsdt-with-asset-NULL-should-return-fee-is-0", null, 0], + ["test-convertFeeAssetToUsdt-with-asset-NULL-should-return-fee-is-0", null, 0], [ - "test-calculateFeeByUsdt-with-asset-native-should-return-correctly-fee", + "test-convertFeeAssetToUsdt-with-asset-native-should-return-correctly-fee", { info: oraiInfo, amount: "1" @@ -307,7 +303,7 @@ describe("test-pool-helper", () => { jest.spyOn(poolHelper, "getPriceAssetByUsdt").mockResolvedValueOnce(2); // act - const result = await poolHelper.calculateFeeByUsdt(assetFee); + const result = await poolHelper.convertFeeAssetToUsdt(assetFee); // assert expect(result).toEqual(expectedResult); From 0362f76b33495bd082da020ae9f768e53b6f5010 Mon Sep 17 00:00:00 2001 From: trungbach Date: Wed, 6 Sep 2023 11:50:07 +0700 Subject: [PATCH 25/46] refactor: remove console.time --- packages/oraidex-server/src/index.ts | 7 ------- packages/oraidex-sync/src/db.ts | 4 ++-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/packages/oraidex-server/src/index.ts b/packages/oraidex-server/src/index.ts index bfb32b27..c12c336e 100644 --- a/packages/oraidex-server/src/index.ts +++ b/packages/oraidex-server/src/index.ts @@ -236,22 +236,15 @@ app.get("/v1/candles/", async (req: Request<{}, {}, {}, GetCandlesQuery>, res) = app.get("/v1/pools/", async (_req, res) => { try { - console.time("get util"); const [volumes, allFee7Days] = await Promise.all([getAllVolume24h(), getAllFees()]); - console.timeEnd("get util"); - const pools = await duckDb.getPools(); - console.time("allLiquidities"); const allLiquidities = await Promise.all( pools.map((pair) => { return getPairLiquidity([JSON.parse(pair.firstAssetInfo), JSON.parse(pair.secondAssetInfo)]); }) ); - console.timeEnd("allLiquidities"); - console.time("getApr"); const allApr = await fetchAprResult(pools, allLiquidities); - console.timeEnd("getApr"); res.status(200).send( pools.map((pool, index) => { return { diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index 737333e2..22ff152b 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -457,7 +457,7 @@ export class DuckDb { const result = await this.conn.all( ` SELECT - cast(sum(volume) as UBIGINT) as totalVolume, + sum(volume) as totalVolume, FROM swap_ohlcv WHERE timestamp >= ? AND timestamp <= ? @@ -467,7 +467,7 @@ export class DuckDb { endTime, pair ); - return result[0]?.totalVolume ?? 0; + return BigInt(result[0]?.totalVolume ?? 0); } async getVolumeLiquidity(payload: GetFeeSwap): Promise { From a9d7bd1b2f0f5c0951742485bed54adb437ee029 Mon Sep 17 00:00:00 2001 From: trungbach Date: Wed, 6 Sep 2023 18:21:20 +0700 Subject: [PATCH 26/46] refactor: finished calculate price asset by usdt --- packages/oraidex-sync/src/helper.ts | 31 ++-------- packages/oraidex-sync/src/index.ts | 4 +- .../src/{poolHelper.ts => pool-helper.ts} | 61 +++++++++++++++---- packages/oraidex-sync/src/tx-parsing.ts | 2 +- 4 files changed, 58 insertions(+), 40 deletions(-) rename packages/oraidex-sync/src/{poolHelper.ts => pool-helper.ts} (78%) diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index 37ba745c..28d5e6f4 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -1,18 +1,16 @@ import { AssetInfo, CosmWasmClient, - OraiswapFactoryQueryClient, OraiswapPairQueryClient, OraiswapPairTypes, - PairInfo, SwapOperation } from "@oraichain/oraidex-contracts-sdk"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; import { isEqual, maxBy, minBy } from "lodash"; -import { ORAI, atomic, network, tenAmountInDecimalSix, truncDecimals, usdtCw20Address } from "./constants"; +import { ORAI, atomic, tenAmountInDecimalSix, truncDecimals, usdtCw20Address } from "./constants"; import { DuckDb } from "./db"; import { pairs, pairsOnlyDenom } from "./pairs"; -import { getPairByAssetInfos, getPriceAssetByUsdt, getPriceByAsset } from "./poolHelper"; +import { getPriceAssetByUsdt, getPriceByAsset } from "./pool-helper"; import { Ohlcv, OraiDexType, @@ -412,15 +410,7 @@ async function getPairLiquidity(assetInfos: [AssetInfo, AssetInfo]): Promise { const [baseDenom, quoteDenom] = [parseAssetInfoOnlyDenom(baseAssetInfo), parseAssetInfoOnlyDenom(quoteAssetInfo)]; const volumePairInBaseAsset = await getVolumePairByAsset([baseDenom, quoteDenom], startTime, endTime); - let priceBaseAssetInUsdt = await getPriceAssetByUsdt(baseAssetInfo); - - // it means this asset not pair with ORAI - // in our pairs, if base asset not pair with ORAI, surely quote asset will pair with ORAI - // TODO: should refactor this, not need to check 0 in this case, update later. - if (priceBaseAssetInUsdt === 0) { - const priceQuoteAssetInUsdt = await getPriceAssetByUsdt(quoteAssetInfo); - const priceBaseInQuote = await getPriceByAsset([baseAssetInfo, quoteAssetInfo], "base_in_quote"); - priceBaseAssetInUsdt = priceBaseInQuote * priceQuoteAssetInUsdt; - } + const priceBaseAssetInUsdt = await getPriceAssetByUsdt(baseAssetInfo); const volumeInUsdt = priceBaseAssetInUsdt * Number(volumePairInBaseAsset); return BigInt(Math.round(volumeInUsdt)); } @@ -539,8 +520,8 @@ export { getPairLiquidity, getSpecificDateBeforeNow, getSymbolFromAsset, + getVolumePairByAsset, getVolumePairByUsdt, parseAssetInfo, - parseAssetInfoOnlyDenom, - getVolumePairByAsset + parseAssetInfoOnlyDenom }; diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index d87f41cd..b32379a7 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -3,7 +3,7 @@ import { CosmWasmClient } from "@oraichain/oraidex-contracts-sdk"; import "dotenv/config"; import { DuckDb } from "./db"; import { collectAccumulateLpData, getSymbolFromAsset } from "./helper"; -import { getAllPairInfos, getPoolInfos } from "./poolHelper"; +import { getAllPairInfos, getPoolInfos } from "./pool-helper"; import { parseAssetInfo, parseTxs } from "./tx-parsing"; import { Env, @@ -160,6 +160,6 @@ export * from "./constants"; export * from "./db"; export * from "./helper"; export * from "./pairs"; -export * from "./poolHelper"; +export * from "./pool-helper"; export * from "./query"; export * from "./types"; diff --git a/packages/oraidex-sync/src/poolHelper.ts b/packages/oraidex-sync/src/pool-helper.ts similarity index 78% rename from packages/oraidex-sync/src/poolHelper.ts rename to packages/oraidex-sync/src/pool-helper.ts index e8aaa784..e459ab3e 100644 --- a/packages/oraidex-sync/src/poolHelper.ts +++ b/packages/oraidex-sync/src/pool-helper.ts @@ -1,4 +1,4 @@ -import { MulticallQueryClient } from "@oraichain/common-contracts-sdk"; +import { MulticallQueryClient, Ratio } from "@oraichain/common-contracts-sdk"; import { Asset, AssetInfo, @@ -83,7 +83,7 @@ async function getPriceByAsset(assetInfos: [AssetInfo, AssetInfo], ratioDirectio const poolInfo = await duckDb.getPoolByAssetInfos(assetInfos); if (!poolInfo) throw new Error(`Cannot found pool info: ${JSON.stringify(assetInfos)}`); - // get info of last tx in lp_ops_data, if not have data => get info from contract + // get info of latest tx in lp_ops_data table, if lp_ops_data not have data yet => get info from contract let poolAmounts = (await duckDb.getPoolAmountFromAssetInfos(assetInfos)) ?? (await fetchPoolInfoAmount(...assetInfos, poolInfo.pairAddr)); @@ -99,19 +99,56 @@ async function getPriceByAsset(assetInfos: [AssetInfo, AssetInfo], ratioDirectio return ratioDirection === "base_in_quote" ? basePrice : 1 / basePrice; } -// find pool match this asset with orai => calculate price this asset token in ORAI. -// then, calculate price of this asset token in USDT based on price ORAI in USDT. +/** + * @param asset + * asset is: + * 1, usdt=1, + * 2, orai=getOraiPrice, + * 3, pair with usdt: getPriceByAsset, + * 4, pair with orai: get price in orai * price orai in usdt, + * 5, otherwise, pair with orai || usdt: find pair of input asset vs other asset that mapped with: + * 5.1, orai (ex: scAtom -> scAtom/Atom -> Atom/orai -> step 4) + * 5.2, usdt: this case does not occurs. + * @returns price asset by USDT + */ async function getPriceAssetByUsdt(asset: AssetInfo): Promise { if (parseAssetInfoOnlyDenom(asset) === parseAssetInfoOnlyDenom(usdtInfo)) return 1; if (parseAssetInfoOnlyDenom(asset) === parseAssetInfoOnlyDenom(oraiInfo)) return await getOraiPrice(); + let foundPair: PairMapping; - const foundPair = getPairByAssetInfos([asset, oraiInfo]); - // TODO: should refactor this to find asset that matched pair with input asset instead return 0 - if (!foundPair) return 0; + // find pair map with usdt + foundPair = getPairByAssetInfos([asset, usdtInfo]); + if (foundPair) { + // assume asset mapped with usdt should be base asset + return await getPriceByAsset(foundPair.asset_infos, "base_in_quote"); + } + + // find pair map with orai + let priceInOrai = 0; + foundPair = getPairByAssetInfos([asset, oraiInfo]); + if (foundPair) { + const ratioDirection: RatioDirection = + parseAssetInfoOnlyDenom(foundPair.asset_infos[0]) === ORAI ? "quote_in_base" : "base_in_quote"; + priceInOrai = await getPriceByAsset(foundPair.asset_infos, ratioDirection); + } else { + // case 5.1 + const pairWithAsset = pairs.find((pair) => + pair.asset_infos.some((info) => parseAssetInfoOnlyDenom(info) === parseAssetInfoOnlyDenom(asset)) + ); + if (!pairWithAsset) throw new Error("Something wrong with whitelist pair in oraiDEX."); + const otherAssetIndex = pairWithAsset.asset_infos.findIndex( + (item) => parseAssetInfoOnlyDenom(item) !== parseAssetInfoOnlyDenom(asset) + ); + const priceAssetVsOtherAsset = await getPriceByAsset( + pairWithAsset.asset_infos, + otherAssetIndex === 1 ? "base_in_quote" : "quote_in_base" + ); + const pairOtherAssetVsOrai = getPairByAssetInfos([pairWithAsset.asset_infos[otherAssetIndex], oraiInfo]); + const ratioDirection: RatioDirection = + parseAssetInfoOnlyDenom(pairOtherAssetVsOrai.asset_infos[0]) === ORAI ? "quote_in_base" : "base_in_quote"; + priceInOrai = priceAssetVsOtherAsset * (await getPriceByAsset(pairOtherAssetVsOrai.asset_infos, ratioDirection)); + } - const ratioDirection: RatioDirection = - parseAssetInfoOnlyDenom(foundPair.asset_infos[0]) === ORAI ? "quote_in_base" : "base_in_quote"; - const priceInOrai = await getPriceByAsset(foundPair.asset_infos, ratioDirection); const priceOraiInUsdt = await getOraiPrice(); return priceInOrai * priceOraiInUsdt; } @@ -163,7 +200,7 @@ async function calculateLiquidityFee(pair: PairInfoData, txHeight: number, withd return BigInt(Math.round(feeByUsdt)); } -// ==== calculate APR ==== +// <==== calculate APR ==== export const calculateAprResult = async ( pairs: PairMapping[], allLiquidities: number[], @@ -200,7 +237,6 @@ function getStakingAssetInfo(assetInfos: AssetInfo[]): AssetInfo { return parseAssetInfoOnlyDenom(assetInfos[0]) === ORAI ? assetInfos[1] : assetInfos[0]; } -// Fetch APR const fetchAprResult = async (pairInfos: PairInfoData[], allLiquidities: number[]): Promise => { const assetTokens = pairInfos.map((pair) => getStakingAssetInfo([JSON.parse(pair.firstAssetInfo), JSON.parse(pair.secondAssetInfo)]) @@ -216,6 +252,7 @@ const fetchAprResult = async (pairInfos: PairInfoData[], allLiquidities: number[ console.log({ errorFetchAprResult: error }); } }; +// ==== end of calculate APR ====> async function getAllPairInfos(): Promise { const cosmwasmClient = await getCosmwasmClient(); diff --git a/packages/oraidex-sync/src/tx-parsing.ts b/packages/oraidex-sync/src/tx-parsing.ts index 3921ddfd..4e97fbc8 100644 --- a/packages/oraidex-sync/src/tx-parsing.ts +++ b/packages/oraidex-sync/src/tx-parsing.ts @@ -32,7 +32,7 @@ import { } from "./helper"; import { pairs } from "./pairs"; import { DuckDb } from "./db"; -import { calculateLiquidityFee, isPoolHasFee } from "./poolHelper"; +import { calculateLiquidityFee, isPoolHasFee } from "./pool-helper"; function parseWasmEvents(events: readonly Event[]): (readonly Attribute[])[] { return events.filter((event) => event.type === "wasm").map((event) => event.attributes); From 5bed1d60cd3c8fec6c60a28e00bb476393f4620c Mon Sep 17 00:00:00 2001 From: trungbach Date: Thu, 7 Sep 2023 01:20:40 +0700 Subject: [PATCH 27/46] testcase: fixed error mock function in same module by use export const instead export function --- packages/oraidex-sync/src/helper.ts | 12 +- packages/oraidex-sync/src/index.ts | 15 +- packages/oraidex-sync/src/pool-helper.ts | 75 +++---- packages/oraidex-sync/src/tx-parsing.ts | 9 +- .../oraidex-sync/tests/pool-helper.spec.ts | 209 ++++++++++-------- 5 files changed, 165 insertions(+), 155 deletions(-) diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index 28d5e6f4..72cd6fa5 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -10,7 +10,7 @@ import { isEqual, maxBy, minBy } from "lodash"; import { ORAI, atomic, tenAmountInDecimalSix, truncDecimals, usdtCw20Address } from "./constants"; import { DuckDb } from "./db"; import { pairs, pairsOnlyDenom } from "./pairs"; -import { getPriceAssetByUsdt, getPriceByAsset } from "./pool-helper"; +import { getPriceAssetByUsdt } from "./pool-helper"; import { Ohlcv, OraiDexType, @@ -399,6 +399,7 @@ async function fetchPoolInfoAmount(fromInfo: AssetInfo, toInfo: AssetInfo, pairA return { offerPoolAmount, askPoolAmount }; } +// get liquidity of pair from assetInfos async function getPairLiquidity(assetInfos: [AssetInfo, AssetInfo]): Promise { const duckDb = DuckDb.instances; const poolInfo = await duckDb.getPoolByAssetInfos(assetInfos); @@ -410,7 +411,8 @@ async function getPairLiquidity(assetInfos: [AssetInfo, AssetInfo]): Promise { ); return allVolumes; } +// ===== end get volume pairs =====> -// ==== get fee pair ==== +// <==== start get fee pair ==== async function getFeePair(asset_infos: [AssetInfo, AssetInfo], startTime: Date, endTime: Date): Promise { const duckDb = DuckDb.instances; const [swapFee, liquidityFee] = await Promise.all([ @@ -503,6 +506,7 @@ async function getAllFees(): Promise { const allFees = await Promise.all(pairs.map((pair) => getFeePair(pair.asset_infos, oneWeekBeforeNow, currentDate))); return allFees; } +// ==== end get fee pair ====> export { calculatePriceByPool, diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index b32379a7..26238ff5 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -1,5 +1,4 @@ import { SyncData, Txs, WriteData } from "@oraichain/cosmos-rpc-sync"; -import { CosmWasmClient } from "@oraichain/oraidex-contracts-sdk"; import "dotenv/config"; import { DuckDb } from "./db"; import { collectAccumulateLpData, getSymbolFromAsset } from "./helper"; @@ -15,10 +14,8 @@ import { } from "./types"; class WriteOrders extends WriteData { - private firstWrite: boolean; constructor(private duckDb: DuckDb, private rpcUrl: string, private env: Env, private initialData: InitialData) { super(); - this.firstWrite = true; } private async insertParsedTxs(txs: TxAnlysisResult) { @@ -71,16 +68,10 @@ class WriteOrders extends WriteData { } class OraiDexSync { - protected constructor( - private readonly duckDb: DuckDb, - private readonly rpcUrl: string, - private cosmwasmClient: CosmWasmClient, - private readonly env: Env - ) {} + protected constructor(private readonly duckDb: DuckDb, private readonly rpcUrl: string, private readonly env: Env) {} public static async create(duckDb: DuckDb, rpcUrl: string, env: Env): Promise { - const cosmwasmClient = await CosmWasmClient.connect(rpcUrl); - return new OraiDexSync(duckDb, rpcUrl, cosmwasmClient, env); + return new OraiDexSync(duckDb, rpcUrl, env); } private async updateLatestPairInfos() { @@ -89,7 +80,7 @@ class OraiDexSync { const pairInfos = await getAllPairInfos(); await this.duckDb.insertPairInfos( - pairInfos.map((pair, index) => { + pairInfos.map((pair) => { const symbols = getSymbolFromAsset(pair.asset_infos); return { firstAssetInfo: parseAssetInfo(pair.asset_infos[0]), diff --git a/packages/oraidex-sync/src/pool-helper.ts b/packages/oraidex-sync/src/pool-helper.ts index e459ab3e..a3e36bb0 100644 --- a/packages/oraidex-sync/src/pool-helper.ts +++ b/packages/oraidex-sync/src/pool-helper.ts @@ -1,4 +1,4 @@ -import { MulticallQueryClient, Ratio } from "@oraichain/common-contracts-sdk"; +import { MulticallQueryClient } from "@oraichain/common-contracts-sdk"; import { Asset, AssetInfo, @@ -37,7 +37,7 @@ export type RatioDirection = "base_in_quote" | "quote_in_base"; * Check pool if has native token is not ORAI -> has fee * @returns boolean */ -function isPoolHasFee(assetInfos: [AssetInfo, AssetInfo]): boolean { +export const isPoolHasFee = (assetInfos: [AssetInfo, AssetInfo]): boolean => { let hasNative = false; for (const asset of assetInfos) { if ("native_token" in asset) { @@ -49,18 +49,18 @@ function isPoolHasFee(assetInfos: [AssetInfo, AssetInfo]): boolean { } if (hasNative) return true; return false; -} +}; -async function getPoolInfos(pairAddrs: string[], wantedHeight?: number): Promise { +export const getPoolInfos = async (pairAddrs: string[], wantedHeight?: number): Promise => { // adjust the query height to get data from the past const cosmwasmClient = await getCosmwasmClient(); cosmwasmClient.setQueryClientWithHeight(wantedHeight); const multicall = new MulticallQueryClient(cosmwasmClient, network.multicall); const res = await queryPoolInfos(pairAddrs, multicall); return res; -} +}; -function getPairByAssetInfos(assetInfos: [AssetInfo, AssetInfo]): PairMapping { +export const getPairByAssetInfos = (assetInfos: [AssetInfo, AssetInfo]): PairMapping => { return pairs.find((pair) => { const [baseAsset, quoteAsset] = pair.asset_infos; const denoms = [parseAssetInfoOnlyDenom(baseAsset), parseAssetInfoOnlyDenom(quoteAsset)]; @@ -68,17 +68,21 @@ function getPairByAssetInfos(assetInfos: [AssetInfo, AssetInfo]): PairMapping { denoms.includes(parseAssetInfoOnlyDenom(assetInfos[0])) && denoms.includes(parseAssetInfoOnlyDenom(assetInfos[1])) ); }); -} +}; // get price ORAI in USDT base on ORAI/USDT pool. -async function getOraiPrice(): Promise { +// async function getOraiPrice(): Promise { +export const getOraiPrice = async (): Promise => { const oraiUsdtPair = getPairByAssetInfos([oraiInfo, usdtInfo]); const ratioDirection: RatioDirection = parseAssetInfoOnlyDenom(oraiUsdtPair.asset_infos[0]) === ORAI ? "base_in_quote" : "quote_in_base"; return getPriceByAsset([oraiInfo, usdtInfo], ratioDirection); -} +}; -async function getPriceByAsset(assetInfos: [AssetInfo, AssetInfo], ratioDirection: RatioDirection): Promise { +export const getPriceByAsset = async ( + assetInfos: [AssetInfo, AssetInfo], + ratioDirection: RatioDirection +): Promise => { const duckDb = DuckDb.instances; const poolInfo = await duckDb.getPoolByAssetInfos(assetInfos); if (!poolInfo) throw new Error(`Cannot found pool info: ${JSON.stringify(assetInfos)}`); @@ -97,7 +101,7 @@ async function getPriceByAsset(assetInfos: [AssetInfo, AssetInfo], ratioDirectio +poolInfo.commissionRate ); return ratioDirection === "base_in_quote" ? basePrice : 1 / basePrice; -} +}; /** * @param asset @@ -111,7 +115,7 @@ async function getPriceByAsset(assetInfos: [AssetInfo, AssetInfo], ratioDirectio * 5.2, usdt: this case does not occurs. * @returns price asset by USDT */ -async function getPriceAssetByUsdt(asset: AssetInfo): Promise { +export const getPriceAssetByUsdt = async (asset: AssetInfo): Promise => { if (parseAssetInfoOnlyDenom(asset) === parseAssetInfoOnlyDenom(usdtInfo)) return 1; if (parseAssetInfoOnlyDenom(asset) === parseAssetInfoOnlyDenom(oraiInfo)) return await getOraiPrice(); let foundPair: PairMapping; @@ -135,7 +139,6 @@ async function getPriceAssetByUsdt(asset: AssetInfo): Promise { const pairWithAsset = pairs.find((pair) => pair.asset_infos.some((info) => parseAssetInfoOnlyDenom(info) === parseAssetInfoOnlyDenom(asset)) ); - if (!pairWithAsset) throw new Error("Something wrong with whitelist pair in oraiDEX."); const otherAssetIndex = pairWithAsset.asset_infos.findIndex( (item) => parseAssetInfoOnlyDenom(item) !== parseAssetInfoOnlyDenom(asset) ); @@ -151,15 +154,15 @@ async function getPriceAssetByUsdt(asset: AssetInfo): Promise { const priceOraiInUsdt = await getOraiPrice(); return priceInOrai * priceOraiInUsdt; -} +}; -async function convertFeeAssetToUsdt(fee: Asset | null): Promise { +export const convertFeeAssetToUsdt = async (fee: Asset | null): Promise => { if (!fee) return 0; const priceInUsdt = await getPriceAssetByUsdt(fee.info); return priceInUsdt * +fee.amount; -} +}; -function calculateFeeByAsset(asset: Asset, shareRatio: number): Asset { +export const calculateFeeByAsset = (asset: Asset, shareRatio: number): Asset => { const TAX_CAP = 10 ** 6; const TAX_RATE = 0.3; // just native_token not ORAI has fee @@ -171,10 +174,10 @@ function calculateFeeByAsset(asset: Asset, shareRatio: number): Asset { amount: fee.toString(), info: asset.info }; -} +}; /** - * First, calculate fee by offer asset & askAsset + * First, calculate fee by offer asset & ask asset * then, calculate fee of those asset to ORAI * finally, convert this fee in ORAI to USDT. * @param pair @@ -182,7 +185,11 @@ function calculateFeeByAsset(asset: Asset, shareRatio: number): Asset { * @param withdrawnShare * @returns fee in USDT */ -async function calculateLiquidityFee(pair: PairInfoData, txHeight: number, withdrawnShare: number): Promise { +export const calculateLiquidityFee = async ( + pair: PairInfoData, + txHeight: number, + withdrawnShare: number +): Promise => { const cosmwasmClient = await getCosmwasmClient(); cosmwasmClient.setQueryClientWithHeight(txHeight); @@ -198,11 +205,10 @@ async function calculateLiquidityFee(pair: PairInfoData, txHeight: number, withd const feeByUsdt = (await convertFeeAssetToUsdt(feeByAssetFrom)) + (await convertFeeAssetToUsdt(feeByAssetTo)); return BigInt(Math.round(feeByUsdt)); -} +}; // <==== calculate APR ==== export const calculateAprResult = async ( - pairs: PairMapping[], allLiquidities: number[], allTokenInfo: TokenInfoResponse[], allLpTokenAsset: OraiswapStakingTypes.PoolInfoResponse[], @@ -232,12 +238,12 @@ export const calculateAprResult = async ( return aprResult; }; -function getStakingAssetInfo(assetInfos: AssetInfo[]): AssetInfo { +export const getStakingAssetInfo = (assetInfos: AssetInfo[]): AssetInfo => { if (isAssetInfoPairReverse(assetInfos)) assetInfos.reverse(); return parseAssetInfoOnlyDenom(assetInfos[0]) === ORAI ? assetInfos[1] : assetInfos[0]; -} +}; -const fetchAprResult = async (pairInfos: PairInfoData[], allLiquidities: number[]): Promise => { +export const fetchAprResult = async (pairInfos: PairInfoData[], allLiquidities: number[]): Promise => { const assetTokens = pairInfos.map((pair) => getStakingAssetInfo([JSON.parse(pair.firstAssetInfo), JSON.parse(pair.secondAssetInfo)]) ); @@ -247,31 +253,16 @@ const fetchAprResult = async (pairInfos: PairInfoData[], allLiquidities: number[ fetchAllTokenAssetPools(assetTokens), fetchAllRewardPerSecInfos(assetTokens) ]); - return calculateAprResult(pairs, allLiquidities, allTokenInfo, allLpTokenAsset, allRewardPerSec); + return calculateAprResult(allLiquidities, allTokenInfo, allLpTokenAsset, allRewardPerSec); } catch (error) { console.log({ errorFetchAprResult: error }); } }; // ==== end of calculate APR ====> -async function getAllPairInfos(): Promise { +export const getAllPairInfos = async (): Promise => { const cosmwasmClient = await getCosmwasmClient(); const firstFactoryClient = new OraiswapFactoryQueryClient(cosmwasmClient, network.factory); const secondFactoryClient = new OraiswapFactoryQueryClient(cosmwasmClient, network.factory_v2); return queryAllPairInfos(firstFactoryClient, secondFactoryClient); -} - -export { - calculateFeeByAsset, - calculateLiquidityFee, - fetchAprResult, - getAllPairInfos, - getOraiPrice, - getPairByAssetInfos, - getPoolInfos, - getPriceAssetByUsdt, - getPriceByAsset, - isPoolHasFee, - getStakingAssetInfo, - convertFeeAssetToUsdt }; diff --git a/packages/oraidex-sync/src/tx-parsing.ts b/packages/oraidex-sync/src/tx-parsing.ts index 4e97fbc8..c6d2d79c 100644 --- a/packages/oraidex-sync/src/tx-parsing.ts +++ b/packages/oraidex-sync/src/tx-parsing.ts @@ -123,8 +123,7 @@ async function getFeeLiquidity( [baseDenom, quoteDenom]: [string, string], opType: LiquidityOpType, attrs: readonly Attribute[], - txheight: number, - duckDb: DuckDb + txheight: number ): Promise { // we only have one pair order. If the order is reversed then we also reverse the order let findedPair = pairs.find((pair) => @@ -151,6 +150,7 @@ async function getFeeLiquidity( opType === "provide" ? attrs.find((attr) => attr.key === "share").value : attrs.find((attr) => attr.key === "withdrawn_share").value; + const duckDb = DuckDb.instances; const pair = await duckDb.getPoolByAssetInfos(findedPair.asset_infos); fee = await calculateLiquidityFee(pair, txheight, +lpShare); console.log(`fee ${opType} liquidity: $${fee}`); @@ -184,8 +184,7 @@ async function extractMsgProvideLiquidity( [parseAssetInfoOnlyDenom(baseAsset.info), parseAssetInfoOnlyDenom(quoteAsset.info)], "provide", attrs, - txData.txheight, - duckDb + txData.txheight ); return { basePrice: calculatePriceByPool(BigInt(firstAmount), BigInt(secAmount)), @@ -254,7 +253,7 @@ async function extractMsgWithdrawLiquidity( } if (assets.length !== 4) continue; - const fee = await getFeeLiquidity([baseAsset, quoteAsset], "withdraw", attrs, txData.txheight, duckDb); + const fee = await getFeeLiquidity([baseAsset, quoteAsset], "withdraw", attrs, txData.txheight); withdrawData.push({ basePrice: calculatePriceByPool(BigInt(baseAssetAmount), BigInt(quoteAssetAmount)), diff --git a/packages/oraidex-sync/tests/pool-helper.spec.ts b/packages/oraidex-sync/tests/pool-helper.spec.ts index f499fbec..ebbc8081 100644 --- a/packages/oraidex-sync/tests/pool-helper.spec.ts +++ b/packages/oraidex-sync/tests/pool-helper.spec.ts @@ -12,7 +12,7 @@ import { import * as helper from "../src/helper"; import { DuckDb } from "../src/index"; -import * as poolHelper from "../src/poolHelper"; +import * as poolHelper from "../src/pool-helper"; import { PairInfoData, PairMapping, ProvideLiquidityOperationData } from "../src/types"; describe("test-pool-helper", () => { @@ -29,6 +29,7 @@ describe("test-pool-helper", () => { }); afterEach(() => { jest.clearAllMocks(); + jest.restoreAllMocks(); }); it.each<[string, [AssetInfo, AssetInfo], boolean]>([ @@ -72,40 +73,6 @@ describe("test-pool-helper", () => { expect(result).toBe(expectIsHasFee); }); - it.each([ - [ - "test-calculateFeeByAsset-with-case-asset-is-cw20-token-should-return-null", - { - info: { - token: { - contract_addr: airiCw20Adress - } - }, - amount: "100" - }, - null - ], - [ - "test-calculateFeeByAsset-with-case-asset-is-native-token-should-return-correctly-fee", - { - info: { - native_token: { - denom: atomIbcDenom - } - }, - amount: "100" - }, - { - amount: "1000000", - info: { native_token: { denom: atomIbcDenom } } - } - ] - ])("%s", (_caseName: string, inputAsset: Asset, expectedFee: Asset | null) => { - const shareRatio = 100000; - const result = poolHelper.calculateFeeByAsset(inputAsset, shareRatio); - expect(result).toStrictEqual(expectedFee); - }); - it.each<[string, [AssetInfo, AssetInfo], PairMapping | undefined]>([ [ "assetInfos-valid-in-list-pairs", @@ -142,8 +109,6 @@ describe("test-pool-helper", () => { describe("test-calculate-price-group-funcs", () => { // use orai/usdt in this test suite - afterEach(jest.clearAllMocks); - it("test-getPriceByAsset-when-duckdb-empty-should-throw-error", async () => { await expect(poolHelper.getPriceByAsset([oraiInfo, usdtInfo], "base_in_quote")).rejects.toBeInstanceOf(Error); }); @@ -186,21 +151,39 @@ describe("test-pool-helper", () => { it.each([ ["asset-is-cw20-USDT", usdtInfo, 1], - ["asset-is-ORAI", oraiInfo, 0.5], [ - "asset-is-not-pair-with-ORAI", + "asset-is-MILKY-that-mapped-with-USDT", + { + token: { + contract_addr: milkyCw20Address + } + }, + 0.5 + ], + ["asset-is-ORAI", oraiInfo, 2], + [ + "asset-is-pair-with-ORAI", { native_token: { denom: atomIbcDenom } }, 1 + ], + [ + "asset-is-NOT-pair-with-ORAI", + { + token: { + contract_addr: scAtomCw20Address + } + }, + 0.5 ] ])( - "test-getPriceAssetByUsdt-with-%p-should-return-price-of-asset-in-USDT", + "test-getPriceAssetByUsdt-with-%p-should-return-correctly-price-of-asset-in-USDT", async (_caseName: string, assetInfo: AssetInfo, expectedPrice: number) => { - // setup - let pairInfoData: PairInfoData[] = [ + // setup & mock + const pairInfoData: PairInfoData[] = [ { firstAssetInfo: JSON.stringify(oraiInfo as AssetInfo), secondAssetInfo: JSON.stringify(usdtInfo as AssetInfo), @@ -252,8 +235,8 @@ describe("test-pool-helper", () => { } ]; await duckDb.insertLpOps(lpOpsData); - jest.spyOn(poolHelper, "getPriceByAsset").mockResolvedValue(4); jest.spyOn(poolHelper, "getOraiPrice").mockResolvedValue(2); + jest.spyOn(poolHelper, "getPriceByAsset").mockResolvedValue(0.5); // act const result = await poolHelper.getPriceAssetByUsdt(assetInfo); @@ -265,27 +248,67 @@ describe("test-pool-helper", () => { }); describe("test-calculate-fee-of-pools", () => { - // setup - const shareRatio = 0.5; - it.each([ - ["test-calculateFeeByAsset-with-asset-info-is-NOT-native-token-should-return-null", usdtInfo, null], [ - "test-calculateFeeByAsset-with-asset-info-is-native-token-should-return-correctly-fee", - oraiInfo, - { amount: "0.11538461538461542", info: { native_token: { denom: "orai" } } } + "with-case-asset-is-cw20-token-should-return-null", + { + info: { + token: { + contract_addr: airiCw20Adress + } + }, + amount: "100" + }, + null + ], + [ + "with-case-asset-is-native-token-should-return-correctly-fee", + { + info: { + native_token: { + denom: atomIbcDenom + } + }, + amount: "100" + }, + { + amount: "11.53846153846154", + info: { native_token: { denom: atomIbcDenom } } + } ] - ])("%s", (_caseName: string, assetInfo: AssetInfo, expectedResult: Asset | null) => { + ])("test-calculateFeeByAsset-%s", (_caseName: string, inputAsset: Asset, expectedFee: Asset | null) => { + const shareRatio = 0.5; + const result = poolHelper.calculateFeeByAsset(inputAsset, shareRatio); + expect(result).toStrictEqual(expectedFee); + }); + + it("test-calculateLiquidityFee-should-return-correctly-fee-in-USDT", async () => { + // mock + jest.spyOn(poolHelper, "convertFeeAssetToUsdt").mockResolvedValue(1e6); + // act - const result = poolHelper.calculateFeeByAsset( + const liquidityFee = await poolHelper.calculateLiquidityFee( { - info: assetInfo, - amount: "1" + firstAssetInfo: "1", + secondAssetInfo: "1", + commissionRate: "1", + pairAddr: "orai1c5s03c3l336dgesne7dylnmhszw8554tsyy9yt", + liquidityAddr: "1", + oracleAddr: "1", + symbols: "1", + fromIconUrl: "1", + toIconUrl: "1", + volume24Hour: 1n, + apr: 1, + totalLiquidity: 1, + fee7Days: 1n }, - shareRatio + 13344890, + 1 ); - // assert - expect(result).toEqual(expectedResult); + + // assertion + expect(liquidityFee).toEqual(2000000n); }); it.each([ @@ -296,7 +319,7 @@ describe("test-pool-helper", () => { info: oraiInfo, amount: "1" }, - 0.5 + 2 ] ])("%s", async (_caseName: string, assetFee: Asset | null, expectedResult: number) => { // mock @@ -310,54 +333,56 @@ describe("test-pool-helper", () => { }); }); - it.each<[string, AssetInfo[], AssetInfo]>([ - [ - "case-asset-info-pairs-is-NOT-reversed-and-base-asset-NOT-ORAI", + describe("test-calculate-APR-pool", () => { + it.each<[string, AssetInfo[], AssetInfo]>([ [ + "case-asset-info-pairs-is-NOT-reversed-and-base-asset-NOT-ORAI", + [ + { + token: { + contract_addr: scAtomCw20Address + } + }, + { + native_token: { + denom: atomIbcDenom + } + } + ], { token: { contract_addr: scAtomCw20Address } - }, - { - native_token: { - denom: atomIbcDenom - } } ], - { - token: { - contract_addr: scAtomCw20Address - } - } - ], - ["case-asset-info-pairs-is-NOT-reversed-and-base-asset-is-ORAI", [oraiInfo, usdtInfo], usdtInfo], - [ - "case-asset-info-pairs-is-reversed-and-base-asset-NOT-ORAI", + ["case-asset-info-pairs-is-NOT-reversed-and-base-asset-is-ORAI", [oraiInfo, usdtInfo], usdtInfo], [ - { - native_token: { - denom: atomIbcDenom + "case-asset-info-pairs-is-reversed-and-base-asset-NOT-ORAI", + [ + { + native_token: { + denom: atomIbcDenom + } + }, + { + token: { + contract_addr: scAtomCw20Address + } } - }, + ], { token: { contract_addr: scAtomCw20Address } } ], - { - token: { - contract_addr: scAtomCw20Address - } + ["case-asset-info-pairs-is-reversed-and-base-asset-is-ORAI", [usdtInfo, oraiInfo], usdtInfo] + ])( + "test-getStakingAssetInfo-with-%p-should-return-correctly-staking-asset-info", + (_caseName: string, assetInfos: AssetInfo[], expectedStakingAssetInfo: AssetInfo) => { + const result = poolHelper.getStakingAssetInfo(assetInfos); + expect(result).toStrictEqual(expectedStakingAssetInfo); } - ], - ["case-asset-info-pairs-is-reversed-and-base-asset-is-ORAI", [usdtInfo, oraiInfo], usdtInfo] - ])( - "test-getStakingAssetInfo-with-%p-should-return-correctly-staking-asset-info", - (_caseName: string, assetInfos: AssetInfo[], expectedStakingAssetInfo: AssetInfo) => { - const result = poolHelper.getStakingAssetInfo(assetInfos); - expect(result).toStrictEqual(expectedStakingAssetInfo); - } - ); + ); + }); }); From 2b76c15d821c7f52df9e08c56382fe4b72678eaa Mon Sep 17 00:00:00 2001 From: trungbach Date: Thu, 7 Sep 2023 09:40:56 +0700 Subject: [PATCH 28/46] refactor: remove redunant code --- packages/oraidex-sync/src/index.ts | 2 +- packages/oraidex-sync/src/tx-parsing.ts | 20 +++++--------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index 26238ff5..bafd0387 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -44,7 +44,7 @@ class WriteOrders extends WriteData { const currentOffset = await this.duckDb.loadHeightSnapshot(); // edge case. If no new block has been found, then we skip processing to prevent duplication handling if (currentOffset === newOffset) return true; - let result = await parseTxs(txs, this.duckDb); + let result = await parseTxs(txs); // accumulate liquidity pool amount await this.accumulatePoolAmount([...result.provideLiquidityOpsData, ...result.withdrawLiquidityOpsData]); diff --git a/packages/oraidex-sync/src/tx-parsing.ts b/packages/oraidex-sync/src/tx-parsing.ts index c6d2d79c..e3bf611e 100644 --- a/packages/oraidex-sync/src/tx-parsing.ts +++ b/packages/oraidex-sync/src/tx-parsing.ts @@ -162,8 +162,7 @@ async function extractMsgProvideLiquidity( txData: BasicTxData, msg: MsgType, txCreator: string, - wasmAttributes: (readonly Attribute[])[], - duckDb: DuckDb + wasmAttributes: (readonly Attribute[])[] ): Promise { if ("provide_liquidity" in msg) { for (let attrs of wasmAttributes) { @@ -225,8 +224,7 @@ function parseWithdrawLiquidityAssets(assets: string): string[] { async function extractMsgWithdrawLiquidity( txData: BasicTxData, wasmAttributes: (readonly Attribute[])[], - txCreator: string, - duckDb: DuckDb + txCreator: string ): Promise { const withdrawData: WithdrawLiquidityOperationData[] = []; @@ -312,7 +310,7 @@ function parseExecuteContractToOraidexMsgs(msgs: MsgExecuteContractWithLogs[]): return objs; } -async function parseTxs(txs: Tx[], duckDb: DuckDb): Promise { +async function parseTxs(txs: Tx[]): Promise { let transactions: Tx[] = []; let swapOpsData: SwapOperationData[] = []; let accountTxs: AccountTx[] = []; @@ -332,17 +330,9 @@ async function parseTxs(txs: Tx[], duckDb: DuckDb): Promise { const sender = msg.sender; const wasmAttributes = parseWasmEvents(msg.logs.events); swapOpsData.push(...extractSwapOperations(basicTxData, wasmAttributes)); - const provideLiquidityData = await extractMsgProvideLiquidity( - basicTxData, - msg.msg, - sender, - wasmAttributes, - duckDb - ); + const provideLiquidityData = await extractMsgProvideLiquidity(basicTxData, msg.msg, sender, wasmAttributes); if (provideLiquidityData) provideLiquidityOpsData.push(provideLiquidityData); - withdrawLiquidityOpsData.push( - ...(await extractMsgWithdrawLiquidity(basicTxData, wasmAttributes, sender, duckDb)) - ); + withdrawLiquidityOpsData.push(...(await extractMsgWithdrawLiquidity(basicTxData, wasmAttributes, sender))); accountTxs.push({ txhash: basicTxData.txhash, accountAddress: sender }); } } From 0d068dafcb638b23f16625c01f45eb18e8c8d3fa Mon Sep 17 00:00:00 2001 From: trungbach Date: Thu, 7 Sep 2023 17:45:35 +0700 Subject: [PATCH 29/46] update: accumulated pool amount in pair_infos table to calculate price, lp faster --- packages/oraidex-server/src/index.ts | 1 + packages/oraidex-sync/src/db.ts | 16 +++ packages/oraidex-sync/src/helper.ts | 130 +++++++++++++++++++---- packages/oraidex-sync/src/index.ts | 42 ++++++-- packages/oraidex-sync/src/pool-helper.ts | 13 +-- packages/oraidex-sync/src/types.ts | 2 + 6 files changed, 162 insertions(+), 42 deletions(-) diff --git a/packages/oraidex-server/src/index.ts b/packages/oraidex-server/src/index.ts index c12c336e..65adc755 100644 --- a/packages/oraidex-server/src/index.ts +++ b/packages/oraidex-server/src/index.ts @@ -257,6 +257,7 @@ app.get("/v1/pools/", async (_req, res) => { }) ); } catch (error) { + console.log({ error }); res.status(500).send(error.message); } }); diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index 22ff152b..03ec9d76 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -148,6 +148,8 @@ export class DuckDb { apr DOUBLE, totalLiquidity UINT64, fee7Days UBIGINT, + offerPoolAmount UBIGINT, + askPoolAmount UBIGINT, PRIMARY KEY (pairAddr) )` ); } @@ -156,6 +158,19 @@ export class DuckDb { await this.insertBulkData(ops, "pair_infos", true); } + async updatePairInfoAmount(offerPoolAmount: bigint, askPoolAmount: bigint, pairAddr: string) { + console.log({ offerPoolAmount, askPoolAmount, pairAddr }); + await this.conn.all( + `UPDATE pair_infos + SET offerPoolAmount = ?, askPoolAmount = ? + WHERE pairAddr = ? + `, + Number(offerPoolAmount), + Number(askPoolAmount), + pairAddr + ); + } + async insertPriceInfos(ops: PriceInfo[]) { await this.insertBulkData(ops, "price_infos", false, `price_infos-${Math.random() * 1000}`); } @@ -503,6 +518,7 @@ export class DuckDb { console.dir({ nullForPair: assetInfos }, { depth: null }); return null; } + console.dir({ result }, { depth: null }); return { offerPoolAmount: BigInt(result[0].baseTokenReserve), diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index 72cd6fa5..ef8b2eeb 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -227,9 +227,10 @@ export function isAssetInfoPairReverse(assetInfos: AssetInfo[]): boolean { * @param poolInfos - pool info data for initial lp accumulation */ // TODO: write test cases for this function -export function collectAccumulateLpData( +export async function collectAccumulateLpData( data: ProvideLiquidityOperationData[] | WithdrawLiquidityOperationData[], - poolInfos: PoolResponse[] + poolInfos: PoolResponse[], + pairInfos: PairInfoData[] ) { let accumulateData: { [key: string]: { @@ -237,6 +238,7 @@ export function collectAccumulateLpData( quoteTokenAmount: bigint; }; } = {}; + const duckDb = DuckDb.instances; for (let op of data) { const pool = poolInfos.find( (info) => @@ -252,27 +254,119 @@ export function collectAccumulateLpData( baseAmount = -BigInt(op.baseTokenAmount); quoteAmount = -BigInt(op.quoteTokenAmount); } - const denom = `${op.baseTokenDenom}-${op.quoteTokenDenom}`; - if (!accumulateData[denom]) { + + let assetInfos = pool.assets.map((asset) => asset.info) as [AssetInfo, AssetInfo]; + if (isAssetInfoPairReverse(assetInfos)) assetInfos.reverse(); + const pairInfo = await duckDb.getPoolByAssetInfos(assetInfos); + const { pairAddr } = pairInfo; + if (!accumulateData[pairAddr]) { const initialFirstTokenAmount = parseInt( pool.assets.find((asset) => parseAssetInfoOnlyDenom(asset.info) === op.baseTokenDenom).amount ); const initialSecondTokenAmount = parseInt( pool.assets.find((asset) => parseAssetInfoOnlyDenom(asset.info) === op.quoteTokenDenom).amount ); - accumulateData[denom] = { + accumulateData[pairAddr] = { baseTokenAmount: BigInt(initialFirstTokenAmount) + baseAmount, quoteTokenAmount: BigInt(initialSecondTokenAmount) + quoteAmount }; - op.baseTokenReserve = accumulateData[denom].baseTokenAmount; - op.quoteTokenReserve = accumulateData[denom].quoteTokenAmount; - continue; + op.baseTokenReserve = accumulateData[pairAddr].baseTokenAmount; + op.quoteTokenReserve = accumulateData[pairAddr].quoteTokenAmount; + } else { + accumulateData[pairAddr].baseTokenAmount += baseAmount; + accumulateData[pairAddr].quoteTokenAmount += quoteAmount; + op.baseTokenReserve = accumulateData[pairAddr].baseTokenAmount; + op.quoteTokenReserve = accumulateData[pairAddr].quoteTokenAmount; } - accumulateData[denom].baseTokenAmount += baseAmount; - accumulateData[denom].quoteTokenAmount += quoteAmount; - op.baseTokenReserve = accumulateData[denom].baseTokenAmount; - op.quoteTokenReserve = accumulateData[denom].quoteTokenAmount; } + + // update new offer, ask pool amount to pair_infos + await Promise.all( + pairInfos + .map(({ pairAddr }) => { + if (accumulateData[pairAddr]) { + return duckDb.updatePairInfoAmount( + accumulateData[pairAddr].baseTokenAmount, + accumulateData[pairAddr].quoteTokenAmount, + pairAddr + ); + } + }) + .filter(Boolean) + ); +} + +/** + * This function will accumulate the lp amount and modify the parameter + * @param data - swap ops. This param will be mutated. + * @param poolInfos - pool info data for initial lp accumulation + */ +export async function collectAccumulateSwapData( + data: SwapOperationData[], + poolInfos: PoolResponse[], + pairInfos: PairInfoData[] +) { + let accumulateData: { + [key: string]: { + baseTokenAmount: bigint; + quoteTokenAmount: bigint; + }; + } = {}; + const duckDb = DuckDb.instances; + + for (let op of data) { + const pool = poolInfos.find( + (info) => + info.assets.some((assetInfo) => parseAssetInfoOnlyDenom(assetInfo.info) === op.offerDenom) && + info.assets.some((assetInfo) => parseAssetInfoOnlyDenom(assetInfo.info) === op.askDenom) + ); + if (!pool) continue; + + let baseAmount = BigInt(op.offerAmount); + let quoteAmount = -BigInt(op.returnAmount); + if (op.direction === "Sell") { + // reverse sign since sell means lp base decrease, quote increase + baseAmount = -BigInt(op.offerAmount); + quoteAmount = BigInt(op.returnAmount); + } + + let assetInfos = pool.assets.map((asset) => asset.info) as [AssetInfo, AssetInfo]; + if (isAssetInfoPairReverse(assetInfos)) assetInfos.reverse(); + const pairInfo = await duckDb.getPoolByAssetInfos(assetInfos); + const { pairAddr } = pairInfo; + if (!accumulateData[pairAddr]) { + let initialFirstTokenAmount = parseInt( + pool.assets.find((asset) => parseAssetInfoOnlyDenom(asset.info) === op.offerDenom).amount + ); + let initialSecondTokenAmount = parseInt( + pool.assets.find((asset) => parseAssetInfoOnlyDenom(asset.info) === op.askDenom).amount + ); + if (op.direction === "Buy") { + [initialFirstTokenAmount, initialSecondTokenAmount] = [initialSecondTokenAmount, initialFirstTokenAmount]; + } + accumulateData[pairAddr] = { + baseTokenAmount: BigInt(initialFirstTokenAmount) + baseAmount, + quoteTokenAmount: BigInt(initialSecondTokenAmount) + quoteAmount + }; + } else { + accumulateData[pairAddr].baseTokenAmount += baseAmount; + accumulateData[pairAddr].quoteTokenAmount += quoteAmount; + } + } + // update new offer, ask pool amount to pair_infos + await Promise.all( + pairInfos + .map(({ pairAddr }) => { + if (accumulateData[pairAddr]) { + return duckDb.updatePairInfoAmount( + accumulateData[pairAddr].baseTokenAmount, + accumulateData[pairAddr].quoteTokenAmount, + pairAddr + ); + } + }) + .filter(Boolean) + ); } export function removeOpsDuplication(ops: OraiDexType[]): OraiDexType[] { @@ -402,18 +496,12 @@ async function fetchPoolInfoAmount(fromInfo: AssetInfo, toInfo: AssetInfo, pairA // get liquidity of pair from assetInfos async function getPairLiquidity(assetInfos: [AssetInfo, AssetInfo]): Promise { const duckDb = DuckDb.instances; + // get info of pool in pair_infos, ask & offer are accumulated in sync process (via swap ops and lp ops). const poolInfo = await duckDb.getPoolByAssetInfos(assetInfos); - if (!poolInfo) throw new Error(`Cannot found pool info when get pair liquidity: ${JSON.stringify(assetInfos)}`); - - // get info of last tx in lp_ops_data, if not have data => get info from contract - let poolAmounts = - (await duckDb.getPoolAmountFromAssetInfos(assetInfos)) ?? - (await fetchPoolInfoAmount(...assetInfos, poolInfo.pairAddr)); - if (!poolAmounts) throw new Error(` Cannot found pool amount: ${JSON.stringify(assetInfos)}`); - + if (!poolInfo.askPoolAmount || !poolInfo.offerPoolAmount) return 0; const baseAssetInfo = assetInfos[0]; const priceBaseAssetInUsdt = await getPriceAssetByUsdt(baseAssetInfo); - return priceBaseAssetInUsdt * Number(poolAmounts.offerPoolAmount) * 2; + return priceBaseAssetInUsdt * Number(poolInfo.offerPoolAmount) * 2; } /** diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index bafd0387..d113a13e 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -1,7 +1,7 @@ import { SyncData, Txs, WriteData } from "@oraichain/cosmos-rpc-sync"; import "dotenv/config"; import { DuckDb } from "./db"; -import { collectAccumulateLpData, getSymbolFromAsset } from "./helper"; +import { collectAccumulateLpData, collectAccumulateSwapData, getSymbolFromAsset } from "./helper"; import { getAllPairInfos, getPoolInfos } from "./pool-helper"; import { parseAssetInfo, parseTxs } from "./tx-parsing"; import { @@ -9,6 +9,7 @@ import { InitialData, PairInfoData, ProvideLiquidityOperationData, + SwapOperationData, TxAnlysisResult, WithdrawLiquidityOperationData } from "./types"; @@ -28,14 +29,25 @@ class WriteOrders extends WriteData { await this.duckDb.insertLpOps(txs.withdrawLiquidityOpsData); } - private async accumulatePoolAmount(data: ProvideLiquidityOperationData[] | WithdrawLiquidityOperationData[]) { - if (data.length === 0) return; // guard. If theres no data then we wont process anything + private async accumulatePoolAmount( + data: ProvideLiquidityOperationData[] | WithdrawLiquidityOperationData[], + swapData: SwapOperationData[] + ) { const pairInfos = await this.duckDb.queryPairInfos(); - const poolInfos = await getPoolInfos( - pairInfos.map((pair) => pair.pairAddr), - data[0].txheight // assume data is sorted by height and timestamp - ); - collectAccumulateLpData(data, poolInfos); + if (data.length > 0) { + const poolInfos = await getPoolInfos( + pairInfos.map((pair) => pair.pairAddr), + data[0].txheight // assume data is sorted by height and timestamp + ); + await collectAccumulateLpData(data, poolInfos, pairInfos); + } + if (swapData.length > 0) { + const poolInfos = await getPoolInfos( + pairInfos.map((pair) => pair.pairAddr), + swapData[0].txheight // assume data is sorted by height and timestamp + ); + await collectAccumulateSwapData(swapData, poolInfos, pairInfos); + } } async process(chunk: any): Promise { @@ -46,8 +58,11 @@ class WriteOrders extends WriteData { if (currentOffset === newOffset) return true; let result = await parseTxs(txs); - // accumulate liquidity pool amount - await this.accumulatePoolAmount([...result.provideLiquidityOpsData, ...result.withdrawLiquidityOpsData]); + // accumulate liquidity pool amount via provide/withdraw liquidity and swap ops + await this.accumulatePoolAmount( + [...result.provideLiquidityOpsData, ...result.withdrawLiquidityOpsData], + [...result.swapOpsData] + ); // collect the latest offer & ask volume to accumulate the results // insert txs @@ -79,6 +94,8 @@ class OraiDexSync { console.time("timer-updateLatestPairInfos"); const pairInfos = await getAllPairInfos(); + const allPools = await this.duckDb.getPools(); + if (allPools.length > 0) return; await this.duckDb.insertPairInfos( pairInfos.map((pair) => { const symbols = getSymbolFromAsset(pair.asset_infos); @@ -95,10 +112,13 @@ class OraiDexSync { volume24Hour: 0n, apr: 0, totalLiquidity: 0, - fee7Days: 0n + fee7Days: 0n, + offerPoolAmount: 0n, + askPoolAmount: 0n } as PairInfoData; }) ); + console.timeEnd("timer-updateLatestPairInfos"); } catch (error) { console.log("error in updateLatestPairInfos: ", error); diff --git a/packages/oraidex-sync/src/pool-helper.ts b/packages/oraidex-sync/src/pool-helper.ts index a3e36bb0..27f4ada7 100644 --- a/packages/oraidex-sync/src/pool-helper.ts +++ b/packages/oraidex-sync/src/pool-helper.ts @@ -85,19 +85,12 @@ export const getPriceByAsset = async ( ): Promise => { const duckDb = DuckDb.instances; const poolInfo = await duckDb.getPoolByAssetInfos(assetInfos); - if (!poolInfo) throw new Error(`Cannot found pool info: ${JSON.stringify(assetInfos)}`); - - // get info of latest tx in lp_ops_data table, if lp_ops_data not have data yet => get info from contract - let poolAmounts = - (await duckDb.getPoolAmountFromAssetInfos(assetInfos)) ?? - (await fetchPoolInfoAmount(...assetInfos, poolInfo.pairAddr)); - if (!poolAmounts) throw new Error(` Cannot found pool amount: ${JSON.stringify(assetInfos)}`); - + if (!poolInfo.askPoolAmount || !poolInfo.offerPoolAmount) return 0; // offer: orai, ask: usdt -> price offer in ask = calculatePriceByPool([ask, offer]) // offer: orai, ask: atom -> price ask in offer = calculatePriceByPool([offer, ask]) const basePrice = calculatePriceByPool( - BigInt(poolAmounts.askPoolAmount), - BigInt(poolAmounts.offerPoolAmount), + BigInt(poolInfo.askPoolAmount), + BigInt(poolInfo.offerPoolAmount), +poolInfo.commissionRate ); return ratioDirection === "base_in_quote" ? basePrice : 1 / basePrice; diff --git a/packages/oraidex-sync/src/types.ts b/packages/oraidex-sync/src/types.ts index b5c0c47c..9250182a 100644 --- a/packages/oraidex-sync/src/types.ts +++ b/packages/oraidex-sync/src/types.ts @@ -47,6 +47,8 @@ export type PairInfoData = { volume24Hour: bigint; apr: number; totalLiquidity: number; + offerPoolAmount: bigint; + askPoolAmount: bigint; fee7Days: bigint; }; From 7e75c48991369b3da43f6b4e63723d77a574cd72 Mon Sep 17 00:00:00 2001 From: trungbach Date: Thu, 7 Sep 2023 18:43:41 +0700 Subject: [PATCH 30/46] update: refactor accumulate lp amount --- packages/oraidex-sync/src/helper.ts | 93 +++++++++++++++++++++++++---- packages/oraidex-sync/src/index.ts | 73 ++++++++++++++++------ packages/oraidex-sync/src/types.ts | 9 +++ 3 files changed, 145 insertions(+), 30 deletions(-) diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index ef8b2eeb..a2d46f6c 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -12,6 +12,7 @@ import { DuckDb } from "./db"; import { pairs, pairsOnlyDenom } from "./pairs"; import { getPriceAssetByUsdt } from "./pool-helper"; import { + LpOpsData, Ohlcv, OraiDexType, PairInfoData, @@ -108,8 +109,6 @@ export function replaceAllNonAlphaBetChar(columnName: string): string { } function parseAssetInfo(info: AssetInfo): string { - // if ("native_token" in info) return info.native_token.denom; - // return info.token.contract_addr; return JSON.stringify(info); } @@ -251,8 +250,8 @@ export async function collectAccumulateLpData( let quoteAmount = BigInt(op.quoteTokenAmount); if (op.opType === "withdraw") { // reverse sign since withdraw means lp decreases - baseAmount = -BigInt(op.baseTokenAmount); - quoteAmount = -BigInt(op.quoteTokenAmount); + baseAmount = -baseAmount; + quoteAmount = -quoteAmount; } let assetInfos = pool.assets.map((asset) => asset.info) as [AssetInfo, AssetInfo]; @@ -270,16 +269,89 @@ export async function collectAccumulateLpData( baseTokenAmount: BigInt(initialFirstTokenAmount) + baseAmount, quoteTokenAmount: BigInt(initialSecondTokenAmount) + quoteAmount }; - op.baseTokenReserve = accumulateData[pairAddr].baseTokenAmount; - op.quoteTokenReserve = accumulateData[pairAddr].quoteTokenAmount; } else { accumulateData[pairAddr].baseTokenAmount += baseAmount; accumulateData[pairAddr].quoteTokenAmount += quoteAmount; - op.baseTokenReserve = accumulateData[pairAddr].baseTokenAmount; - op.quoteTokenReserve = accumulateData[pairAddr].quoteTokenAmount; } } + // update new offer, ask pool amount to pair_infos + await Promise.all( + pairInfos + .map(({ pairAddr }) => { + if (accumulateData[pairAddr]) { + return duckDb.updatePairInfoAmount( + accumulateData[pairAddr].baseTokenAmount, + accumulateData[pairAddr].quoteTokenAmount, + pairAddr + ); + } + }) + .filter(Boolean) + ); +} + +/** + * This function will accumulate the lp amount and modify the parameter + * @param data - lp ops & swap ops. + * @param poolInfos - pool info data for initial lp accumulation + * @param pairInfos - pool info data from db + */ +// TODO: write test cases for this function +export async function collectAccumulateLpAndSwapData(data: LpOpsData[], pairInfos: PairInfoData[]) { + let accumulateData: { + [key: string]: { + baseTokenAmount: bigint; + quoteTokenAmount: bigint; + }; + } = {}; + const duckDb = DuckDb.instances; + for (let op of data) { + let baseAmount = BigInt(op.baseTokenAmount); + let quoteAmount = BigInt(op.quoteTokenAmount); + if (op.opType === "withdraw" || op.direction === "Sell") { + // reverse sign since withdraw means lp decreases + baseAmount = -baseAmount; + quoteAmount = -quoteAmount; + } + + const pairMapping = pairs.find( + (pair) => + pair.asset_infos.some((assetInfo) => parseAssetInfoOnlyDenom(assetInfo) === op.baseTokenDenom) && + pair.asset_infos.some((assetInfo) => parseAssetInfoOnlyDenom(assetInfo) === op.quoteTokenDenom) + ); + const pairInfo = await duckDb.getPoolByAssetInfos(pairMapping.asset_infos); + if (!pairInfo) continue; + const { pairAddr } = pairInfo; + if (!accumulateData[pairAddr]) { + // let initialFirstTokenAmount = BigInt(pairInfo.offerPoolAmount); + // let initialSecondTokenAmount = BigInt(pairInfo.askPoolAmount); + // if (op.direction === "Buy") { + // [initialFirstTokenAmount, initialSecondTokenAmount] = [initialSecondTokenAmount, initialFirstTokenAmount]; + // } + + // accumulateData[pairAddr] = { + // baseTokenAmount: initialFirstTokenAmount + baseAmount, + // quoteTokenAmount: initialSecondTokenAmount + quoteAmount + // }; + let initialFirstTokenAmount = parseInt( + pool.assets.find((asset) => parseAssetInfoOnlyDenom(asset.info) === op.offerDenom).amount + ); + let initialSecondTokenAmount = parseInt( + pool.assets.find((asset) => parseAssetInfoOnlyDenom(asset.info) === op.askDenom).amount + ); + if (op.direction === "Buy") { + [initialFirstTokenAmount, initialSecondTokenAmount] = [initialSecondTokenAmount, initialFirstTokenAmount]; + } + accumulateData[pairAddr] = { + baseTokenAmount: BigInt(initialFirstTokenAmount) + baseAmount, + quoteTokenAmount: BigInt(initialSecondTokenAmount) + quoteAmount + }; + } else { + accumulateData[pairAddr].baseTokenAmount += baseAmount; + accumulateData[pairAddr].quoteTokenAmount += quoteAmount; + } + } // update new offer, ask pool amount to pair_infos await Promise.all( pairInfos @@ -313,7 +385,6 @@ export async function collectAccumulateSwapData( }; } = {}; const duckDb = DuckDb.instances; - for (let op of data) { const pool = poolInfos.find( (info) => @@ -326,8 +397,8 @@ export async function collectAccumulateSwapData( let quoteAmount = -BigInt(op.returnAmount); if (op.direction === "Sell") { // reverse sign since sell means lp base decrease, quote increase - baseAmount = -BigInt(op.offerAmount); - quoteAmount = BigInt(op.returnAmount); + baseAmount = -baseAmount; + quoteAmount = -quoteAmount; } let assetInfos = pool.assets.map((asset) => asset.info) as [AssetInfo, AssetInfo]; diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index d113a13e..eea47091 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -1,12 +1,18 @@ import { SyncData, Txs, WriteData } from "@oraichain/cosmos-rpc-sync"; import "dotenv/config"; import { DuckDb } from "./db"; -import { collectAccumulateLpData, collectAccumulateSwapData, getSymbolFromAsset } from "./helper"; +import { + collectAccumulateLpAndSwapData, + collectAccumulateLpData, + collectAccumulateSwapData, + getSymbolFromAsset +} from "./helper"; import { getAllPairInfos, getPoolInfos } from "./pool-helper"; import { parseAssetInfo, parseTxs } from "./tx-parsing"; import { Env, InitialData, + LpOpsData, PairInfoData, ProvideLiquidityOperationData, SwapOperationData, @@ -34,19 +40,44 @@ class WriteOrders extends WriteData { swapData: SwapOperationData[] ) { const pairInfos = await this.duckDb.queryPairInfos(); - if (data.length > 0) { - const poolInfos = await getPoolInfos( - pairInfos.map((pair) => pair.pairAddr), - data[0].txheight // assume data is sorted by height and timestamp - ); - await collectAccumulateLpData(data, poolInfos, pairInfos); - } - if (swapData.length > 0) { - const poolInfos = await getPoolInfos( - pairInfos.map((pair) => pair.pairAddr), - swapData[0].txheight // assume data is sorted by height and timestamp - ); - await collectAccumulateSwapData(swapData, poolInfos, pairInfos); + // if (data.length > 0) { + // const poolInfos = await getPoolInfos( + // pairInfos.map((pair) => pair.pairAddr), + // data[0].txheight // assume data is sorted by height and timestamp + // ); + // await collectAccumulateLpData(data, poolInfos, pairInfos); + // } + // if (swapData.length > 0) { + // const poolInfos = await getPoolInfos( + // pairInfos.map((pair) => pair.pairAddr), + // swapData[0].txheight // assume data is sorted by height and timestamp + // ); + // await collectAccumulateSwapData(swapData, poolInfos, pairInfos); + // } + if (data.length > 0 || swapData.length > 0) { + const lpOpsData: LpOpsData[] = [ + ...data.map((item) => { + return { + baseTokenAmount: item.baseTokenAmount, + baseTokenDenom: item.baseTokenDenom, + quoteTokenAmount: item.quoteTokenAmount, + quoteTokenDenom: item.quoteTokenDenom, + opType: item.opType, + direction: null + } as LpOpsData; + }), + ...swapData.map((item) => { + return { + baseTokenAmount: item.offerAmount, + baseTokenDenom: item.offerDenom, + quoteTokenAmount: -item.returnAmount, + quoteTokenDenom: item.askDenom, + opType: null, + direction: item.direction + } as LpOpsData; + }) + ]; + await collectAccumulateLpAndSwapData(lpOpsData, pairInfos); } } @@ -89,15 +120,19 @@ class OraiDexSync { return new OraiDexSync(duckDb, rpcUrl, env); } - private async updateLatestPairInfos() { + private async updateLatestPairInfos(currentHeight: number) { try { console.time("timer-updateLatestPairInfos"); const pairInfos = await getAllPairInfos(); const allPools = await this.duckDb.getPools(); if (allPools.length > 0) return; + const poolInfos = await getPoolInfos( + pairInfos.map((pair) => pair.contract_addr), + currentHeight + ); await this.duckDb.insertPairInfos( - pairInfos.map((pair) => { + pairInfos.map((pair, index) => { const symbols = getSymbolFromAsset(pair.asset_infos); return { firstAssetInfo: parseAssetInfo(pair.asset_infos[0]), @@ -113,8 +148,8 @@ class OraiDexSync { apr: 0, totalLiquidity: 0, fee7Days: 0n, - offerPoolAmount: 0n, - askPoolAmount: 0n + offerPoolAmount: BigInt(poolInfos[index].assets[0].amount), + askPoolAmount: BigInt(poolInfos[index].assets[1].amount) } as PairInfoData; }) ); @@ -142,7 +177,7 @@ class OraiDexSync { currentInd = initialSyncHeight; } console.log("current ind: ", currentInd); - await this.updateLatestPairInfos(); + await this.updateLatestPairInfos(currentInd); new SyncData({ offset: currentInd, rpcUrl: this.rpcUrl, diff --git a/packages/oraidex-sync/src/types.ts b/packages/oraidex-sync/src/types.ts index 9250182a..4056155a 100644 --- a/packages/oraidex-sync/src/types.ts +++ b/packages/oraidex-sync/src/types.ts @@ -84,6 +84,15 @@ export type WithdrawLiquidityOperationData = ProvideLiquidityOperationData; export type OraiDexType = SwapOperationData | ProvideLiquidityOperationData | WithdrawLiquidityOperationData | Ohlcv; +export type LpOpsData = { + baseTokenAmount: number; + baseTokenDenom: string; // eg: orai, orai1234... + quoteTokenAmount: number; + quoteTokenDenom: string; + opType?: LiquidityOpType; + direction?: SwapDirection; +}; + export type TxAnlysisResult = { // transactions: Tx[]; swapOpsData: SwapOperationData[]; From 231ebc0b8b3500c236a9c65bfb20b65d2cfdc14b Mon Sep 17 00:00:00 2001 From: trungbach Date: Fri, 8 Sep 2023 11:44:44 +0700 Subject: [PATCH 31/46] refactor: finished accumulate lp pool via swap and lp trans --- packages/oraidex-sync/src/helper.ts | 167 ++-------------------------- packages/oraidex-sync/src/index.ts | 84 +++++++------- 2 files changed, 49 insertions(+), 202 deletions(-) diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index a2d46f6c..acb510df 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -221,13 +221,14 @@ export function isAssetInfoPairReverse(assetInfos: AssetInfo[]): boolean { } /** - * This function will accumulate the lp amount and modify the parameter - * @param data - lp ops. This param will be mutated. + * This function will accumulate the lp amount + * @param data - lp ops & swap ops. * @param poolInfos - pool info data for initial lp accumulation + * @param pairInfos - pool info data from db */ // TODO: write test cases for this function -export async function collectAccumulateLpData( - data: ProvideLiquidityOperationData[] | WithdrawLiquidityOperationData[], +export async function collectAccumulateLpAndSwapData( + data: LpOpsData[], poolInfos: PoolResponse[], pairInfos: PairInfoData[] ) { @@ -248,7 +249,7 @@ export async function collectAccumulateLpData( let baseAmount = BigInt(op.baseTokenAmount); let quoteAmount = BigInt(op.quoteTokenAmount); - if (op.opType === "withdraw") { + if (op.opType === "withdraw" || op.direction === "Buy") { // reverse sign since withdraw means lp decreases baseAmount = -baseAmount; quoteAmount = -quoteAmount; @@ -258,163 +259,17 @@ export async function collectAccumulateLpData( if (isAssetInfoPairReverse(assetInfos)) assetInfos.reverse(); const pairInfo = await duckDb.getPoolByAssetInfos(assetInfos); const { pairAddr } = pairInfo; + if (!accumulateData[pairAddr]) { const initialFirstTokenAmount = parseInt( - pool.assets.find((asset) => parseAssetInfoOnlyDenom(asset.info) === op.baseTokenDenom).amount + pool.assets.find((asset) => parseAssetInfoOnlyDenom(asset.info) === parseAssetInfoOnlyDenom(assetInfos[0])) + .amount ); const initialSecondTokenAmount = parseInt( - pool.assets.find((asset) => parseAssetInfoOnlyDenom(asset.info) === op.quoteTokenDenom).amount + pool.assets.find((asset) => parseAssetInfoOnlyDenom(asset.info) === parseAssetInfoOnlyDenom(assetInfos[1])) + .amount ); - accumulateData[pairAddr] = { - baseTokenAmount: BigInt(initialFirstTokenAmount) + baseAmount, - quoteTokenAmount: BigInt(initialSecondTokenAmount) + quoteAmount - }; - } else { - accumulateData[pairAddr].baseTokenAmount += baseAmount; - accumulateData[pairAddr].quoteTokenAmount += quoteAmount; - } - } - // update new offer, ask pool amount to pair_infos - await Promise.all( - pairInfos - .map(({ pairAddr }) => { - if (accumulateData[pairAddr]) { - return duckDb.updatePairInfoAmount( - accumulateData[pairAddr].baseTokenAmount, - accumulateData[pairAddr].quoteTokenAmount, - pairAddr - ); - } - }) - .filter(Boolean) - ); -} -/** - * This function will accumulate the lp amount and modify the parameter - * @param data - lp ops & swap ops. - * @param poolInfos - pool info data for initial lp accumulation - * @param pairInfos - pool info data from db - */ -// TODO: write test cases for this function -export async function collectAccumulateLpAndSwapData(data: LpOpsData[], pairInfos: PairInfoData[]) { - let accumulateData: { - [key: string]: { - baseTokenAmount: bigint; - quoteTokenAmount: bigint; - }; - } = {}; - const duckDb = DuckDb.instances; - for (let op of data) { - let baseAmount = BigInt(op.baseTokenAmount); - let quoteAmount = BigInt(op.quoteTokenAmount); - if (op.opType === "withdraw" || op.direction === "Sell") { - // reverse sign since withdraw means lp decreases - baseAmount = -baseAmount; - quoteAmount = -quoteAmount; - } - - const pairMapping = pairs.find( - (pair) => - pair.asset_infos.some((assetInfo) => parseAssetInfoOnlyDenom(assetInfo) === op.baseTokenDenom) && - pair.asset_infos.some((assetInfo) => parseAssetInfoOnlyDenom(assetInfo) === op.quoteTokenDenom) - ); - const pairInfo = await duckDb.getPoolByAssetInfos(pairMapping.asset_infos); - if (!pairInfo) continue; - - const { pairAddr } = pairInfo; - if (!accumulateData[pairAddr]) { - // let initialFirstTokenAmount = BigInt(pairInfo.offerPoolAmount); - // let initialSecondTokenAmount = BigInt(pairInfo.askPoolAmount); - // if (op.direction === "Buy") { - // [initialFirstTokenAmount, initialSecondTokenAmount] = [initialSecondTokenAmount, initialFirstTokenAmount]; - // } - - // accumulateData[pairAddr] = { - // baseTokenAmount: initialFirstTokenAmount + baseAmount, - // quoteTokenAmount: initialSecondTokenAmount + quoteAmount - // }; - let initialFirstTokenAmount = parseInt( - pool.assets.find((asset) => parseAssetInfoOnlyDenom(asset.info) === op.offerDenom).amount - ); - let initialSecondTokenAmount = parseInt( - pool.assets.find((asset) => parseAssetInfoOnlyDenom(asset.info) === op.askDenom).amount - ); - if (op.direction === "Buy") { - [initialFirstTokenAmount, initialSecondTokenAmount] = [initialSecondTokenAmount, initialFirstTokenAmount]; - } - accumulateData[pairAddr] = { - baseTokenAmount: BigInt(initialFirstTokenAmount) + baseAmount, - quoteTokenAmount: BigInt(initialSecondTokenAmount) + quoteAmount - }; - } else { - accumulateData[pairAddr].baseTokenAmount += baseAmount; - accumulateData[pairAddr].quoteTokenAmount += quoteAmount; - } - } - // update new offer, ask pool amount to pair_infos - await Promise.all( - pairInfos - .map(({ pairAddr }) => { - if (accumulateData[pairAddr]) { - return duckDb.updatePairInfoAmount( - accumulateData[pairAddr].baseTokenAmount, - accumulateData[pairAddr].quoteTokenAmount, - pairAddr - ); - } - }) - .filter(Boolean) - ); -} - -/** - * This function will accumulate the lp amount and modify the parameter - * @param data - swap ops. This param will be mutated. - * @param poolInfos - pool info data for initial lp accumulation - */ -export async function collectAccumulateSwapData( - data: SwapOperationData[], - poolInfos: PoolResponse[], - pairInfos: PairInfoData[] -) { - let accumulateData: { - [key: string]: { - baseTokenAmount: bigint; - quoteTokenAmount: bigint; - }; - } = {}; - const duckDb = DuckDb.instances; - for (let op of data) { - const pool = poolInfos.find( - (info) => - info.assets.some((assetInfo) => parseAssetInfoOnlyDenom(assetInfo.info) === op.offerDenom) && - info.assets.some((assetInfo) => parseAssetInfoOnlyDenom(assetInfo.info) === op.askDenom) - ); - if (!pool) continue; - - let baseAmount = BigInt(op.offerAmount); - let quoteAmount = -BigInt(op.returnAmount); - if (op.direction === "Sell") { - // reverse sign since sell means lp base decrease, quote increase - baseAmount = -baseAmount; - quoteAmount = -quoteAmount; - } - - let assetInfos = pool.assets.map((asset) => asset.info) as [AssetInfo, AssetInfo]; - if (isAssetInfoPairReverse(assetInfos)) assetInfos.reverse(); - const pairInfo = await duckDb.getPoolByAssetInfos(assetInfos); - const { pairAddr } = pairInfo; - if (!accumulateData[pairAddr]) { - let initialFirstTokenAmount = parseInt( - pool.assets.find((asset) => parseAssetInfoOnlyDenom(asset.info) === op.offerDenom).amount - ); - let initialSecondTokenAmount = parseInt( - pool.assets.find((asset) => parseAssetInfoOnlyDenom(asset.info) === op.askDenom).amount - ); - if (op.direction === "Buy") { - [initialFirstTokenAmount, initialSecondTokenAmount] = [initialSecondTokenAmount, initialFirstTokenAmount]; - } accumulateData[pairAddr] = { baseTokenAmount: BigInt(initialFirstTokenAmount) + baseAmount, quoteTokenAmount: BigInt(initialSecondTokenAmount) + quoteAmount diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index eea47091..fe48bd79 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -1,12 +1,7 @@ import { SyncData, Txs, WriteData } from "@oraichain/cosmos-rpc-sync"; import "dotenv/config"; import { DuckDb } from "./db"; -import { - collectAccumulateLpAndSwapData, - collectAccumulateLpData, - collectAccumulateSwapData, - getSymbolFromAsset -} from "./helper"; +import { collectAccumulateLpAndSwapData, getSymbolFromAsset } from "./helper"; import { getAllPairInfos, getPoolInfos } from "./pool-helper"; import { parseAssetInfo, parseTxs } from "./tx-parsing"; import { @@ -36,49 +31,46 @@ class WriteOrders extends WriteData { } private async accumulatePoolAmount( - data: ProvideLiquidityOperationData[] | WithdrawLiquidityOperationData[], + lpData: ProvideLiquidityOperationData[] | WithdrawLiquidityOperationData[], swapData: SwapOperationData[] ) { + if (lpData.length === 0 && swapData.length === 0) return; + const pairInfos = await this.duckDb.queryPairInfos(); - // if (data.length > 0) { - // const poolInfos = await getPoolInfos( - // pairInfos.map((pair) => pair.pairAddr), - // data[0].txheight // assume data is sorted by height and timestamp - // ); - // await collectAccumulateLpData(data, poolInfos, pairInfos); - // } - // if (swapData.length > 0) { - // const poolInfos = await getPoolInfos( - // pairInfos.map((pair) => pair.pairAddr), - // swapData[0].txheight // assume data is sorted by height and timestamp - // ); - // await collectAccumulateSwapData(swapData, poolInfos, pairInfos); - // } - if (data.length > 0 || swapData.length > 0) { - const lpOpsData: LpOpsData[] = [ - ...data.map((item) => { - return { - baseTokenAmount: item.baseTokenAmount, - baseTokenDenom: item.baseTokenDenom, - quoteTokenAmount: item.quoteTokenAmount, - quoteTokenDenom: item.quoteTokenDenom, - opType: item.opType, - direction: null - } as LpOpsData; - }), - ...swapData.map((item) => { - return { - baseTokenAmount: item.offerAmount, - baseTokenDenom: item.offerDenom, - quoteTokenAmount: -item.returnAmount, - quoteTokenDenom: item.askDenom, - opType: null, - direction: item.direction - } as LpOpsData; - }) - ]; - await collectAccumulateLpAndSwapData(lpOpsData, pairInfos); - } + const minSwapTxHeight = swapData[0]?.txheight; + const minLpTxHeight = lpData[0]?.txheight; + let minTxHeight; + if (minSwapTxHeight && minLpTxHeight) { + minTxHeight = Math.min(minSwapTxHeight, minLpTxHeight); + } else minTxHeight = minSwapTxHeight ?? minLpTxHeight; + + const poolInfos = await getPoolInfos( + pairInfos.map((pair) => pair.pairAddr), + minTxHeight // assume data is sorted by height and timestamp + ); + const lpOpsData: LpOpsData[] = [ + ...lpData.map((item) => { + return { + baseTokenAmount: item.baseTokenAmount, + baseTokenDenom: item.baseTokenDenom, + quoteTokenAmount: item.quoteTokenAmount, + quoteTokenDenom: item.quoteTokenDenom, + opType: item.opType, + direction: null + } as LpOpsData; + }), + ...swapData.map((item) => { + return { + baseTokenAmount: item.offerAmount, + baseTokenDenom: item.offerDenom, + quoteTokenAmount: -item.returnAmount, // reverse sign because we assume first case is sell, check buy later. + quoteTokenDenom: item.askDenom, + opType: null, + direction: item.direction + } as LpOpsData; + }) + ]; + await collectAccumulateLpAndSwapData(lpOpsData, poolInfos, pairInfos); } async process(chunk: any): Promise { From 23c7e9888db1d54e45857c7422b0f6711de2e498 Mon Sep 17 00:00:00 2001 From: trungbach Date: Fri, 8 Sep 2023 17:22:29 +0700 Subject: [PATCH 32/46] test: added full testcase for helper --- packages/oraidex-sync/src/helper.ts | 90 ++-- packages/oraidex-sync/src/index.ts | 21 +- packages/oraidex-sync/tests/helper.spec.ts | 409 +++++++++++++++--- .../oraidex-sync/tests/pool-helper.spec.ts | 16 +- 4 files changed, 404 insertions(+), 132 deletions(-) diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index acb510df..ac46e907 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -11,17 +11,7 @@ import { ORAI, atomic, tenAmountInDecimalSix, truncDecimals, usdtCw20Address } f import { DuckDb } from "./db"; import { pairs, pairsOnlyDenom } from "./pairs"; import { getPriceAssetByUsdt } from "./pool-helper"; -import { - LpOpsData, - Ohlcv, - OraiDexType, - PairInfoData, - PoolInfo, - ProvideLiquidityOperationData, - SwapDirection, - SwapOperationData, - WithdrawLiquidityOperationData -} from "./types"; +import { LpOpsData, Ohlcv, OraiDexType, PairInfoData, PoolInfo, SwapDirection, SwapOperationData } from "./types"; export function toObject(data: any) { return JSON.parse( @@ -84,9 +74,9 @@ export function concatDataToUniqueKey(data: { return `${data.txheight}-${data.firstDenom}-${data.firstAmount}-${data.secondDenom}-${data.secondAmount}`; } -export function concatOhlcvToUniqueKey(data: { timestamp: number; pair: string; volume: bigint }): string { +export const concatOhlcvToUniqueKey = (data: { timestamp: number; pair: string; volume: bigint }): string => { return `${data.timestamp}-${data.pair}-${data.volume.toString()}`; -} +}; export function isoToTimestampNumber(time: string) { return Math.floor(new Date(time).getTime() / 1000); @@ -227,11 +217,7 @@ export function isAssetInfoPairReverse(assetInfos: AssetInfo[]): boolean { * @param pairInfos - pool info data from db */ // TODO: write test cases for this function -export async function collectAccumulateLpAndSwapData( - data: LpOpsData[], - poolInfos: PoolResponse[], - pairInfos: PairInfoData[] -) { +export const collectAccumulateLpAndSwapData = async (data: LpOpsData[], poolInfos: PoolResponse[]) => { let accumulateData: { [key: string]: { baseTokenAmount: bigint; @@ -279,21 +265,9 @@ export async function collectAccumulateLpAndSwapData( accumulateData[pairAddr].quoteTokenAmount += quoteAmount; } } - // update new offer, ask pool amount to pair_infos - await Promise.all( - pairInfos - .map(({ pairAddr }) => { - if (accumulateData[pairAddr]) { - return duckDb.updatePairInfoAmount( - accumulateData[pairAddr].baseTokenAmount, - accumulateData[pairAddr].quoteTokenAmount, - pairAddr - ); - } - }) - .filter(Boolean) - ); -} + + return accumulateData; +}; export function removeOpsDuplication(ops: OraiDexType[]): OraiDexType[] { let newOps: OraiDexType[] = []; @@ -308,7 +282,7 @@ export function removeOpsDuplication(ops: OraiDexType[]): OraiDexType[] { * @param swapOps * @returns */ -export function groupSwapOpsByPair(ops: SwapOperationData[]): { [key: string]: SwapOperationData[] } { +export const groupSwapOpsByPair = (ops: SwapOperationData[]): { [key: string]: SwapOperationData[] } => { let opsByPair = {}; for (const op of ops) { const pairIndex = findPairIndexFromDenoms(op.offerDenom, op.askDenom); @@ -321,9 +295,9 @@ export function groupSwapOpsByPair(ops: SwapOperationData[]): { [key: string]: S opsByPair[pair].push(op); } return opsByPair; -} +}; -export function calculateSwapOhlcv(ops: SwapOperationData[], pair: string): Ohlcv { +export const calculateSwapOhlcv = (ops: SwapOperationData[], pair: string): Ohlcv => { const timestamp = ops[0].timestamp; const prices = ops.map((op) => calculateBasePriceFromSwapOp(op)); const open = prices[0]; @@ -347,7 +321,7 @@ export function calculateSwapOhlcv(ops: SwapOperationData[], pair: string): Ohlc low, high }; -} +}; export function buildOhlcv(ops: SwapOperationData[]): Ohlcv[] { let ohlcv: Ohlcv[] = []; @@ -359,14 +333,14 @@ export function buildOhlcv(ops: SwapOperationData[]): Ohlcv[] { return ohlcv; } -export function calculateBasePriceFromSwapOp(op: SwapOperationData): number { +export const calculateBasePriceFromSwapOp = (op: SwapOperationData): number => { if (!op || !op.offerAmount || !op.returnAmount) { return 0; } const offerAmount = op.offerAmount; const askAmount = op.returnAmount; return op.direction === "Buy" ? Number(offerAmount) / Number(askAmount) : Number(askAmount) / Number(offerAmount); -} +}; export function getSwapDirection(offerDenom: string, askDenom: string): SwapDirection { const pair = pairsOnlyDenom.find((pair) => { @@ -391,8 +365,10 @@ export function findPairIndexFromDenoms(offerDenom: string, askDenom: string): n function getSymbolFromAsset(asset_infos: [AssetInfo, AssetInfo]): string { const findedPair = pairs.find( (p) => - JSON.stringify(p.asset_infos) === JSON.stringify(asset_infos) || - JSON.stringify(p.asset_infos) === JSON.stringify(asset_infos.reverse()) + p.asset_infos.some( + (assetInfo) => parseAssetInfoOnlyDenom(assetInfo) === parseAssetInfoOnlyDenom(asset_infos[0]) + ) && + p.asset_infos.some((assetInfo) => parseAssetInfoOnlyDenom(assetInfo) === parseAssetInfoOnlyDenom(asset_infos[1])) ); if (!findedPair) { throw new Error(`cannot found pair with asset_infos: ${JSON.stringify(asset_infos)}`); @@ -406,9 +382,9 @@ async function getCosmwasmClient(): Promise { return client; } -function parsePoolAmount(poolInfo: OraiswapPairTypes.PoolResponse, trueAsset: AssetInfo): bigint { +export const parsePoolAmount = (poolInfo: OraiswapPairTypes.PoolResponse, trueAsset: AssetInfo): bigint => { return BigInt(poolInfo.assets.find((asset) => isEqual(asset.info, trueAsset))?.amount || "0"); -} +}; async function fetchPoolInfoAmount(fromInfo: AssetInfo, toInfo: AssetInfo, pairAddr: string): Promise { const client = await getCosmwasmClient(); @@ -420,7 +396,7 @@ async function fetchPoolInfoAmount(fromInfo: AssetInfo, toInfo: AssetInfo, pairA } // get liquidity of pair from assetInfos -async function getPairLiquidity(assetInfos: [AssetInfo, AssetInfo]): Promise { +export const getPairLiquidity = async (assetInfos: [AssetInfo, AssetInfo]): Promise => { const duckDb = DuckDb.instances; // get info of pool in pair_infos, ask & offer are accumulated in sync process (via swap ops and lp ops). const poolInfo = await duckDb.getPoolByAssetInfos(assetInfos); @@ -428,7 +404,7 @@ async function getPairLiquidity(assetInfos: [AssetInfo, AssetInfo]): Promise { +): Promise => { const duckDb = DuckDb.instances; const pair = `${baseDenom}-${quoteDenom}`; const [volumeSwapPairInBaseAsset, volumeLiquidityPairInBaseAsset] = await Promise.all([ @@ -468,19 +444,19 @@ async function getVolumePairByAsset( }) ]); return volumeSwapPairInBaseAsset + volumeLiquidityPairInBaseAsset; -} +}; -async function getVolumePairByUsdt( +export const getVolumePairByUsdt = async ( [baseAssetInfo, quoteAssetInfo]: [AssetInfo, AssetInfo], startTime: Date, endTime: Date -): Promise { +): Promise => { const [baseDenom, quoteDenom] = [parseAssetInfoOnlyDenom(baseAssetInfo), parseAssetInfoOnlyDenom(quoteAssetInfo)]; const volumePairInBaseAsset = await getVolumePairByAsset([baseDenom, quoteDenom], startTime, endTime); const priceBaseAssetInUsdt = await getPriceAssetByUsdt(baseAssetInfo); const volumeInUsdt = priceBaseAssetInUsdt * Number(volumePairInBaseAsset); return BigInt(Math.round(volumeInUsdt)); -} +}; async function getAllVolume24h(): Promise { const tf = 24 * 60 * 60; // second of 24h @@ -494,7 +470,11 @@ async function getAllVolume24h(): Promise { // ===== end get volume pairs =====> // <==== start get fee pair ==== -async function getFeePair(asset_infos: [AssetInfo, AssetInfo], startTime: Date, endTime: Date): Promise { +export const getFeePair = async ( + asset_infos: [AssetInfo, AssetInfo], + startTime: Date, + endTime: Date +): Promise => { const duckDb = DuckDb.instances; const [swapFee, liquidityFee] = await Promise.all([ duckDb.getFeeSwap({ @@ -511,7 +491,7 @@ async function getFeePair(asset_infos: [AssetInfo, AssetInfo], startTime: Date, }) ]); return swapFee + liquidityFee; -} +}; async function getAllFees(): Promise { const tf = 7 * 24 * 60 * 60; // second of 7 days @@ -534,12 +514,8 @@ export { getAllFees, getAllVolume24h, getCosmwasmClient, - getFeePair, - getPairLiquidity, getSpecificDateBeforeNow, getSymbolFromAsset, - getVolumePairByAsset, - getVolumePairByUsdt, parseAssetInfo, parseAssetInfoOnlyDenom }; diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index fe48bd79..bcfde6f1 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -55,8 +55,7 @@ class WriteOrders extends WriteData { baseTokenDenom: item.baseTokenDenom, quoteTokenAmount: item.quoteTokenAmount, quoteTokenDenom: item.quoteTokenDenom, - opType: item.opType, - direction: null + opType: item.opType } as LpOpsData; }), ...swapData.map((item) => { @@ -65,12 +64,26 @@ class WriteOrders extends WriteData { baseTokenDenom: item.offerDenom, quoteTokenAmount: -item.returnAmount, // reverse sign because we assume first case is sell, check buy later. quoteTokenDenom: item.askDenom, - opType: null, direction: item.direction } as LpOpsData; }) ]; - await collectAccumulateLpAndSwapData(lpOpsData, poolInfos, pairInfos); + + const accumulatedData = await collectAccumulateLpAndSwapData(lpOpsData, poolInfos); + // update new lp pool amount to pair_infos + await Promise.all( + pairInfos + .map(({ pairAddr }) => { + if (accumulatedData[pairAddr]) { + return this.duckDb.updatePairInfoAmount( + accumulatedData[pairAddr].baseTokenAmount, + accumulatedData[pairAddr].quoteTokenAmount, + pairAddr + ); + } + }) + .filter(Boolean) + ); } async process(chunk: any): Promise { diff --git a/packages/oraidex-sync/tests/helper.spec.ts b/packages/oraidex-sync/tests/helper.spec.ts index 0ee4dcf3..1e74486e 100644 --- a/packages/oraidex-sync/tests/helper.spec.ts +++ b/packages/oraidex-sync/tests/helper.spec.ts @@ -1,4 +1,5 @@ -import { AssetInfo } from "@oraichain/oraidex-contracts-sdk"; +import fs from "fs"; +import { AssetInfo, SwapOperation } from "@oraichain/oraidex-contracts-sdk"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; import { ORAI, @@ -19,7 +20,6 @@ import { import { calculateBasePriceFromSwapOp, calculatePriceByPool, - collectAccumulateLpData, concatDataToUniqueKey, findAssetInfoPathToUsdt, findMappedTargetedAssetInfo, @@ -35,12 +35,16 @@ import { toDisplay } from "../src/helper"; import { extractUniqueAndFlatten, pairs } from "../src/pairs"; -import { PairInfoData, ProvideLiquidityOperationData, SwapDirection, SwapOperationData } from "../src/types"; -import { DuckDb, getVolumePairByAsset, getVolumePairByUsdt } from "../src"; +import { LpOpsData, PairInfoData, ProvideLiquidityOperationData, SwapDirection, SwapOperationData } from "../src/types"; +import { DuckDb, collectAccumulateLpAndSwapData, getVolumePairByAsset, getVolumePairByUsdt } from "../src"; +import * as poolHelper from "../src/pool-helper"; +import * as helper from "../src/helper"; describe("test-helper", () => { let duckDb: DuckDb; + afterEach(jest.restoreAllMocks); + describe("bigint", () => { describe("toAmount", () => { it("toAmount-percent", () => { @@ -200,7 +204,9 @@ describe("test-helper", () => { volume24Hour: 1n, apr: 1, totalLiquidity: 1, - fee7Days: 1n + fee7Days: 1n, + offerPoolAmount: 1n, + askPoolAmount: 1n } ]; let assetInfos: [AssetInfo, AssetInfo] = [{ native_token: { denom: ORAI } }, assetInfo]; @@ -376,81 +382,110 @@ describe("test-helper", () => { } ); - it("test-collectAccumulateLpData-should-aggregate-ops-with-same-pairs", () => { + it("test-collectAccumulateLpAndSwapData-should-aggregate-ops-with-same-pairs", async () => { + // setup, test with orai/usdt & orai/atom pair const poolResponses: PoolResponse[] = [ { assets: [ - { info: { native_token: { denom: ORAI } }, amount: "1" }, - { info: { token: { contract_addr: usdtCw20Address } }, amount: "1" } + { info: oraiInfo, amount: "1" }, + { info: usdtInfo, amount: "1" } ], - total_share: "2" + total_share: "1" }, { assets: [ - { info: { native_token: { denom: ORAI } }, amount: "4" }, - { info: { token: { contract_addr: atomIbcDenom } }, amount: "4" } + { info: oraiInfo, amount: "4" }, + { info: { native_token: { denom: atomIbcDenom } }, amount: "4" } ], total_share: "8" } ]; - const ops: ProvideLiquidityOperationData[] = [ + + const lpOpsData: LpOpsData[] = [ { - basePrice: 1, baseTokenAmount: 1, baseTokenDenom: ORAI, quoteTokenAmount: 1, quoteTokenDenom: usdtCw20Address, - baseTokenReserve: 1, - quoteTokenReserve: 1, - opType: "provide", - uniqueKey: "1", - timestamp: 1, - txCreator: "a", - txhash: "a", - txheight: 1, - taxRate: 1n + opType: "withdraw" + }, + { + baseTokenAmount: 2, + baseTokenDenom: ORAI, + quoteTokenAmount: 2, + quoteTokenDenom: usdtCw20Address, + opType: "provide" }, { - basePrice: 1, baseTokenAmount: 1, baseTokenDenom: ORAI, - quoteTokenAmount: 1, + quoteTokenAmount: -1, quoteTokenDenom: usdtCw20Address, - baseTokenReserve: 1, - quoteTokenReserve: 1, - opType: "withdraw", - uniqueKey: "2", - timestamp: 1, - txCreator: "a", - txhash: "a", - txheight: 1, - taxRate: 1n + direction: "Buy" }, { - basePrice: 1, baseTokenAmount: 1, baseTokenDenom: ORAI, - quoteTokenAmount: 1, + quoteTokenAmount: -1, + quoteTokenDenom: usdtCw20Address, + direction: "Sell" + }, + + { + baseTokenAmount: 1, + baseTokenDenom: ORAI, + quoteTokenAmount: -1, quoteTokenDenom: atomIbcDenom, - baseTokenReserve: 1, - quoteTokenReserve: 1, - opType: "withdraw", - uniqueKey: "3", - timestamp: 1, - txCreator: "a", - txhash: "a", - txheight: 1, - taxRate: 1n + direction: "Sell" } ]; + duckDb = await DuckDb.create(":memory:"); + await duckDb.createPairInfosTable(); + await duckDb.insertPairInfos([ + { + firstAssetInfo: JSON.stringify(oraiInfo), + secondAssetInfo: JSON.stringify(usdtInfo), + commissionRate: "", + pairAddr: "oraiUsdtPairAddr", + liquidityAddr: "", + oracleAddr: "", + symbols: "1", + fromIconUrl: "1", + toIconUrl: "1", + volume24Hour: 1n, + apr: 1, + totalLiquidity: 1, + fee7Days: 1n, + offerPoolAmount: 1n, + askPoolAmount: 1n + }, + { + firstAssetInfo: JSON.stringify(oraiInfo), + secondAssetInfo: JSON.stringify({ native_token: { denom: atomIbcDenom } }), + commissionRate: "", + pairAddr: "oraiAtomPairAddr", + liquidityAddr: "", + oracleAddr: "", + symbols: "1", + fromIconUrl: "1", + toIconUrl: "1", + volume24Hour: 1n, + apr: 1, + totalLiquidity: 1, + fee7Days: 1n, + offerPoolAmount: 1n, + askPoolAmount: 1n + } + ]); + + // act + const accumulatedData = await collectAccumulateLpAndSwapData(lpOpsData, poolResponses); - collectAccumulateLpData(ops, poolResponses); - expect(ops[0].baseTokenReserve.toString()).toEqual("2"); - expect(ops[0].quoteTokenReserve.toString()).toEqual("2"); - expect(ops[1].baseTokenReserve.toString()).toEqual("1"); - expect(ops[1].quoteTokenReserve.toString()).toEqual("1"); - expect(ops[2].baseTokenReserve.toString()).toEqual("3"); - expect(ops[2].quoteTokenReserve.toString()).toEqual("3"); + // assertion + expect(accumulatedData).toStrictEqual({ + oraiUsdtPairAddr: { baseTokenAmount: 2n, quoteTokenAmount: 2n }, + oraiAtomPairAddr: { baseTokenAmount: 5n, quoteTokenAmount: 3n } + }); }); it("test-concatDataToUniqueKey-should-return-unique-key-in-correct-order-from-timestamp-to-first-to-second-amount-and-denom", () => { @@ -524,10 +559,95 @@ describe("test-helper", () => { expect(newOps[1].uniqueKey).toEqual("2"); }); + describe("test-ohlcv-calculation", () => { + // setup + const ops: SwapOperationData[] = [ + { + offerAmount: 2, + offerDenom: ORAI, + returnAmount: 1, + askDenom: usdtCw20Address, + direction: "Buy", + uniqueKey: "1", + timestamp: 1, + txCreator: "a", + txhash: "a", + txheight: 1, + spreadAmount: 1, + taxAmount: 1, + commissionAmount: 1 + } as SwapOperationData, + { + offerAmount: 2, + offerDenom: ORAI, + returnAmount: 1, + askDenom: usdtCw20Address, + direction: "Sell", + uniqueKey: "1", + timestamp: 1, + txCreator: "a", + txhash: "a", + txheight: 1, + spreadAmount: 1, + taxAmount: 1, + commissionAmount: 1 + } as SwapOperationData, + { + offerAmount: 2, + offerDenom: ORAI, + returnAmount: 1, + askDenom: atomIbcDenom, + direction: "Sell", + uniqueKey: "1", + timestamp: 1, + txCreator: "a", + txhash: "a", + txheight: 1, + spreadAmount: 1, + taxAmount: 1, + commissionAmount: 1 + } as SwapOperationData + ]; + const opsByPair = ops.slice(0, 2); + + it("test-calculateSwapOhlcv-should-return-correctly-swap-ohlcv", () => { + // setup + const pair = "orai-usdt"; + jest.spyOn(helper, "calculateBasePriceFromSwapOp").mockReturnValue(1); + jest.spyOn(helper, "concatOhlcvToUniqueKey").mockReturnValue("orai-usdt-unique-key"); + + // act + const swapOhlcv = helper.calculateSwapOhlcv(opsByPair, pair); + + // assertion + expect(swapOhlcv).toStrictEqual({ + uniqueKey: "orai-usdt-unique-key", + timestamp: 1, + pair, + volume: 3n, + open: 1, + close: 1, + low: 1, + high: 1 + }); + }); + + it("test-groupSwapOpsByPair-should-return-correctly-group-swap-ops-by-pair", () => { + // act + const result = helper.groupSwapOpsByPair(ops); + + // assertion + expect(result[`${ORAI}-${usdtCw20Address}`].length).toEqual(opsByPair.length); + expect(result[`${ORAI}-${usdtCw20Address}`][0]).toStrictEqual(opsByPair[0]); + expect(result[`${ORAI}-${usdtCw20Address}`][1]).toStrictEqual(opsByPair[1]); + expect(result[`${ORAI}-${atomIbcDenom}`][0]).toStrictEqual(ops[2]); + }); + }); + it.each([ ["Buy" as SwapDirection, 2], ["Sell" as SwapDirection, 0.5] - ])("test-calculatePriceFromOrder", (direction: SwapDirection, expectedPrice: number) => { + ])("test-calculateBasePriceFromSwapOp", (direction: SwapDirection, expectedPrice: number) => { const swapOp = { offerAmount: 2, offerDenom: ORAI, @@ -589,6 +709,84 @@ describe("test-helper", () => { } ); + it("test-getSymbolFromAsset-should-throw-error-for-assetInfos-not-valid", () => { + const asset_infos = [oraiInfo, { token: { contract_addr: "invalid-token" } }] as [AssetInfo, AssetInfo]; + expect(() => helper.getSymbolFromAsset(asset_infos)).toThrowError( + `cannot found pair with asset_infos: ${JSON.stringify(asset_infos)}` + ); + }); + + it("test-getSymbolFromAsset-should-return-correctly-symbol-of-pair-for-valid-assetInfos", () => { + const asset_infos = [oraiInfo, usdtInfo] as [AssetInfo, AssetInfo]; + expect(helper.getSymbolFromAsset(asset_infos)).toEqual("ORAI/USDT"); + }); + + it.each([ + [oraiInfo, 1n], + [{ native_token: { denom: atomIbcDenom } }, 0n] + ])("test-parsePoolAmount-given-trueAsset-%p-should-return-%p", (assetInfo: AssetInfo, expectedResult: bigint) => { + // setup + const poolInfo: PoolResponse = { + assets: [ + { + info: oraiInfo, + amount: "1" + }, + { + info: usdtInfo, + amount: "1" + } + ], + total_share: "5" + }; + + // act + const result = helper.parsePoolAmount(poolInfo, assetInfo); + + // assertion + expect(result).toEqual(expectedResult); + }); + + describe("test-get-pair-liquidity", () => { + beforeEach(async () => { + duckDb = await DuckDb.create(":memory:"); + }); + + it.each([ + [0n, 0n, 0], + [1n, 1n, 4] + ])( + "test-getPairLiquidity-should-return-correctly-liquidity-by-USDT", + async (offerAmount: bigint, askAmount: bigint, expectedResult: number) => { + // setup + jest.spyOn(duckDb, "getPoolByAssetInfos").mockResolvedValue({ + firstAssetInfo: JSON.stringify(oraiInfo), + secondAssetInfo: JSON.stringify(usdtInfo), + commissionRate: "", + pairAddr: "oraiUsdtPairAddr", + liquidityAddr: "", + oracleAddr: "", + symbols: "1", + fromIconUrl: "1", + toIconUrl: "1", + volume24Hour: 1n, + apr: 1, + totalLiquidity: 1, + fee7Days: 1n, + offerPoolAmount: offerAmount, + askPoolAmount: askAmount + }); + jest.spyOn(poolHelper, "getPriceAssetByUsdt").mockResolvedValue(2); + + // act + const result = await helper.getPairLiquidity([oraiInfo, usdtInfo]); + + // assertion + expect(result).toEqual(expectedResult); + } + ); + }); + describe("test-get-volume-pairs", () => { it("test-getVolumePairByAsset-should-return-correctly-sum-volume-swap-&-liquidity", async () => { //setup mock @@ -603,20 +801,97 @@ describe("test-helper", () => { expect(result).toEqual(2n); }); - // it("test-getVolumePairByUsdt-should-return-correctly-volume-pair-in-USDT", async () => { - // //setup - // duckDb = await DuckDb.create(":memory:"); - // const [baseAssetInfo, quoteAssetInfo] = [oraiInfo, usdtInfo]; - - // // act - // const result = await getVolumePairByUsdt( - // [baseAssetInfo, quoteAssetInfo], - // new Date(1693394183), - // new Date(1693394183) - // ); - - // // assert - // expect(result).toEqual(2n); - // }); + it("test-getVolumePairByUsdt-should-return-correctly-volume-pair-in-USDT", async () => { + //setup + const [baseAssetInfo, quoteAssetInfo] = [oraiInfo, usdtInfo]; + jest.spyOn(helper, "getVolumePairByAsset").mockResolvedValue(1n); + jest.spyOn(poolHelper, "getPriceAssetByUsdt").mockResolvedValue(2); + + // act + const result = await getVolumePairByUsdt( + [baseAssetInfo, quoteAssetInfo], + new Date(1693394183), + new Date(1693394183) + ); + + // assert + expect(result).toEqual(2n); + }); + + it("test-getAllVolume24h-should-return-correctly-volume-all-pair", async () => { + //setup mock + jest.spyOn(helper, "getVolumePairByUsdt").mockResolvedValue(1n); + + // act + const result = await helper.getAllVolume24h(); + + // assert + expect(result.length).toEqual(pairs.length); + expect(result.every((value) => value === 1n)); + }); }); + + describe("test-get-fee-pair", () => { + it("test-getFeePair-should-return-correctly-sum-fee-swap-&-liquidity", async () => { + //setup mock + duckDb = await DuckDb.create(":memory:"); + jest.spyOn(duckDb, "getFeeSwap").mockResolvedValue(1n); + jest.spyOn(duckDb, "getFeeLiquidity").mockResolvedValue(1n); + + // act + const result = await helper.getFeePair([oraiInfo, usdtInfo], new Date(1693394183), new Date(1693394183)); + + // assert + expect(result).toEqual(2n); + }); + + it("test-getAllFees-should-return-correctly-fee-all-pair", async () => { + //setup mock + jest.spyOn(helper, "getFeePair").mockResolvedValue(1n); + + // act + const result = await helper.getAllFees(); + + // assert + expect(result.length).toEqual(pairs.length); + expect(result.every((value) => value === 1n)); + }); + }); + + it.each([ + [ + [oraiInfo, usdtInfo], + [ + { + orai_swap: { + offer_asset_info: oraiInfo, + ask_asset_info: usdtInfo + } + } + ] + ], + [ + [oraiInfo, usdtInfo, { native_token: { denom: atomIbcDenom } }], + [ + { + orai_swap: { + offer_asset_info: oraiInfo, + ask_asset_info: usdtInfo + } + }, + { + orai_swap: { + offer_asset_info: usdtInfo, + ask_asset_info: { native_token: { denom: atomIbcDenom } } + } + } + ] + ] + ])( + "test-generateSwapOperations-should-return-correctly-swap-ops", + (infoPath: AssetInfo[], expectedResult: SwapOperation[]) => { + const result = helper.generateSwapOperations(infoPath); + expect(result).toStrictEqual(expectedResult); + } + ); }); diff --git a/packages/oraidex-sync/tests/pool-helper.spec.ts b/packages/oraidex-sync/tests/pool-helper.spec.ts index ebbc8081..2ae35667 100644 --- a/packages/oraidex-sync/tests/pool-helper.spec.ts +++ b/packages/oraidex-sync/tests/pool-helper.spec.ts @@ -132,7 +132,9 @@ describe("test-pool-helper", () => { volume24Hour: 1n, apr: 1, totalLiquidity: 1, - fee7Days: 1n + fee7Days: 1n, + offerPoolAmount: 1n, + askPoolAmount: 1n } ]; await duckDb.insertPairInfos(pairInfoData); @@ -197,7 +199,9 @@ describe("test-pool-helper", () => { volume24Hour: 1n, apr: 1, totalLiquidity: 1, - fee7Days: 1n + fee7Days: 1n, + offerPoolAmount: 1n, + askPoolAmount: 1n }, { firstAssetInfo: JSON.stringify(oraiInfo as AssetInfo), @@ -212,7 +216,9 @@ describe("test-pool-helper", () => { volume24Hour: 1n, apr: 1, totalLiquidity: 1, - fee7Days: 1n + fee7Days: 1n, + offerPoolAmount: 1n, + askPoolAmount: 1n } ]; await duckDb.insertPairInfos(pairInfoData); @@ -301,7 +307,9 @@ describe("test-pool-helper", () => { volume24Hour: 1n, apr: 1, totalLiquidity: 1, - fee7Days: 1n + fee7Days: 1n, + offerPoolAmount: 1n, + askPoolAmount: 1n }, 13344890, 1 From 2b8cd0f6150cfd7e705d11d7bb6e652aa83472ab Mon Sep 17 00:00:00 2001 From: trungbach Date: Fri, 8 Sep 2023 18:32:36 +0700 Subject: [PATCH 33/46] test: finished testcase for pool apr --- packages/oraidex-sync/src/pool-helper.ts | 21 +++++------ .../oraidex-sync/tests/pool-helper.spec.ts | 37 ++++++++++++++++++- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/packages/oraidex-sync/src/pool-helper.ts b/packages/oraidex-sync/src/pool-helper.ts index 27f4ada7..b25a7fef 100644 --- a/packages/oraidex-sync/src/pool-helper.ts +++ b/packages/oraidex-sync/src/pool-helper.ts @@ -8,17 +8,16 @@ import { PairInfo } from "@oraichain/oraidex-contracts-sdk"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; -import { TokenInfoResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapToken.types"; import { isEqual } from "lodash"; import { ORAI, ORAIXOCH_INFO, SEC_PER_YEAR, atomic, network, oraiInfo, usdtInfo } from "./constants"; import { calculatePriceByPool, - fetchPoolInfoAmount, getCosmwasmClient, isAssetInfoPairReverse, parseAssetInfoOnlyDenom, validateNumber } from "./helper"; +import { DuckDb } from "./index"; import { pairs } from "./pairs"; import { fetchAllRewardPerSecInfos, @@ -28,7 +27,6 @@ import { queryPoolInfos } from "./query"; import { PairInfoData, PairMapping } from "./types"; -import { DuckDb } from "./index"; // use this type to determine the ratio of price of base to the quote or vice versa export type RatioDirection = "base_in_quote" | "quote_in_base"; @@ -203,21 +201,20 @@ export const calculateLiquidityFee = async ( // <==== calculate APR ==== export const calculateAprResult = async ( allLiquidities: number[], - allTokenInfo: TokenInfoResponse[], - allLpTokenAsset: OraiswapStakingTypes.PoolInfoResponse[], + allTotalSupplies: string[], + allBondAmounts: string[], allRewardPerSec: OraiswapStakingTypes.RewardsPerSecResponse[] ): Promise => { let aprResult = []; let ind = 0; for (const _pair of pairs) { const liquidityAmount = allLiquidities[ind] * Math.pow(10, -6); - const lpToken = allLpTokenAsset[ind]; - const tokenSupply = allTokenInfo[ind]; + const totalBondAmount = allBondAmounts[ind]; + const tokenSupply = allTotalSupplies[ind]; const rewardsPerSecData = allRewardPerSec[ind]; - if (!lpToken || !tokenSupply || !rewardsPerSecData) continue; + if (!totalBondAmount || !tokenSupply || !rewardsPerSecData) continue; - const bondValue = - (validateNumber(lpToken.total_bond_amount) * liquidityAmount) / validateNumber(tokenSupply.total_supply); + const bondValue = (validateNumber(totalBondAmount) * liquidityAmount) / validateNumber(tokenSupply); let rewardsPerYearValue = 0; for (const { amount, info } of rewardsPerSecData.assets) { @@ -246,7 +243,9 @@ export const fetchAprResult = async (pairInfos: PairInfoData[], allLiquidities: fetchAllTokenAssetPools(assetTokens), fetchAllRewardPerSecInfos(assetTokens) ]); - return calculateAprResult(allLiquidities, allTokenInfo, allLpTokenAsset, allRewardPerSec); + const allTotalSupplies = allTokenInfo.map((info) => info.total_supply); + const allBondAmounts = allLpTokenAsset.map((info) => info.total_bond_amount); + return calculateAprResult(allLiquidities, allTotalSupplies, allBondAmounts, allRewardPerSec); } catch (error) { console.log({ errorFetchAprResult: error }); } diff --git a/packages/oraidex-sync/tests/pool-helper.spec.ts b/packages/oraidex-sync/tests/pool-helper.spec.ts index 2ae35667..07c90895 100644 --- a/packages/oraidex-sync/tests/pool-helper.spec.ts +++ b/packages/oraidex-sync/tests/pool-helper.spec.ts @@ -1,4 +1,4 @@ -import { Asset, AssetInfo } from "@oraichain/oraidex-contracts-sdk"; +import { Asset, AssetInfo, OraiswapStakingTypes } from "@oraichain/oraidex-contracts-sdk"; import { ORAI, airiCw20Adress, @@ -11,7 +11,7 @@ import { } from "../src/constants"; import * as helper from "../src/helper"; -import { DuckDb } from "../src/index"; +import { DuckDb, pairs } from "../src/index"; import * as poolHelper from "../src/pool-helper"; import { PairInfoData, PairMapping, ProvideLiquidityOperationData } from "../src/types"; @@ -393,4 +393,37 @@ describe("test-pool-helper", () => { } ); }); + + it("test-calculateAprResult-should-return-correctly-APR", async () => { + // setup + const allLiquidities = Array(pairs.length).fill(1e6); + const allTotalSupplies = Array(pairs.length).fill("100000"); + const allBondAmounts = Array(pairs.length).fill("1"); + const allRewardPerSec: OraiswapStakingTypes.RewardsPerSecResponse[] = Array(pairs.length).fill({ + assets: [ + { + amount: "1", + info: oraiInfo + } + ] + }); + jest.spyOn(poolHelper, "getPriceAssetByUsdt").mockResolvedValueOnce(1); + + // act + const result = await poolHelper.calculateAprResult( + allLiquidities, + allTotalSupplies, + allBondAmounts, + allRewardPerSec + ); + console.dir({ result }, { depth: null }); + + // assertion + expect(result.length).toEqual(pairs.length); + expect(result).toStrictEqual([ + 31.535999999999998, 31.535999999999998, 31.535999999999998, 31.535999999999998, 31.535999999999998, + 31.535999999999998, 31.535999999999998, 31.535999999999998, 31.535999999999998, 31.535999999999998, + 31.535999999999998 + ]); + }); }); From 31b177a99e2b15998caa893f2ef0de26be643dcc Mon Sep 17 00:00:00 2001 From: trungbach Date: Fri, 8 Sep 2023 18:51:19 +0700 Subject: [PATCH 34/46] fix: fixed error get pool by asset infos --- packages/oraidex-sync/src/helper.ts | 1 + packages/oraidex-sync/src/index.ts | 15 ++++++--------- packages/oraidex-sync/src/pool-helper.ts | 3 ++- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index ac46e907..90a3875c 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -244,6 +244,7 @@ export const collectAccumulateLpAndSwapData = async (data: LpOpsData[], poolInfo let assetInfos = pool.assets.map((asset) => asset.info) as [AssetInfo, AssetInfo]; if (isAssetInfoPairReverse(assetInfos)) assetInfos.reverse(); const pairInfo = await duckDb.getPoolByAssetInfos(assetInfos); + if (!pairInfo) throw new Error("cannot find pair info when collectAccumulateLpAndSwapData"); const { pairAddr } = pairInfo; if (!accumulateData[pairAddr]) { diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index bcfde6f1..102c715e 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -2,7 +2,7 @@ import { SyncData, Txs, WriteData } from "@oraichain/cosmos-rpc-sync"; import "dotenv/config"; import { DuckDb } from "./db"; import { collectAccumulateLpAndSwapData, getSymbolFromAsset } from "./helper"; -import { getAllPairInfos, getPoolInfos } from "./pool-helper"; +import { getAllPairInfos, getPairByAssetInfos, getPoolInfos } from "./pool-helper"; import { parseAssetInfo, parseTxs } from "./tx-parsing"; import { Env, @@ -132,16 +132,13 @@ class OraiDexSync { const allPools = await this.duckDb.getPools(); if (allPools.length > 0) return; - const poolInfos = await getPoolInfos( - pairInfos.map((pair) => pair.contract_addr), - currentHeight - ); await this.duckDb.insertPairInfos( pairInfos.map((pair, index) => { const symbols = getSymbolFromAsset(pair.asset_infos); + const pairMapping = getPairByAssetInfos(pair.asset_infos); return { - firstAssetInfo: parseAssetInfo(pair.asset_infos[0]), - secondAssetInfo: parseAssetInfo(pair.asset_infos[1]), + firstAssetInfo: parseAssetInfo(pairMapping.asset_infos[0]), + secondAssetInfo: parseAssetInfo(pairMapping.asset_infos[1]), commissionRate: pair.commission_rate, pairAddr: pair.contract_addr, liquidityAddr: pair.liquidity_token, @@ -153,8 +150,8 @@ class OraiDexSync { apr: 0, totalLiquidity: 0, fee7Days: 0n, - offerPoolAmount: BigInt(poolInfos[index].assets[0].amount), - askPoolAmount: BigInt(poolInfos[index].assets[1].amount) + offerPoolAmount: 0n, + askPoolAmount: 0n } as PairInfoData; }) ); diff --git a/packages/oraidex-sync/src/pool-helper.ts b/packages/oraidex-sync/src/pool-helper.ts index b25a7fef..05406161 100644 --- a/packages/oraidex-sync/src/pool-helper.ts +++ b/packages/oraidex-sync/src/pool-helper.ts @@ -63,7 +63,8 @@ export const getPairByAssetInfos = (assetInfos: [AssetInfo, AssetInfo]): PairMap const [baseAsset, quoteAsset] = pair.asset_infos; const denoms = [parseAssetInfoOnlyDenom(baseAsset), parseAssetInfoOnlyDenom(quoteAsset)]; return ( - denoms.includes(parseAssetInfoOnlyDenom(assetInfos[0])) && denoms.includes(parseAssetInfoOnlyDenom(assetInfos[1])) + denoms.some((denom) => denom === parseAssetInfoOnlyDenom(assetInfos[0])) && + denoms.some((denom) => denom === parseAssetInfoOnlyDenom(assetInfos[1])) ); }); }; From d0c9df09e18e1c6f036ced6f0fdee788d8ecbe24 Mon Sep 17 00:00:00 2001 From: trungbach Date: Sun, 10 Sep 2023 11:15:51 +0700 Subject: [PATCH 35/46] test: added more testcase for duckdb --- packages/oraidex-sync/src/db.ts | 24 +-- packages/oraidex-sync/tests/db.spec.ts | 219 ++++++++++++++++++++++++- 2 files changed, 213 insertions(+), 30 deletions(-) diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index 03ec9d76..60d5b868 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -159,7 +159,6 @@ export class DuckDb { } async updatePairInfoAmount(offerPoolAmount: bigint, askPoolAmount: bigint, pairAddr: string) { - console.log({ offerPoolAmount, askPoolAmount, pairAddr }); await this.conn.all( `UPDATE pair_infos SET offerPoolAmount = ?, askPoolAmount = ? @@ -444,7 +443,7 @@ export class DuckDb { offerDenom ) ]); - return BigInt(feeRightDirection[0]?.totalFee ?? 0 + feeReverseDirection[0]?.totalFee ?? 0); + return BigInt(feeRightDirection[0]?.totalFee + feeReverseDirection[0]?.totalFee); } async getFeeLiquidity(payload: GetFeeSwap): Promise { @@ -504,25 +503,4 @@ export class DuckDb { ); return BigInt(result[0]?.totalVolume ?? 0); } - - async getPoolAmountFromAssetInfos(assetInfos: [AssetInfo, AssetInfo]): Promise { - const baseTokenDenom = parseAssetInfoOnlyDenom(assetInfos[0]); - const quoteTokenDenom = parseAssetInfoOnlyDenom(assetInfos[1]); - const result = await this.conn.all( - `SELECT * from lp_ops_data WHERE baseTokenDenom = ? AND quoteTokenDenom = ? ORDER BY txheight LIMIT 1`, - baseTokenDenom, - quoteTokenDenom - ); - - if (result.length === 0) { - console.dir({ nullForPair: assetInfos }, { depth: null }); - return null; - } - console.dir({ result }, { depth: null }); - - return { - offerPoolAmount: BigInt(result[0].baseTokenReserve), - askPoolAmount: BigInt(result[0].quoteTokenReserve) - }; - } } diff --git a/packages/oraidex-sync/tests/db.spec.ts b/packages/oraidex-sync/tests/db.spec.ts index c5fcb57b..55c378ef 100644 --- a/packages/oraidex-sync/tests/db.spec.ts +++ b/packages/oraidex-sync/tests/db.spec.ts @@ -1,14 +1,11 @@ +import fs from "fs"; +import { oraiInfo, usdtInfo } from "../src"; import { DuckDb } from "../src/db"; import { isoToTimestampNumber } from "../src/helper"; -import { ProvideLiquidityOperationData } from "../src/types"; -import fs from "fs"; +import { GetFeeSwap, GetVolumeQuery, PairInfoData, ProvideLiquidityOperationData } from "../src/types"; describe("test-duckdb", () => { let duckDb: DuckDb; - afterEach(() => { - fs.unlink(":memory:", () => {}); - }); - it.each<[string[], number[]]>([ [ ["orai", "atom"], @@ -154,7 +151,6 @@ describe("test-duckdb", () => { isoToTimestampNumber("2023-07-16T16:07:48.000Z"), isoToTimestampNumber("2023-07-17T16:07:48.000Z") ); - console.log("result: ", queryResult); expect(queryResult.volume["orai"]).toEqual(110); expect(queryResult.volume["atom"]).toEqual(10001); @@ -267,4 +263,213 @@ describe("test-duckdb", () => { queryResult = await duckDb.queryLpOps(); expect(queryResult.length).toEqual(2); }); + + it("test-updatePairInfoAmount-should-success", async () => { + // setup + duckDb = await DuckDb.create(":memory:"); + await duckDb.createPairInfosTable(); + await duckDb.insertPairInfos([ + { + firstAssetInfo: JSON.stringify(oraiInfo), + secondAssetInfo: JSON.stringify(usdtInfo), + commissionRate: "", + pairAddr: "orai1c5s03c3l336dgesne7dylnmhszw8554tsyy9yt", + liquidityAddr: "", + oracleAddr: "", + symbols: "1", + fromIconUrl: "1", + toIconUrl: "1", + volume24Hour: 1n, + apr: 1, + totalLiquidity: 1, + fee7Days: 1n, + offerPoolAmount: 1n, + askPoolAmount: 1n + } as PairInfoData + ]); + + // act + await duckDb.updatePairInfoAmount(2n, 3n, "orai1c5s03c3l336dgesne7dylnmhszw8554tsyy9yt"); + + // assertion + const pairInfoAfterUpdate = await duckDb.getPoolByAssetInfos([oraiInfo, usdtInfo]); + expect(pairInfoAfterUpdate.offerPoolAmount).toEqual(2); + expect(pairInfoAfterUpdate.askPoolAmount).toEqual(3); + }); + + it("test-getFeeSwap-should-return-correctly-fee-in-USDT", async () => { + // setup + duckDb = await DuckDb.create(":memory:"); + await duckDb.createSwapOpsTable(); + await duckDb.insertSwapOps([ + { + askDenom: "orai", + commissionAmount: 1e6, + direction: "Buy", + offerAmount: 10, + offerDenom: "atom", + uniqueKey: "2", + returnAmount: 1, + spreadAmount: 0, + taxAmount: 0, + timestamp: 1589610068000 / 1000, + txhash: "foo", + txheight: 1 + }, + { + askDenom: "atom", + commissionAmount: 1e6, + direction: "Sell", + offerAmount: 10, + offerDenom: "orai", + uniqueKey: "3", + returnAmount: 1, + spreadAmount: 0, + taxAmount: 0, + timestamp: 1589610068000 / 1000, + txhash: "foo", + txheight: 1 + } + ]); + const payload: GetFeeSwap = { + offerDenom: "orai", + askDenom: "atom", + startTime: 1589610068000 / 1000, + endTime: 1689610068000 / 1000 + }; + + // act + const feeSwap = await duckDb.getFeeSwap(payload); + + // assertion + expect(feeSwap).toEqual(2000000n); + }); + + it.each([ + ["invalid-pair", 1, 3, 0n], + ["orai-usdt", 1, 3, 2n], + ["orai-usdt", 1, 5, 3n] + ])( + "test-getVolumeSwap-should-return-correctly-volume-in-base-asset", + async (pair: string, startTime: number, endTime: number, expectedResult: bigint) => { + // setup + duckDb = await DuckDb.create(":memory:"); + await duckDb.createSwapOhlcv(); + await duckDb.insertOhlcv([ + { + uniqueKey: "1", + timestamp: 1, + pair: "orai-usdt", + volume: 1n, // base volume + open: 2, + close: 2, // base price + low: 2, + high: 2 + }, + { + uniqueKey: "2", + timestamp: 3, + pair: "orai-usdt", + volume: 1n, // base volume + open: 2, + close: 2, // base price + low: 2, + high: 2 + }, + { + uniqueKey: "3", + timestamp: 5, + pair: "orai-usdt", + volume: 1n, // base volume + open: 2, + close: 2, // base price + low: 2, + high: 2 + } + ]); + + const payload: GetVolumeQuery = { + pair, + startTime, + endTime + }; + + // act + const volumeSwap = await duckDb.getVolumeSwap(payload); + + // assertion + expect(volumeSwap).toEqual(expectedResult); + } + ); + + describe("test-get-fee-&-volume-liquidity", () => { + // setup + beforeAll(async () => { + duckDb = await DuckDb.create(":memory:"); + await duckDb.createLiquidityOpsTable(); + await duckDb.insertLpOps([ + { + basePrice: 1, + baseTokenAmount: 1, + baseTokenDenom: "orai", + baseTokenReserve: 0, + opType: "withdraw", + uniqueKey: "1", + quoteTokenAmount: 2, + quoteTokenDenom: "atom", + quoteTokenReserve: 0, + timestamp: 1589610068000 / 1000, + txCreator: "foobar", + txhash: "foo", + txheight: 1, + taxRate: 1 + }, + { + basePrice: 1, + baseTokenAmount: 1, + baseTokenDenom: "orai", + baseTokenReserve: 0, + opType: "provide", + uniqueKey: "2", + quoteTokenAmount: 2, + quoteTokenDenom: "atom", + quoteTokenReserve: 0, + timestamp: 1589610068000 / 1000, + txCreator: "foobar", + txhash: "foo", + txheight: 1, + taxRate: 2 + } + ]); + }); + + it("test-getFeeLiquidity-should-return-correctly-fee-in-USDT", async () => { + const payload: GetFeeSwap = { + offerDenom: "orai", + askDenom: "atom", + startTime: 1589610068000 / 1000, + endTime: 1689610068000 / 1000 + }; + + // act + const feeSwap = await duckDb.getFeeLiquidity(payload); + + // assertion + expect(feeSwap).toEqual(3n); + }); + + it("test-getVolumeLiquidity-should-return-correctly-volume-liquidity-in-base-asset", async () => { + // act + const payload: GetFeeSwap = { + offerDenom: "orai", + askDenom: "atom", + startTime: 1589610068000 / 1000, + endTime: 1689610068000 / 1000 + }; + const volumeByBaseAsset = await duckDb.getVolumeLiquidity(payload); + + // assertion + expect(volumeByBaseAsset).toEqual(2n); + }); + }); }); From ea2ada3664c5731540ca604bbff7021d59294a37 Mon Sep 17 00:00:00 2001 From: trungbach Date: Mon, 11 Sep 2023 18:35:16 +0700 Subject: [PATCH 36/46] removed column not use in table pair info --- packages/oraidex-server/src/index.ts | 47 ++++++++++--------- packages/oraidex-sync/src/db.ts | 4 -- packages/oraidex-sync/src/helper.ts | 8 ++-- packages/oraidex-sync/src/index.ts | 4 -- packages/oraidex-sync/src/types.ts | 11 +++-- packages/oraidex-sync/tests/db.spec.ts | 10 ++-- packages/oraidex-sync/tests/helper.spec.ts | 31 ++++-------- .../oraidex-sync/tests/pool-helper.spec.ts | 26 ++-------- 8 files changed, 53 insertions(+), 88 deletions(-) diff --git a/packages/oraidex-server/src/index.ts b/packages/oraidex-server/src/index.ts index 65adc755..c16da11a 100644 --- a/packages/oraidex-server/src/index.ts +++ b/packages/oraidex-server/src/index.ts @@ -1,32 +1,33 @@ #!/usr/bin/env node -import "dotenv/config"; -import express, { Request } from "express"; +import { CosmWasmClient } from "@cosmjs/cosmwasm-stargate"; +import { OraiswapRouterQueryClient } from "@oraichain/oraidex-contracts-sdk"; import { DuckDb, - TickerInfo, - pairs, - parseAssetInfoOnlyDenom, - findPairAddress, - toDisplay, + GetCandlesQuery, + ORAI, OraiDexSync, - simulateSwapPrice, - pairsOnlyDenom, + PairInfoDataResponse, + TickerInfo, VolumeRange, - oraiUsdtPairOnlyDenom, - ORAI, - getAllVolume24h, + fetchAprResult, + findPairAddress, getAllFees, + getAllVolume24h, getPairLiquidity, - fetchAprResult + oraiUsdtPairOnlyDenom, + pairs, + pairsOnlyDenom, + parseAssetInfoOnlyDenom, + simulateSwapPrice, + toDisplay } from "@oraichain/oraidex-sync"; import cors from "cors"; -import { CosmWasmClient } from "@cosmjs/cosmwasm-stargate"; -import { OraiswapRouterQueryClient } from "@oraichain/oraidex-contracts-sdk"; -import { getDate24hBeforeNow, getSpecificDateBeforeNow, pairToString, parseSymbolsToTickerId } from "./helper"; -import { GetCandlesQuery } from "@oraichain/oraidex-sync"; +import "dotenv/config"; +import express, { Request } from "express"; import fs from "fs"; import path from "path"; +import { getDate24hBeforeNow, getSpecificDateBeforeNow, pairToString, parseSymbolsToTickerId } from "./helper"; const app = express(); app.use(cors()); @@ -239,11 +240,11 @@ app.get("/v1/pools/", async (_req, res) => { const [volumes, allFee7Days] = await Promise.all([getAllVolume24h(), getAllFees()]); const pools = await duckDb.getPools(); - const allLiquidities = await Promise.all( - pools.map((pair) => { - return getPairLiquidity([JSON.parse(pair.firstAssetInfo), JSON.parse(pair.secondAssetInfo)]); - }) - ); + const allLiquidities = (await Promise.allSettled(pools.map((pair) => getPairLiquidity(pair)))).map((result) => { + if (result.status === "fulfilled") return result.value; + else console.error("error get allLiquidities: ", result.reason); + }); + const allApr = await fetchAprResult(pools, allLiquidities); res.status(200).send( pools.map((pool, index) => { @@ -253,7 +254,7 @@ app.get("/v1/pools/", async (_req, res) => { fee7Days: allFee7Days[index]?.toString() ?? "0", apr: allApr[index], totalLiquidity: allLiquidities[index] - }; + } as PairInfoDataResponse; }) ); } catch (error) { diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index 60d5b868..a10026f6 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -144,10 +144,6 @@ export class DuckDb { symbols VARCHAR, fromIconUrl VARCHAR, toIconUrl VARCHAR, - volume24Hour UBIGINT, - apr DOUBLE, - totalLiquidity UINT64, - fee7Days UBIGINT, offerPoolAmount UBIGINT, askPoolAmount UBIGINT, PRIMARY KEY (pairAddr) )` diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index 90a3875c..5b197c57 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -216,7 +216,6 @@ export function isAssetInfoPairReverse(assetInfos: AssetInfo[]): boolean { * @param poolInfos - pool info data for initial lp accumulation * @param pairInfos - pool info data from db */ -// TODO: write test cases for this function export const collectAccumulateLpAndSwapData = async (data: LpOpsData[], poolInfos: PoolResponse[]) => { let accumulateData: { [key: string]: { @@ -397,12 +396,11 @@ async function fetchPoolInfoAmount(fromInfo: AssetInfo, toInfo: AssetInfo, pairA } // get liquidity of pair from assetInfos -export const getPairLiquidity = async (assetInfos: [AssetInfo, AssetInfo]): Promise => { - const duckDb = DuckDb.instances; +export const getPairLiquidity = async (poolInfo: PairInfoData): Promise => { // get info of pool in pair_infos, ask & offer are accumulated in sync process (via swap ops and lp ops). - const poolInfo = await duckDb.getPoolByAssetInfos(assetInfos); if (!poolInfo.askPoolAmount || !poolInfo.offerPoolAmount) return 0; - const baseAssetInfo = assetInfos[0]; + + const baseAssetInfo = JSON.parse(poolInfo.firstAssetInfo); const priceBaseAssetInUsdt = await getPriceAssetByUsdt(baseAssetInfo); return priceBaseAssetInUsdt * Number(poolInfo.offerPoolAmount) * 2; }; diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index 102c715e..cd80aefb 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -146,10 +146,6 @@ class OraiDexSync { symbols, fromIconUrl: "url1", toIconUrl: "url2", - volume24Hour: 0n, - apr: 0, - totalLiquidity: 0, - fee7Days: 0n, offerPoolAmount: 0n, askPoolAmount: 0n } as PairInfoData; diff --git a/packages/oraidex-sync/src/types.ts b/packages/oraidex-sync/src/types.ts index 4056155a..dbe99769 100644 --- a/packages/oraidex-sync/src/types.ts +++ b/packages/oraidex-sync/src/types.ts @@ -44,12 +44,15 @@ export type PairInfoData = { symbols: string; fromIconUrl: string; toIconUrl: string; - volume24Hour: bigint; - apr: number; - totalLiquidity: number; offerPoolAmount: bigint; askPoolAmount: bigint; - fee7Days: bigint; +}; + +export type PairInfoDataResponse = PairInfoData & { + apr: number; + totalLiquidity: number; + volume24Hour: string; + fee7Days: string; }; export type PriceInfo = { diff --git a/packages/oraidex-sync/tests/db.spec.ts b/packages/oraidex-sync/tests/db.spec.ts index 55c378ef..2d6bc423 100644 --- a/packages/oraidex-sync/tests/db.spec.ts +++ b/packages/oraidex-sync/tests/db.spec.ts @@ -5,7 +5,13 @@ import { isoToTimestampNumber } from "../src/helper"; import { GetFeeSwap, GetVolumeQuery, PairInfoData, ProvideLiquidityOperationData } from "../src/types"; describe("test-duckdb", () => { let duckDb: DuckDb; + afterAll(jest.resetModules); + afterEach(() => { + jest.clearAllMocks(); + jest.restoreAllMocks(); + jest.resetAllMocks(); + }); it.each<[string[], number[]]>([ [ ["orai", "atom"], @@ -279,10 +285,6 @@ describe("test-duckdb", () => { symbols: "1", fromIconUrl: "1", toIconUrl: "1", - volume24Hour: 1n, - apr: 1, - totalLiquidity: 1, - fee7Days: 1n, offerPoolAmount: 1n, askPoolAmount: 1n } as PairInfoData diff --git a/packages/oraidex-sync/tests/helper.spec.ts b/packages/oraidex-sync/tests/helper.spec.ts index 1e74486e..822d5c9b 100644 --- a/packages/oraidex-sync/tests/helper.spec.ts +++ b/packages/oraidex-sync/tests/helper.spec.ts @@ -1,4 +1,3 @@ -import fs from "fs"; import { AssetInfo, SwapOperation } from "@oraichain/oraidex-contracts-sdk"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; import { @@ -43,8 +42,12 @@ import * as helper from "../src/helper"; describe("test-helper", () => { let duckDb: DuckDb; - afterEach(jest.restoreAllMocks); - + afterAll(jest.resetModules); + afterEach(() => { + jest.clearAllMocks(); + jest.restoreAllMocks(); + jest.resetAllMocks(); + }); describe("bigint", () => { describe("toAmount", () => { it("toAmount-percent", () => { @@ -201,10 +204,6 @@ describe("test-helper", () => { symbols: "1", fromIconUrl: "1", toIconUrl: "1", - volume24Hour: 1n, - apr: 1, - totalLiquidity: 1, - fee7Days: 1n, offerPoolAmount: 1n, askPoolAmount: 1n } @@ -452,10 +451,6 @@ describe("test-helper", () => { symbols: "1", fromIconUrl: "1", toIconUrl: "1", - volume24Hour: 1n, - apr: 1, - totalLiquidity: 1, - fee7Days: 1n, offerPoolAmount: 1n, askPoolAmount: 1n }, @@ -469,10 +464,6 @@ describe("test-helper", () => { symbols: "1", fromIconUrl: "1", toIconUrl: "1", - volume24Hour: 1n, - apr: 1, - totalLiquidity: 1, - fee7Days: 1n, offerPoolAmount: 1n, askPoolAmount: 1n } @@ -759,7 +750,7 @@ describe("test-helper", () => { "test-getPairLiquidity-should-return-correctly-liquidity-by-USDT", async (offerAmount: bigint, askAmount: bigint, expectedResult: number) => { // setup - jest.spyOn(duckDb, "getPoolByAssetInfos").mockResolvedValue({ + const poolInfo: PairInfoData = { firstAssetInfo: JSON.stringify(oraiInfo), secondAssetInfo: JSON.stringify(usdtInfo), commissionRate: "", @@ -769,17 +760,13 @@ describe("test-helper", () => { symbols: "1", fromIconUrl: "1", toIconUrl: "1", - volume24Hour: 1n, - apr: 1, - totalLiquidity: 1, - fee7Days: 1n, offerPoolAmount: offerAmount, askPoolAmount: askAmount - }); + }; jest.spyOn(poolHelper, "getPriceAssetByUsdt").mockResolvedValue(2); // act - const result = await helper.getPairLiquidity([oraiInfo, usdtInfo]); + const result = await helper.getPairLiquidity(poolInfo); // assertion expect(result).toEqual(expectedResult); diff --git a/packages/oraidex-sync/tests/pool-helper.spec.ts b/packages/oraidex-sync/tests/pool-helper.spec.ts index 07c90895..a008705c 100644 --- a/packages/oraidex-sync/tests/pool-helper.spec.ts +++ b/packages/oraidex-sync/tests/pool-helper.spec.ts @@ -27,9 +27,11 @@ describe("test-pool-helper", () => { duckDb.createSwapOhlcv() ]); }); + afterAll(jest.resetModules); afterEach(() => { jest.clearAllMocks(); jest.restoreAllMocks(); + jest.resetAllMocks(); }); it.each<[string, [AssetInfo, AssetInfo], boolean]>([ @@ -129,10 +131,6 @@ describe("test-pool-helper", () => { symbols: "1", fromIconUrl: "1", toIconUrl: "1", - volume24Hour: 1n, - apr: 1, - totalLiquidity: 1, - fee7Days: 1n, offerPoolAmount: 1n, askPoolAmount: 1n } @@ -196,10 +194,6 @@ describe("test-pool-helper", () => { symbols: "1", fromIconUrl: "1", toIconUrl: "1", - volume24Hour: 1n, - apr: 1, - totalLiquidity: 1, - fee7Days: 1n, offerPoolAmount: 1n, askPoolAmount: 1n }, @@ -213,10 +207,6 @@ describe("test-pool-helper", () => { symbols: "1", fromIconUrl: "1", toIconUrl: "1", - volume24Hour: 1n, - apr: 1, - totalLiquidity: 1, - fee7Days: 1n, offerPoolAmount: 1n, askPoolAmount: 1n } @@ -304,10 +294,6 @@ describe("test-pool-helper", () => { symbols: "1", fromIconUrl: "1", toIconUrl: "1", - volume24Hour: 1n, - apr: 1, - totalLiquidity: 1, - fee7Days: 1n, offerPoolAmount: 1n, askPoolAmount: 1n }, @@ -407,7 +393,7 @@ describe("test-pool-helper", () => { } ] }); - jest.spyOn(poolHelper, "getPriceAssetByUsdt").mockResolvedValueOnce(1); + jest.spyOn(poolHelper, "getPriceAssetByUsdt").mockResolvedValue(1); // act const result = await poolHelper.calculateAprResult( @@ -420,10 +406,6 @@ describe("test-pool-helper", () => { // assertion expect(result.length).toEqual(pairs.length); - expect(result).toStrictEqual([ - 31.535999999999998, 31.535999999999998, 31.535999999999998, 31.535999999999998, 31.535999999999998, - 31.535999999999998, 31.535999999999998, 31.535999999999998, 31.535999999999998, 31.535999999999998, - 31.535999999999998 - ]); + expect(result).toStrictEqual(Array(pairs.length).fill(315360000)); }); }); From 24b99a42a71d3e360daf51fcc7845d29f1f93d8a Mon Sep 17 00:00:00 2001 From: trungbach Date: Tue, 12 Sep 2023 10:32:57 +0700 Subject: [PATCH 37/46] fixed failed testcase --- packages/oraidex-sync/src/pool-helper.ts | 2 +- packages/oraidex-sync/tests/db.spec.ts | 7 ++-- packages/oraidex-sync/tests/helper.spec.ts | 12 +++---- .../oraidex-sync/tests/pool-helper.spec.ts | 35 +++++++++---------- 4 files changed, 26 insertions(+), 30 deletions(-) diff --git a/packages/oraidex-sync/src/pool-helper.ts b/packages/oraidex-sync/src/pool-helper.ts index 05406161..2f535cdd 100644 --- a/packages/oraidex-sync/src/pool-helper.ts +++ b/packages/oraidex-sync/src/pool-helper.ts @@ -84,7 +84,7 @@ export const getPriceByAsset = async ( ): Promise => { const duckDb = DuckDb.instances; const poolInfo = await duckDb.getPoolByAssetInfos(assetInfos); - if (!poolInfo.askPoolAmount || !poolInfo.offerPoolAmount) return 0; + if (!poolInfo || !poolInfo.askPoolAmount || !poolInfo.offerPoolAmount) return 0; // offer: orai, ask: usdt -> price offer in ask = calculatePriceByPool([ask, offer]) // offer: orai, ask: atom -> price ask in offer = calculatePriceByPool([offer, ask]) const basePrice = calculatePriceByPool( diff --git a/packages/oraidex-sync/tests/db.spec.ts b/packages/oraidex-sync/tests/db.spec.ts index 2d6bc423..302fabb6 100644 --- a/packages/oraidex-sync/tests/db.spec.ts +++ b/packages/oraidex-sync/tests/db.spec.ts @@ -12,6 +12,7 @@ describe("test-duckdb", () => { jest.restoreAllMocks(); jest.resetAllMocks(); }); + it.each<[string[], number[]]>([ [ ["orai", "atom"], @@ -200,7 +201,7 @@ describe("test-duckdb", () => { it("test-duckdb-insert-bulk-should-pass-and-can-query", async () => { //setup duckDb = await DuckDb.create(":memory:"); - await Promise.all([duckDb.createLiquidityOpsTable()]); + await duckDb.createLiquidityOpsTable(); // act & test const newDate = 1689610068000 / 1000; const data: ProvideLiquidityOperationData[] = [ @@ -228,10 +229,10 @@ describe("test-duckdb", () => { expect(queryResult[0]).toEqual(data[0]); }); - it("test-insert-same-unique-key-should-replace-data", async () => { + test("test-insert-same-unique-key-should-replace-data", async () => { // setup duckDb = await DuckDb.create(":memory:"); - await Promise.all([duckDb.createHeightSnapshot(), duckDb.createLiquidityOpsTable(), duckDb.createSwapOpsTable()]); + await duckDb.createLiquidityOpsTable(); const currentTimeStamp = Math.round(new Date().getTime() / 1000); let data: ProvideLiquidityOperationData[] = [ { diff --git a/packages/oraidex-sync/tests/helper.spec.ts b/packages/oraidex-sync/tests/helper.spec.ts index 822d5c9b..1869cd1a 100644 --- a/packages/oraidex-sync/tests/helper.spec.ts +++ b/packages/oraidex-sync/tests/helper.spec.ts @@ -440,6 +440,7 @@ describe("test-helper", () => { ]; duckDb = await DuckDb.create(":memory:"); await duckDb.createPairInfosTable(); + await duckDb.insertPairInfos([ { firstAssetInfo: JSON.stringify(oraiInfo), @@ -663,15 +664,10 @@ describe("test-helper", () => { it.each([ [usdtCw20Address, "orai", "Buy" as SwapDirection], - ["orai", usdtCw20Address, "Sell" as SwapDirection] - ])("test-getSwapDirection", (offerDenom: string, askDenom: string, expectedDirection: SwapDirection) => { + ["orai", usdtCw20Address, "Sell" as SwapDirection], + ["foo", "bar", undefined] + ])("test-getSwapDirection", (offerDenom: string, askDenom: string, expectedDirection: SwapDirection | undefined) => { // execute - // throw error case when offer & ask not in pair - try { - getSwapDirection("foo", "bar"); - } catch (error) { - expect(error).toEqual(new Error("Cannot find asset infos in list of pairs")); - } const result = getSwapDirection(offerDenom, askDenom); expect(result).toEqual(expectedDirection); }); diff --git a/packages/oraidex-sync/tests/pool-helper.spec.ts b/packages/oraidex-sync/tests/pool-helper.spec.ts index a008705c..e2668121 100644 --- a/packages/oraidex-sync/tests/pool-helper.spec.ts +++ b/packages/oraidex-sync/tests/pool-helper.spec.ts @@ -17,22 +17,10 @@ import { PairInfoData, PairMapping, ProvideLiquidityOperationData } from "../src describe("test-pool-helper", () => { let duckDb: DuckDb; - beforeAll(async () => { - duckDb = await DuckDb.create(":memory:"); - await Promise.all([ - duckDb.createHeightSnapshot(), - duckDb.createLiquidityOpsTable(), - duckDb.createSwapOpsTable(), - duckDb.createPairInfosTable(), - duckDb.createSwapOhlcv() - ]); - }); + afterAll(jest.resetModules); - afterEach(() => { - jest.clearAllMocks(); - jest.restoreAllMocks(); - jest.resetAllMocks(); - }); + afterEach(jest.resetModules); + afterEach(jest.restoreAllMocks); it.each<[string, [AssetInfo, AssetInfo], boolean]>([ [ @@ -111,15 +99,24 @@ describe("test-pool-helper", () => { describe("test-calculate-price-group-funcs", () => { // use orai/usdt in this test suite - it("test-getPriceByAsset-when-duckdb-empty-should-throw-error", async () => { - await expect(poolHelper.getPriceByAsset([oraiInfo, usdtInfo], "base_in_quote")).rejects.toBeInstanceOf(Error); - }); + // it("test-getPriceByAsset-when-duckdb-empty-should-return-0", async () => { + // // setup + // duckDb = await DuckDb.create(":memory:"); + // await Promise.all([duckDb.createPairInfosTable()]); + + // // act & assertion + // const result = await poolHelper.getPriceByAsset([oraiInfo, usdtInfo], "base_in_quote"); + // expect(result).toEqual(0); + // }); it.each<[[AssetInfo, AssetInfo], poolHelper.RatioDirection, number]>([ + [[oraiInfo, { token: { contract_addr: "invalid-token" } }], "base_in_quote", 0], [[oraiInfo, usdtInfo], "base_in_quote", 0.5], [[oraiInfo, usdtInfo], "quote_in_base", 2] ])("test-getPriceByAsset-should-return-correctly-price", async (assetInfos, ratioDirection, expectedPrice) => { // setup + duckDb = await DuckDb.create(":memory:"); + await duckDb.createPairInfosTable(); let pairInfoData: PairInfoData[] = [ { firstAssetInfo: JSON.stringify({ native_token: { denom: ORAI } } as AssetInfo), @@ -183,6 +180,8 @@ describe("test-pool-helper", () => { "test-getPriceAssetByUsdt-with-%p-should-return-correctly-price-of-asset-in-USDT", async (_caseName: string, assetInfo: AssetInfo, expectedPrice: number) => { // setup & mock + duckDb = await DuckDb.create(":memory:"); + await Promise.all([duckDb.createPairInfosTable(), duckDb.createLiquidityOpsTable()]); const pairInfoData: PairInfoData[] = [ { firstAssetInfo: JSON.stringify(oraiInfo as AssetInfo), From 2f65007e2c894a8f59c3f4f03cd726f97fd194ab Mon Sep 17 00:00:00 2001 From: ducphamle2 Date: Mon, 11 Sep 2023 21:33:35 -0700 Subject: [PATCH 38/46] reused oraiInfo & usdtInfo --- packages/oraidex-sync/src/helper.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index 5b197c57..cc2258a2 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -7,7 +7,7 @@ import { } from "@oraichain/oraidex-contracts-sdk"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; import { isEqual, maxBy, minBy } from "lodash"; -import { ORAI, atomic, tenAmountInDecimalSix, truncDecimals, usdtCw20Address } from "./constants"; +import { ORAI, atomic, oraiInfo, tenAmountInDecimalSix, truncDecimals, usdtInfo } from "./constants"; import { DuckDb } from "./db"; import { pairs, pairsOnlyDenom } from "./pairs"; import { getPriceAssetByUsdt } from "./pool-helper"; @@ -128,8 +128,6 @@ function findAssetInfoPathToUsdt(info: AssetInfo): AssetInfo[] { // first, check usdt mapped target infos because if we the info pairs with usdt directly then we can easily calculate its price // otherwise, we find orai mapped target infos, which can lead to usdt. // finally, if not paired with orai, then we find recusirvely to find a path leading to usdt token - const usdtInfo = { token: { contract_addr: usdtCw20Address } }; - const oraiInfo = { native_token: { denom: ORAI } }; if (parseAssetInfo(info) === parseAssetInfo(usdtInfo)) return [info]; // means there's no path, the price should be 1 const mappedUsdtInfoList = findMappedTargetedAssetInfo(usdtInfo); if (mappedUsdtInfoList.find((assetInfo) => parseAssetInfo(assetInfo) === parseAssetInfo(info))) From bbd6bbcd228dcd7ef9ddafade38ee4a365a53a4f Mon Sep 17 00:00:00 2001 From: trungbach Date: Tue, 12 Sep 2023 15:35:25 +0700 Subject: [PATCH 39/46] use domain name as fallback instead ip --- packages/oraidex-sync/src/helper.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index cc2258a2..def535df 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -375,7 +375,7 @@ function getSymbolFromAsset(asset_infos: [AssetInfo, AssetInfo]): string { } async function getCosmwasmClient(): Promise { - const rpcUrl = process.env.RPC_URL || "http://35.237.59.125:26657"; + const rpcUrl = process.env.RPC_URL || "https://rpc.orai.io"; const client = await CosmWasmClient.connect(rpcUrl); return client; } From a9dde8c76e0f8f949f5cee6b82fb387866d09bb2 Mon Sep 17 00:00:00 2001 From: trungbach Date: Tue, 12 Sep 2023 18:45:01 +0700 Subject: [PATCH 40/46] create pool_amount & apr table --- packages/oraidex-sync/src/db.ts | 39 ++++++++++++++++++++++++- packages/oraidex-sync/src/helper.ts | 32 ++++++++++++++------ packages/oraidex-sync/src/index.ts | 45 ++++++++++++++++------------- packages/oraidex-sync/src/types.ts | 11 +++++++ 4 files changed, 97 insertions(+), 30 deletions(-) diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index a10026f6..0a4e2cd1 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -12,7 +12,8 @@ import { GetCandlesQuery, GetFeeSwap, GetVolumeQuery, - PoolInfo + PoolInfo, + PoolAmountHistory } from "./types"; import fs, { rename } from "fs"; import { @@ -499,4 +500,40 @@ export class DuckDb { ); return BigInt(result[0]?.totalVolume ?? 0); } + + async createPoolOpsTable() { + await this.conn.exec( + `CREATE TABLE IF NOT EXISTS lp_amount_history ( + uniqueKey varchar UNIQUE, + pairAddr varchar, + timestamp uinteger, + height uinteger, + offerPoolAmount ubigint, + askPoolAmount ubigint) + ` + ); + } + + async insertPoolAmountHistory(ops: PoolAmountHistory[]) { + try { + await this.insertBulkData(ops, "lp_amount_history"); + } catch (error) { + console.dir({ ops, error }, { depth: null }); + } + } + + async createAprInfoPair() { + await this.conn.exec( + `CREATE TABLE IF NOT EXISTS pool_apr ( + uniqueKey varchar UNIQUE, + pairAddr varchar, + timestamp uinteger, + height uinteger, + totalSupply ubigint, + totalBondAmount ubigint, + rewardPerSec varchar, + ) + ` + ); + } } diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index def535df..30aa505c 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -11,7 +11,16 @@ import { ORAI, atomic, oraiInfo, tenAmountInDecimalSix, truncDecimals, usdtInfo import { DuckDb } from "./db"; import { pairs, pairsOnlyDenom } from "./pairs"; import { getPriceAssetByUsdt } from "./pool-helper"; -import { LpOpsData, Ohlcv, OraiDexType, PairInfoData, PoolInfo, SwapDirection, SwapOperationData } from "./types"; +import { + LpOpsData, + Ohlcv, + OraiDexType, + PairInfoData, + PoolAmountHistory, + PoolInfo, + SwapDirection, + SwapOperationData +} from "./types"; export function toObject(data: any) { return JSON.parse( @@ -78,6 +87,10 @@ export const concatOhlcvToUniqueKey = (data: { timestamp: number; pair: string; return `${data.timestamp}-${data.pair}-${data.volume.toString()}`; }; +export const concatLpHistoryToUniqueKey = (data: { timestamp: number; pairAddr: string }): string => { + return `${data.timestamp}-${data.pairAddr}`; +}; + export function isoToTimestampNumber(time: string) { return Math.floor(new Date(time).getTime() / 1000); } @@ -216,10 +229,7 @@ export function isAssetInfoPairReverse(assetInfos: AssetInfo[]): boolean { */ export const collectAccumulateLpAndSwapData = async (data: LpOpsData[], poolInfos: PoolResponse[]) => { let accumulateData: { - [key: string]: { - baseTokenAmount: bigint; - quoteTokenAmount: bigint; - }; + [key: string]: Omit; } = {}; const duckDb = DuckDb.instances; for (let op of data) { @@ -255,12 +265,16 @@ export const collectAccumulateLpAndSwapData = async (data: LpOpsData[], poolInfo ); accumulateData[pairAddr] = { - baseTokenAmount: BigInt(initialFirstTokenAmount) + baseAmount, - quoteTokenAmount: BigInt(initialSecondTokenAmount) + quoteAmount + offerPoolAmount: BigInt(initialFirstTokenAmount) + baseAmount, + askPoolAmount: BigInt(initialSecondTokenAmount) + quoteAmount, + height: op.height, + timestamp: op.timestamp }; } else { - accumulateData[pairAddr].baseTokenAmount += baseAmount; - accumulateData[pairAddr].quoteTokenAmount += quoteAmount; + accumulateData[pairAddr].offerPoolAmount += baseAmount; + accumulateData[pairAddr].askPoolAmount += quoteAmount; + accumulateData[pairAddr].height = op.height; + accumulateData[pairAddr].timestamp = op.timestamp; } } diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index cd80aefb..f3148864 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -1,7 +1,7 @@ import { SyncData, Txs, WriteData } from "@oraichain/cosmos-rpc-sync"; import "dotenv/config"; import { DuckDb } from "./db"; -import { collectAccumulateLpAndSwapData, getSymbolFromAsset } from "./helper"; +import { collectAccumulateLpAndSwapData, concatLpHistoryToUniqueKey, getSymbolFromAsset } from "./helper"; import { getAllPairInfos, getPairByAssetInfos, getPoolInfos } from "./pool-helper"; import { parseAssetInfo, parseTxs } from "./tx-parsing"; import { @@ -55,7 +55,9 @@ class WriteOrders extends WriteData { baseTokenDenom: item.baseTokenDenom, quoteTokenAmount: item.quoteTokenAmount, quoteTokenDenom: item.quoteTokenDenom, - opType: item.opType + opType: item.opType, + timestamp: item.timestamp, + height: item.txheight } as LpOpsData; }), ...swapData.map((item) => { @@ -64,26 +66,28 @@ class WriteOrders extends WriteData { baseTokenDenom: item.offerDenom, quoteTokenAmount: -item.returnAmount, // reverse sign because we assume first case is sell, check buy later. quoteTokenDenom: item.askDenom, - direction: item.direction + direction: item.direction, + height: item.txheight, + timestamp: item.timestamp } as LpOpsData; }) ]; const accumulatedData = await collectAccumulateLpAndSwapData(lpOpsData, poolInfos); - // update new lp pool amount to pair_infos - await Promise.all( - pairInfos - .map(({ pairAddr }) => { - if (accumulatedData[pairAddr]) { - return this.duckDb.updatePairInfoAmount( - accumulatedData[pairAddr].baseTokenAmount, - accumulatedData[pairAddr].quoteTokenAmount, - pairAddr - ); - } - }) - .filter(Boolean) - ); + const poolAmountHitories = pairInfos.reduce((accumulator, { pairAddr }) => { + if (accumulatedData[pairAddr]) { + accumulator.push({ + ...accumulatedData[pairAddr], + pairAddr, + uniqueKey: concatLpHistoryToUniqueKey({ + timestamp: accumulatedData[pairAddr].timestamp, + pairAddr + }) + }); + } + return accumulator; + }, []); + await this.duckDb.insertPoolAmountHistory(poolAmountHitories); } async process(chunk: any): Promise { @@ -165,7 +169,8 @@ class OraiDexSync { this.duckDb.createLiquidityOpsTable(), this.duckDb.createSwapOpsTable(), this.duckDb.createPairInfosTable(), - this.duckDb.createSwapOhlcv() + this.duckDb.createSwapOhlcv(), + this.duckDb.createPoolOpsTable() ]); let currentInd = await this.duckDb.loadHeightSnapshot(); let initialData: InitialData = { tokenPrices: [], blockHeader: undefined }; @@ -180,7 +185,7 @@ class OraiDexSync { offset: currentInd, rpcUrl: this.rpcUrl, queryTags: [], - limit: parseInt(process.env.LIMIT) || 100, + limit: 10, maxThreadLevel: parseInt(process.env.MAX_THREAD_LEVEL) || 3, interval: 5000 }).pipe(new WriteOrders(this.duckDb, this.rpcUrl, this.env, initialData)); @@ -196,7 +201,7 @@ async function initSync() { oraidexSync.sync(); } -// initSync(); +initSync(); export { OraiDexSync }; diff --git a/packages/oraidex-sync/src/types.ts b/packages/oraidex-sync/src/types.ts index dbe99769..0d6ee0e9 100644 --- a/packages/oraidex-sync/src/types.ts +++ b/packages/oraidex-sync/src/types.ts @@ -94,6 +94,8 @@ export type LpOpsData = { quoteTokenDenom: string; opType?: LiquidityOpType; direction?: SwapDirection; + height: number; + timestamp: number; }; export type TxAnlysisResult = { @@ -225,3 +227,12 @@ export type PoolInfo = { offerPoolAmount: bigint; askPoolAmount: bigint; }; + +export type PoolAmountHistory = { + offerPoolAmount: bigint; + askPoolAmount: bigint; + timestamp: number; + height: number; + pairAddr: string; + uniqueKey: string; +}; From f54d0587fa96938d8223038891548e93b0903b32 Mon Sep 17 00:00:00 2001 From: trungbach Date: Wed, 13 Sep 2023 18:55:38 +0700 Subject: [PATCH 41/46] in process accumulate apr --- packages/oraidex-server/src/index.ts | 6 +- packages/oraidex-sync/src/db.ts | 64 ++++++++++++---- packages/oraidex-sync/src/helper.ts | 10 +++ packages/oraidex-sync/src/index.ts | 70 +++++++++++++++-- packages/oraidex-sync/src/pool-helper.ts | 95 +++++++++++++++++++++++- packages/oraidex-sync/src/query.ts | 8 +- packages/oraidex-sync/src/tx-parsing.ts | 35 ++++++++- packages/oraidex-sync/src/types.ts | 10 +++ packages/oraidex-sync/tests/db.spec.ts | 75 ++++++++++++++++++- 9 files changed, 338 insertions(+), 35 deletions(-) diff --git a/packages/oraidex-server/src/index.ts b/packages/oraidex-server/src/index.ts index c16da11a..05aac142 100644 --- a/packages/oraidex-server/src/index.ts +++ b/packages/oraidex-server/src/index.ts @@ -10,7 +10,6 @@ import { PairInfoDataResponse, TickerInfo, VolumeRange, - fetchAprResult, findPairAddress, getAllFees, getAllVolume24h, @@ -240,19 +239,20 @@ app.get("/v1/pools/", async (_req, res) => { const [volumes, allFee7Days] = await Promise.all([getAllVolume24h(), getAllFees()]); const pools = await duckDb.getPools(); + const allPoolApr = await duckDb.getApr(); const allLiquidities = (await Promise.allSettled(pools.map((pair) => getPairLiquidity(pair)))).map((result) => { if (result.status === "fulfilled") return result.value; else console.error("error get allLiquidities: ", result.reason); }); - const allApr = await fetchAprResult(pools, allLiquidities); res.status(200).send( pools.map((pool, index) => { + const poolApr = allPoolApr.find((item) => item.pairAddr === pool.pairAddr); return { ...pool, volume24Hour: volumes[index]?.toString() ?? "0", fee7Days: allFee7Days[index]?.toString() ?? "0", - apr: allApr[index], + apr: poolApr?.apr ?? 0, totalLiquidity: allLiquidities[index] } as PairInfoDataResponse; }) diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index 0a4e2cd1..d8af136a 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -12,8 +12,8 @@ import { GetCandlesQuery, GetFeeSwap, GetVolumeQuery, - PoolInfo, - PoolAmountHistory + PoolAmountHistory, + PoolApr } from "./types"; import fs, { rename } from "fs"; import { @@ -504,22 +504,18 @@ export class DuckDb { async createPoolOpsTable() { await this.conn.exec( `CREATE TABLE IF NOT EXISTS lp_amount_history ( - uniqueKey varchar UNIQUE, - pairAddr varchar, - timestamp uinteger, - height uinteger, - offerPoolAmount ubigint, - askPoolAmount ubigint) + offerPoolAmount ubigint, + askPoolAmount ubigint, + height uinteger, + timestamp uinteger, + pairAddr varchar, + uniqueKey varchar UNIQUE) ` ); } async insertPoolAmountHistory(ops: PoolAmountHistory[]) { - try { - await this.insertBulkData(ops, "lp_amount_history"); - } catch (error) { - console.dir({ ops, error }, { depth: null }); - } + await this.insertBulkData(ops, "lp_amount_history"); } async createAprInfoPair() { @@ -527,13 +523,49 @@ export class DuckDb { `CREATE TABLE IF NOT EXISTS pool_apr ( uniqueKey varchar UNIQUE, pairAddr varchar, - timestamp uinteger, height uinteger, - totalSupply ubigint, - totalBondAmount ubigint, + totalSupply varchar, + totalBondAmount varchar, rewardPerSec varchar, + apr double ) ` ); } + + async insertPoolAprs(poolAprs: PoolApr[]) { + console.dir({ poolAprs }, { depth: null }); + await this.insertBulkData(poolAprs, "pool_apr"); + } + + async getLatestPoolApr(pairAddr: string): Promise { + const result = await this.conn.all( + ` + SELECT * FROM pool_apr + WHERE pairAddr = ? + ORDER BY height DESC + LIMIT 1 + `, + pairAddr + ); + + return result[0] as PoolApr; + } + + async getApr() { + const result = await this.conn.all( + ` + SELECT p.pairAddr, p.apr + FROM pool_apr p + JOIN ( + SELECT pairAddr, MAX(height) AS max_height + FROM pool_apr + GROUP BY pairAddr + ) max_heights + ON p.pairAddr = max_heights.pairAddr AND p.height = max_heights.max_height + ORDER BY p.height DESC + ` + ); + return result as Pick[]; + } } diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index 30aa505c..44a82758 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -513,6 +513,16 @@ async function getAllFees(): Promise { } // ==== end get fee pair ====> +export const parsePairDenomToAssetInfo = ([baseDenom, quoteDenom]: [string, string]): [AssetInfo, AssetInfo] => { + const pair = pairs.find( + (pair) => + parseAssetInfoOnlyDenom(pair.asset_infos[0]) === baseDenom && + parseAssetInfoOnlyDenom(pair.asset_infos[1]) === quoteDenom + ); + if (!pair) throw new Error(`cannot find pair for ${baseDenom}-$${quoteDenom}`); + return pair.asset_infos; +}; + export { calculatePriceByPool, convertDateToSecond, diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index f3148864..5d2386a7 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -1,19 +1,34 @@ import { SyncData, Txs, WriteData } from "@oraichain/cosmos-rpc-sync"; import "dotenv/config"; import { DuckDb } from "./db"; -import { collectAccumulateLpAndSwapData, concatLpHistoryToUniqueKey, getSymbolFromAsset } from "./helper"; -import { getAllPairInfos, getPairByAssetInfos, getPoolInfos } from "./pool-helper"; -import { parseAssetInfo, parseTxs } from "./tx-parsing"; +import { + collectAccumulateLpAndSwapData, + concatLpHistoryToUniqueKey, + getPairLiquidity, + getSymbolFromAsset, + parsePairDenomToAssetInfo +} from "./helper"; +import { + calculateAprResult, + fetchAprResult, + getAllPairInfos, + getPairByAssetInfos, + getPoolInfos, + refetchTokenInfos +} from "./pool-helper"; +import { parseAssetInfo, parseTxs, processEventApr } from "./tx-parsing"; import { Env, InitialData, LpOpsData, PairInfoData, + PoolApr, ProvideLiquidityOperationData, SwapOperationData, TxAnlysisResult, WithdrawLiquidityOperationData } from "./types"; +import { AssetInfo } from "@oraichain/common-contracts-sdk"; class WriteOrders extends WriteData { constructor(private duckDb: DuckDb, private rpcUrl: string, private env: Env, private initialData: InitialData) { @@ -104,6 +119,19 @@ class WriteOrders extends WriteData { [...result.swapOpsData] ); + const assetInfosToRefetchTokenInfos = Array.from( + [...result.provideLiquidityOpsData, ...result.withdrawLiquidityOpsData] + .map((op) => [op.baseTokenDenom, op.quoteTokenDenom] as [string, string]) + .reduce((accumulator, tokenDenoms) => { + const assetInfo = parsePairDenomToAssetInfo(tokenDenoms); + accumulator.add(assetInfo); + return accumulator; + }, new Set<[AssetInfo, AssetInfo]>()) + ); + await refetchTokenInfos(assetInfosToRefetchTokenInfos, newOffset); + + await processEventApr(txs); + // collect the latest offer & ask volume to accumulate the results // insert txs console.log("new offset: ", newOffset); @@ -129,7 +157,7 @@ class OraiDexSync { return new OraiDexSync(duckDb, rpcUrl, env); } - private async updateLatestPairInfos(currentHeight: number) { + private async updateLatestPairInfos() { try { console.time("timer-updateLatestPairInfos"); const pairInfos = await getAllPairInfos(); @@ -162,6 +190,29 @@ class OraiDexSync { } } + private async updateLatestPoolApr(height: number) { + const pools = await this.duckDb.getPools(); + const allLiquidities = (await Promise.allSettled(pools.map((pair) => getPairLiquidity(pair)))).map((result) => { + if (result.status === "fulfilled") return result.value; + else console.error("error get allLiquidities: ", result.reason); + }); + const { allTotalSupplies, allBondAmounts, allRewardPerSec } = await fetchAprResult(pools, allLiquidities); + const allAprs = await calculateAprResult(allLiquidities, allTotalSupplies, allBondAmounts, allRewardPerSec); + + const poolAprs = allAprs.map((apr, index) => { + return { + uniqueKey: concatLpHistoryToUniqueKey({ timestamp: height, pairAddr: pools[index].pairAddr }), + pairAddr: pools[index].pairAddr, + height, + totalSupply: allTotalSupplies[index], + totalBondAmount: allBondAmounts[index], + rewardPerSec: JSON.stringify(allRewardPerSec[index]), + apr + } as PoolApr; + }); + await this.duckDb.insertPoolAprs(poolAprs); + } + public async sync() { try { await Promise.all([ @@ -170,7 +221,8 @@ class OraiDexSync { this.duckDb.createSwapOpsTable(), this.duckDb.createPairInfosTable(), this.duckDb.createSwapOhlcv(), - this.duckDb.createPoolOpsTable() + this.duckDb.createPoolOpsTable(), + this.duckDb.createAprInfoPair() ]); let currentInd = await this.duckDb.loadHeightSnapshot(); let initialData: InitialData = { tokenPrices: [], blockHeader: undefined }; @@ -180,12 +232,16 @@ class OraiDexSync { currentInd = initialSyncHeight; } console.log("current ind: ", currentInd); - await this.updateLatestPairInfos(currentInd); + await this.updateLatestPairInfos(); + + // update apr in the first time + await this.updateLatestPoolApr(currentInd); + new SyncData({ offset: currentInd, rpcUrl: this.rpcUrl, queryTags: [], - limit: 10, + limit: 1000, maxThreadLevel: parseInt(process.env.MAX_THREAD_LEVEL) || 3, interval: 5000 }).pipe(new WriteOrders(this.duckDb, this.rpcUrl, this.env, initialData)); diff --git a/packages/oraidex-sync/src/pool-helper.ts b/packages/oraidex-sync/src/pool-helper.ts index 2f535cdd..ad0272f9 100644 --- a/packages/oraidex-sync/src/pool-helper.ts +++ b/packages/oraidex-sync/src/pool-helper.ts @@ -234,23 +234,48 @@ export const getStakingAssetInfo = (assetInfos: AssetInfo[]): AssetInfo => { return parseAssetInfoOnlyDenom(assetInfos[0]) === ORAI ? assetInfos[1] : assetInfos[0]; }; -export const fetchAprResult = async (pairInfos: PairInfoData[], allLiquidities: number[]): Promise => { +// export const fetchAprResult = async (pairInfos: PairInfoData[], allLiquidities: number[]): Promise => { +// const assetTokens = pairInfos.map((pair) => +// getStakingAssetInfo([JSON.parse(pair.firstAssetInfo), JSON.parse(pair.secondAssetInfo)]) +// ); +// const liquidityAddrs = pairInfos.map((pair) => pair.liquidityAddr); +// try { +// const [allTokenInfo, allLpTokenAsset, allRewardPerSec] = await Promise.all([ +// fetchTokenInfos(liquidityAddrs), +// fetchAllTokenAssetPools(assetTokens), +// fetchAllRewardPerSecInfos(assetTokens) +// ]); +// const allTotalSupplies = allTokenInfo.map((info) => info.total_supply); +// const allBondAmounts = allLpTokenAsset.map((info) => info.total_bond_amount); +// return calculateAprResult(allLiquidities, allTotalSupplies, allBondAmounts, allRewardPerSec); +// } catch (error) { +// console.log({ errorFetchAprResult: error }); +// } +// }; + +export const fetchAprResult = async (pairInfos: PairInfoData[], allLiquidities: number[]) => { const assetTokens = pairInfos.map((pair) => getStakingAssetInfo([JSON.parse(pair.firstAssetInfo), JSON.parse(pair.secondAssetInfo)]) ); + const liquidityAddrs = pairInfos.map((pair) => pair.liquidityAddr); try { const [allTokenInfo, allLpTokenAsset, allRewardPerSec] = await Promise.all([ - fetchTokenInfos(pairInfos), + fetchTokenInfos(liquidityAddrs), fetchAllTokenAssetPools(assetTokens), fetchAllRewardPerSecInfos(assetTokens) ]); const allTotalSupplies = allTokenInfo.map((info) => info.total_supply); const allBondAmounts = allLpTokenAsset.map((info) => info.total_bond_amount); - return calculateAprResult(allLiquidities, allTotalSupplies, allBondAmounts, allRewardPerSec); + return { + allTotalSupplies, + allBondAmounts, + allRewardPerSec + }; } catch (error) { console.log({ errorFetchAprResult: error }); } }; + // ==== end of calculate APR ====> export const getAllPairInfos = async (): Promise => { @@ -259,3 +284,67 @@ export const getAllPairInfos = async (): Promise => { const secondFactoryClient = new OraiswapFactoryQueryClient(cosmwasmClient, network.factory_v2); return queryAllPairInfos(firstFactoryClient, secondFactoryClient); }; + +export const triggerCalculateApr = async () => { + // TODO: get all infos relate to apr in duckdb from apr table -> call to calculateAprResult +}; + +export type SupplyAction = "mint" | "burn"; +export type BondAction = "bond" | "unbond" | "deposit_reward"; +export type RewardPerSecAction = "update_reward_per_sec"; + +export const refetchTokenInfos = async (assetInfos: [AssetInfo, AssetInfo][], height: number) => { + const duckDb = DuckDb.instances; + const pools = await Promise.all(assetInfos.map((assetInfo) => duckDb.getPoolByAssetInfos(assetInfo))); + const liquidityAddrs = pools.map((pair) => pair.liquidityAddr); + const tokenInfos = await fetchTokenInfos(liquidityAddrs); + const totalSupplies = tokenInfos.map((info) => info.total_supply); + + const latestPoolAprs = await Promise.all(pools.map((pool) => duckDb.getLatestPoolApr(pool.pairAddr))); + const newPoolAprs = latestPoolAprs.map((poolApr, index) => { + return { + ...poolApr, + height, + totalSupply: totalSupplies[index] + }; + }); + await duckDb.insertPoolAprs(newPoolAprs); +}; + +export const refetchTokenAssetPools = async (assetInfos: [AssetInfo, AssetInfo][], height: number) => { + const duckDb = DuckDb.instances; + const pools = await Promise.all(assetInfos.map((assetInfo) => duckDb.getPoolByAssetInfos(assetInfo))); + const assetTokens = pools.map((pair) => + getStakingAssetInfo([JSON.parse(pair.firstAssetInfo), JSON.parse(pair.secondAssetInfo)]) + ); + const tokenAssetPools = await fetchAllTokenAssetPools(assetTokens); + const totalBondAmounts = tokenAssetPools.map((info) => info.total_bond_amount); + const latestPoolAprs = await Promise.all(pools.map((pool) => duckDb.getLatestPoolApr(pool.pairAddr))); + const newPoolAprs = latestPoolAprs.map((poolApr, index) => { + return { + ...poolApr, + height, + totalBondAmount: totalBondAmounts[index] + }; + }); + await duckDb.insertPoolAprs(newPoolAprs); +}; + +export const refetchRewardPerSecInfos = async (assetInfos: [AssetInfo, AssetInfo][], height: number) => { + const duckDb = DuckDb.instances; + const pools = await Promise.all(assetInfos.map((assetInfo) => duckDb.getPoolByAssetInfos(assetInfo))); + const assetTokens = pools.map((pair) => + getStakingAssetInfo([JSON.parse(pair.firstAssetInfo), JSON.parse(pair.secondAssetInfo)]) + ); + const rewardPerSecInfos = await fetchAllRewardPerSecInfos(assetTokens); + + const latestPoolAprs = await Promise.all(pools.map((pool) => duckDb.getLatestPoolApr(pool.pairAddr))); + const newPoolAprs = latestPoolAprs.map((poolApr, index) => { + return { + ...poolApr, + height, + rewardPerSec: JSON.stringify(rewardPerSecInfos[index]) + }; + }); + await duckDb.insertPoolAprs(newPoolAprs); +}; diff --git a/packages/oraidex-sync/src/query.ts b/packages/oraidex-sync/src/query.ts index bb913588..0a7537da 100644 --- a/packages/oraidex-sync/src/query.ts +++ b/packages/oraidex-sync/src/query.ts @@ -7,7 +7,7 @@ import { } from "@oraichain/oraidex-contracts-sdk"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; import { Asset, AssetInfo } from "@oraichain/oraidex-contracts-sdk"; -import { Call, MulticallQueryClient, MulticallReadOnlyInterface } from "@oraichain/common-contracts-sdk"; +import { Addr, Call, MulticallQueryClient, MulticallReadOnlyInterface } from "@oraichain/common-contracts-sdk"; import { fromBinary, toBinary } from "@cosmjs/cosmwasm-stargate"; import { pairs } from "./pairs"; import { @@ -92,9 +92,9 @@ async function aggregateMulticall(queries: Call[]) { return res.return_data.map((data) => (data.success ? fromBinary(data.data) : undefined)); } -async function fetchTokenInfos(pairInfos: PairInfoData[]): Promise { - const queries = pairInfos.map((pair) => ({ - address: pair.liquidityAddr, +async function fetchTokenInfos(liquidityAddrs: Addr[]): Promise { + const queries = liquidityAddrs.map((address) => ({ + address, data: toBinary({ token_info: {} } as OraiswapTokenTypes.QueryMsg) diff --git a/packages/oraidex-sync/src/tx-parsing.ts b/packages/oraidex-sync/src/tx-parsing.ts index e3bf611e..ee7d853b 100644 --- a/packages/oraidex-sync/src/tx-parsing.ts +++ b/packages/oraidex-sync/src/tx-parsing.ts @@ -32,7 +32,7 @@ import { } from "./helper"; import { pairs } from "./pairs"; import { DuckDb } from "./db"; -import { calculateLiquidityFee, isPoolHasFee } from "./pool-helper"; +import { calculateLiquidityFee, isPoolHasFee, refetchTokenInfos, triggerCalculateApr } from "./pool-helper"; function parseWasmEvents(events: readonly Event[]): (readonly Attribute[])[] { return events.filter((event) => event.type === "wasm").map((event) => event.attributes); @@ -329,6 +329,7 @@ async function parseTxs(txs: Tx[]): Promise { for (let msg of msgs) { const sender = msg.sender; const wasmAttributes = parseWasmEvents(msg.logs.events); + swapOpsData.push(...extractSwapOperations(basicTxData, wasmAttributes)); const provideLiquidityData = await extractMsgProvideLiquidity(basicTxData, msg.msg, sender, wasmAttributes); if (provideLiquidityData) provideLiquidityOpsData.push(provideLiquidityData); @@ -351,4 +352,36 @@ async function parseTxs(txs: Tx[]): Promise { }; } +export const processEventApr = async (txs: Tx[]) => { + const assets = { + infoTokenAssetPools: new Set(), + infoRewardPerSec: new Set() + }; + + for (let tx of txs) { + const msgExecuteContracts = parseTxToMsgExecuteContractMsgs(tx); + const msgs = parseExecuteContractToOraidexMsgs(msgExecuteContracts); + + for (let msg of msgs) { + const wasmAttributes = parseWasmEvents(msg.logs.events); + for (let attrs of wasmAttributes) { + if ( + attrs.find( + (attr) => + attr.key === "action" && + (attr.value === "bond" || attr.value === "unbond" || attr.value === "deposit_reward") + ) + ) { + console.table(attrs); + // TODO: FIND ASSET THEN ADD TO LIST ASSET + // assets.infoTokenAssetPools.add() + } + + // if (attrs.find((attr) => attr.key === "action" && attr.value === "update_reward_per_sec")) + } + } + } + return assets; +}; + export { parseAssetInfo, parseWasmEvents, parseTxs, parseWithdrawLiquidityAssets, parseTxToMsgExecuteContractMsgs }; diff --git a/packages/oraidex-sync/src/types.ts b/packages/oraidex-sync/src/types.ts index 0d6ee0e9..40d9891f 100644 --- a/packages/oraidex-sync/src/types.ts +++ b/packages/oraidex-sync/src/types.ts @@ -236,3 +236,13 @@ export type PoolAmountHistory = { pairAddr: string; uniqueKey: string; }; + +export type PoolApr = { + uniqueKey: string; + pairAddr: string; + height: number; + totalSupply: string; + totalBondAmount: string; + rewardPerSec: string; + apr: number; +}; diff --git a/packages/oraidex-sync/tests/db.spec.ts b/packages/oraidex-sync/tests/db.spec.ts index 302fabb6..c2a81763 100644 --- a/packages/oraidex-sync/tests/db.spec.ts +++ b/packages/oraidex-sync/tests/db.spec.ts @@ -2,7 +2,7 @@ import fs from "fs"; import { oraiInfo, usdtInfo } from "../src"; import { DuckDb } from "../src/db"; import { isoToTimestampNumber } from "../src/helper"; -import { GetFeeSwap, GetVolumeQuery, PairInfoData, ProvideLiquidityOperationData } from "../src/types"; +import { GetFeeSwap, GetVolumeQuery, PairInfoData, PoolApr, ProvideLiquidityOperationData } from "../src/types"; describe("test-duckdb", () => { let duckDb: DuckDb; afterAll(jest.resetModules); @@ -475,4 +475,77 @@ describe("test-duckdb", () => { expect(volumeByBaseAsset).toEqual(2n); }); }); + + describe("test-apr", () => { + beforeEach(async () => { + // setup + duckDb = await DuckDb.create(":memory:"); + await duckDb.createAprInfoPair(); + await duckDb.insertPoolAprs([ + { + uniqueKey: "orai_usdt_2", + pairAddr: "orai_usdt", + height: 2, + totalSupply: "1", + totalBondAmount: "1", + rewardPerSec: "1", + apr: 2 + } as PoolApr, + { + uniqueKey: "orai_usdt_4", + pairAddr: "orai_usdt", + height: 4, + totalSupply: "1", + totalBondAmount: "1", + rewardPerSec: "1", + apr: 4 + } as PoolApr, + { + uniqueKey: "orai_usdt_3", + pairAddr: "orai_usdt", + height: 3, + totalSupply: "1", + totalBondAmount: "1", + rewardPerSec: "1", + apr: 3 + } as PoolApr, + { + uniqueKey: "orai_atom", + pairAddr: "orai_atom", + height: 2, + totalSupply: "1", + totalBondAmount: "1", + rewardPerSec: "1", + apr: 2 + } as PoolApr + ]); + }); + + it("test-getApr-should-return-correctly-apr-for-all-pair", async () => { + // act + const apr = await duckDb.getApr(); + + // assertion + expect(apr).toEqual([ + { pairAddr: "orai_usdt", apr: 4 }, + { pairAddr: "orai_atom", apr: 2 } + ]); + }); + + it("test-getLatestPoolApr-should-return-latest-pool-apr", async () => { + // act + const result = await duckDb.getLatestPoolApr("orai_usdt"); + + // assertion + expect(result).toEqual({ + uniqueKey: "orai_usdt_4", + pairAddr: "orai_usdt", + height: 4, + totalSupply: "1", + totalBondAmount: "1", + rewardPerSec: "1", + apr: 4 + }); + }); + }); }); From 886a9fba51d01f1b724da067de06434a974c3b6f Mon Sep 17 00:00:00 2001 From: trungbach Date: Thu, 14 Sep 2023 00:43:31 +0700 Subject: [PATCH 42/46] use pool amount from lp_amount instead from pair_infos --- packages/oraidex-sync/src/db.ts | 14 ++++++++- packages/oraidex-sync/src/helper.ts | 7 +++-- packages/oraidex-sync/src/pairs.ts | 8 +++++ packages/oraidex-sync/src/pool-helper.ts | 7 +++-- packages/oraidex-sync/src/tx-parsing.ts | 38 ++++++++++++++++++++---- 5 files changed, 62 insertions(+), 12 deletions(-) diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index d8af136a..770837c3 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -514,6 +514,19 @@ export class DuckDb { ); } + async getLatestLpPoolAmount(pairAddr: string) { + const result = await this.conn.all( + ` + SELECT * FROM lp_amount_history + WHERE pairAddr = ? + ORDER BY height DESC + LIMIT 1 + `, + pairAddr + ); + return result[0] as PoolAmountHistory; + } + async insertPoolAmountHistory(ops: PoolAmountHistory[]) { await this.insertBulkData(ops, "lp_amount_history"); } @@ -534,7 +547,6 @@ export class DuckDb { } async insertPoolAprs(poolAprs: PoolApr[]) { - console.dir({ poolAprs }, { depth: null }); await this.insertBulkData(poolAprs, "pool_apr"); } diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index 44a82758..cc23c27d 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -409,12 +409,13 @@ async function fetchPoolInfoAmount(fromInfo: AssetInfo, toInfo: AssetInfo, pairA // get liquidity of pair from assetInfos export const getPairLiquidity = async (poolInfo: PairInfoData): Promise => { - // get info of pool in pair_infos, ask & offer are accumulated in sync process (via swap ops and lp ops). - if (!poolInfo.askPoolAmount || !poolInfo.offerPoolAmount) return 0; + const duckDb = DuckDb.instances; + const poolAmount = await duckDb.getLatestLpPoolAmount(poolInfo.pairAddr); + if (!poolAmount || !poolAmount.askPoolAmount || !poolAmount.offerPoolAmount) return 0; const baseAssetInfo = JSON.parse(poolInfo.firstAssetInfo); const priceBaseAssetInUsdt = await getPriceAssetByUsdt(baseAssetInfo); - return priceBaseAssetInUsdt * Number(poolInfo.offerPoolAmount) * 2; + return priceBaseAssetInUsdt * Number(poolAmount.offerPoolAmount) * 2; }; /** diff --git a/packages/oraidex-sync/src/pairs.ts b/packages/oraidex-sync/src/pairs.ts index c31ed19b..25d2a340 100644 --- a/packages/oraidex-sync/src/pairs.ts +++ b/packages/oraidex-sync/src/pairs.ts @@ -16,6 +16,7 @@ import { usdtCw20Address } from "./constants"; import { PairMapping } from "./types"; +import { getStakingAssetInfo } from "./pool-helper"; // the orders are important! Do not change the order of the asset_infos. export const pairs: PairMapping[] = [ @@ -108,3 +109,10 @@ export const uniqueInfos = extractUniqueAndFlatten(pairs); export const oraiUsdtPairOnlyDenom = pairsOnlyDenom.find( (pair) => JSON.stringify(pair.asset_infos) === JSON.stringify([ORAI, usdtCw20Address]) ).asset_infos; + +export const pairWithStakingAsset = pairs.map((pair) => { + return { + ...pair, + stakingAssetInfo: getStakingAssetInfo(pair.asset_infos) + }; +}); diff --git a/packages/oraidex-sync/src/pool-helper.ts b/packages/oraidex-sync/src/pool-helper.ts index ad0272f9..d66e0ae0 100644 --- a/packages/oraidex-sync/src/pool-helper.ts +++ b/packages/oraidex-sync/src/pool-helper.ts @@ -84,12 +84,13 @@ export const getPriceByAsset = async ( ): Promise => { const duckDb = DuckDb.instances; const poolInfo = await duckDb.getPoolByAssetInfos(assetInfos); - if (!poolInfo || !poolInfo.askPoolAmount || !poolInfo.offerPoolAmount) return 0; + const poolAmount = await duckDb.getLatestLpPoolAmount(poolInfo.pairAddr); + if (!poolAmount || !poolInfo.askPoolAmount || !poolInfo.offerPoolAmount) return 0; // offer: orai, ask: usdt -> price offer in ask = calculatePriceByPool([ask, offer]) // offer: orai, ask: atom -> price ask in offer = calculatePriceByPool([offer, ask]) const basePrice = calculatePriceByPool( - BigInt(poolInfo.askPoolAmount), - BigInt(poolInfo.offerPoolAmount), + BigInt(poolAmount.askPoolAmount), + BigInt(poolAmount.offerPoolAmount), +poolInfo.commissionRate ); return ratioDirection === "base_in_quote" ? basePrice : 1 / basePrice; diff --git a/packages/oraidex-sync/src/tx-parsing.ts b/packages/oraidex-sync/src/tx-parsing.ts index ee7d853b..85cbe215 100644 --- a/packages/oraidex-sync/src/tx-parsing.ts +++ b/packages/oraidex-sync/src/tx-parsing.ts @@ -288,14 +288,33 @@ function parseExecuteContractToOraidexMsgs(msgs: MsgExecuteContractWithLogs[]): msg: JSON.parse(Buffer.from(msg.msg).toString("utf-8")) }; // Should be provide, remove liquidity, swap, or other oraidex related types - if ("provide_liquidity" in obj.msg || "execute_swap_operations" in obj.msg || "execute_swap_operation" in obj.msg) + if ( + "provide_liquidity" in obj.msg || + "execute_swap_operations" in obj.msg || + "execute_swap_operation" in obj.msg || + "bond" in obj.msg || + "unbond" in obj.msg || + "update_reward_per_sec" in obj.msg || + "mint" in obj.msg || + "burn" in obj.msg || + "deposit_reward" in obj.msg + ) objs.push(obj); if ("send" in obj.msg) { try { const contractSendMsg: OraiswapPairCw20HookMsg | OraiswapRouterCw20HookMsg = JSON.parse( Buffer.from(obj.msg.send.msg, "base64").toString("utf-8") ); - if ("execute_swap_operations" in contractSendMsg || "withdraw_liquidity" in contractSendMsg) { + if ( + "execute_swap_operations" in contractSendMsg || + "withdraw_liquidity" in contractSendMsg || + "bond" in contractSendMsg || + "unbond" in contractSendMsg || + "update_reward_per_sec" in contractSendMsg || + "mint" in contractSendMsg || + "burn" in contractSendMsg || + "deposit_reward" in contractSendMsg + ) { objs.push({ ...msg, msg: contractSendMsg }); } } catch (error) { @@ -357,14 +376,19 @@ export const processEventApr = async (txs: Tx[]) => { infoTokenAssetPools: new Set(), infoRewardPerSec: new Set() }; - for (let tx of txs) { const msgExecuteContracts = parseTxToMsgExecuteContractMsgs(tx); const msgs = parseExecuteContractToOraidexMsgs(msgExecuteContracts); - for (let msg of msgs) { const wasmAttributes = parseWasmEvents(msg.logs.events); for (let attrs of wasmAttributes) { + // if (attrs.find((attr) => attr.key === "action" && (attr.value === "mint" || attr.value === "burn"))) { + // console.log("mint-burn"); + // console.table(attrs); + // // TODO: FIND ASSET THEN ADD TO LIST ASSET + // // assets.infoTokenAssetPools.add() + // } + if ( attrs.find( (attr) => @@ -372,12 +396,16 @@ export const processEventApr = async (txs: Tx[]) => { (attr.value === "bond" || attr.value === "unbond" || attr.value === "deposit_reward") ) ) { + console.log("bond-unbond"); console.table(attrs); // TODO: FIND ASSET THEN ADD TO LIST ASSET // assets.infoTokenAssetPools.add() } - // if (attrs.find((attr) => attr.key === "action" && attr.value === "update_reward_per_sec")) + if (attrs.find((attr) => attr.key === "action" && attr.value === "update_reward_per_sec")) { + console.log("update_reward_per_sec"); + console.table(attrs); + } } } } From 0469fedebf6c250bf4a21ee8f4ae32d57dc6d8d7 Mon Sep 17 00:00:00 2001 From: trungbach Date: Thu, 14 Sep 2023 18:38:52 +0700 Subject: [PATCH 43/46] updated calculate apr without test --- packages/oraidex-sync/src/db.ts | 15 +- packages/oraidex-sync/src/helper.ts | 18 ++- packages/oraidex-sync/src/index.ts | 24 +--- packages/oraidex-sync/src/pairs.ts | 12 +- packages/oraidex-sync/src/pool-helper.ts | 130 +++++++++++++----- packages/oraidex-sync/src/tx-parsing.ts | 79 +++++------ packages/oraidex-sync/src/types.ts | 1 + packages/oraidex-sync/tests/helper.spec.ts | 63 ++++++--- .../oraidex-sync/tests/pool-helper.spec.ts | 1 - 9 files changed, 209 insertions(+), 134 deletions(-) diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index 770837c3..c4fb667c 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -540,14 +540,23 @@ export class DuckDb { totalSupply varchar, totalBondAmount varchar, rewardPerSec varchar, - apr double + apr double, + createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) ` ); } async insertPoolAprs(poolAprs: PoolApr[]) { - await this.insertBulkData(poolAprs, "pool_apr"); + await this.insertBulkData( + poolAprs.map((poolApr) => { + return { + ...poolApr, + createdAt: new Date() + }; + }), + "pool_apr" + ); } async getLatestPoolApr(pairAddr: string): Promise { @@ -555,7 +564,7 @@ export class DuckDb { ` SELECT * FROM pool_apr WHERE pairAddr = ? - ORDER BY height DESC + ORDER BY createdAt DESC LIMIT 1 `, pairAddr diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index cc23c27d..7d95cd99 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -91,6 +91,17 @@ export const concatLpHistoryToUniqueKey = (data: { timestamp: number; pairAddr: return `${data.timestamp}-${data.pairAddr}`; }; +export const concatAprHistoryToUniqueKey = (data: { + timestamp: number; + supply: string; + bond: string; + reward: string; + apr: number; + pairAddr: string; +}): string => { + return `${data.timestamp}-${data.pairAddr}-${data.supply}-${data.bond}-${data.reward}-${data.apr}`; +}; + export function isoToTimestampNumber(time: string) { return Math.floor(new Date(time).getTime() / 1000); } @@ -169,18 +180,18 @@ function findPairAddress(pairInfos: PairInfoData[], infos: [AssetInfo, AssetInfo )?.pairAddr; } -function calculatePriceByPool( +export const calculatePriceByPool = ( offerPool: bigint, askPool: bigint, commissionRate?: number, offerAmount?: number -): number { +): number => { const finalOfferAmount = offerAmount || tenAmountInDecimalSix; let bigIntAmount = Number(offerPool - (askPool * offerPool) / (askPool + BigInt(finalOfferAmount))) * (1 - commissionRate || 0); return bigIntAmount / finalOfferAmount; -} +}; export function groupDataByTime(data: any[], timeframe?: number): { [key: string]: any[] } { let ops: { [k: number]: any[] } = {}; @@ -525,7 +536,6 @@ export const parsePairDenomToAssetInfo = ([baseDenom, quoteDenom]: [string, stri }; export { - calculatePriceByPool, convertDateToSecond, delay, fetchPoolInfoAmount, diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index 5d2386a7..77744cfa 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -5,8 +5,7 @@ import { collectAccumulateLpAndSwapData, concatLpHistoryToUniqueKey, getPairLiquidity, - getSymbolFromAsset, - parsePairDenomToAssetInfo + getSymbolFromAsset } from "./helper"; import { calculateAprResult, @@ -14,9 +13,9 @@ import { getAllPairInfos, getPairByAssetInfos, getPoolInfos, - refetchTokenInfos + handleEventApr } from "./pool-helper"; -import { parseAssetInfo, parseTxs, processEventApr } from "./tx-parsing"; +import { parseAssetInfo, parseTxs } from "./tx-parsing"; import { Env, InitialData, @@ -28,7 +27,6 @@ import { TxAnlysisResult, WithdrawLiquidityOperationData } from "./types"; -import { AssetInfo } from "@oraichain/common-contracts-sdk"; class WriteOrders extends WriteData { constructor(private duckDb: DuckDb, private rpcUrl: string, private env: Env, private initialData: InitialData) { @@ -119,18 +117,7 @@ class WriteOrders extends WriteData { [...result.swapOpsData] ); - const assetInfosToRefetchTokenInfos = Array.from( - [...result.provideLiquidityOpsData, ...result.withdrawLiquidityOpsData] - .map((op) => [op.baseTokenDenom, op.quoteTokenDenom] as [string, string]) - .reduce((accumulator, tokenDenoms) => { - const assetInfo = parsePairDenomToAssetInfo(tokenDenoms); - accumulator.add(assetInfo); - return accumulator; - }, new Set<[AssetInfo, AssetInfo]>()) - ); - await refetchTokenInfos(assetInfosToRefetchTokenInfos, newOffset); - - await processEventApr(txs); + await handleEventApr(txs, result, newOffset); // collect the latest offer & ask volume to accumulate the results // insert txs @@ -196,8 +183,7 @@ class OraiDexSync { if (result.status === "fulfilled") return result.value; else console.error("error get allLiquidities: ", result.reason); }); - const { allTotalSupplies, allBondAmounts, allRewardPerSec } = await fetchAprResult(pools, allLiquidities); - const allAprs = await calculateAprResult(allLiquidities, allTotalSupplies, allBondAmounts, allRewardPerSec); + const { allAprs, allTotalSupplies, allBondAmounts, allRewardPerSec } = await fetchAprResult(pools, allLiquidities); const poolAprs = allAprs.map((apr, index) => { return { diff --git a/packages/oraidex-sync/src/pairs.ts b/packages/oraidex-sync/src/pairs.ts index 25d2a340..f3bf5afe 100644 --- a/packages/oraidex-sync/src/pairs.ts +++ b/packages/oraidex-sync/src/pairs.ts @@ -16,7 +16,6 @@ import { usdtCw20Address } from "./constants"; import { PairMapping } from "./types"; -import { getStakingAssetInfo } from "./pool-helper"; // the orders are important! Do not change the order of the asset_infos. export const pairs: PairMapping[] = [ @@ -110,9 +109,18 @@ export const oraiUsdtPairOnlyDenom = pairsOnlyDenom.find( (pair) => JSON.stringify(pair.asset_infos) === JSON.stringify([ORAI, usdtCw20Address]) ).asset_infos; +function parseAssetInfoOnlyDenom1(info: AssetInfo): string { + if ("native_token" in info) return info.native_token.denom; + return info.token.contract_addr; +} + +export const getStakingAssetInfo1 = (assetInfos: AssetInfo[]): AssetInfo => { + return parseAssetInfoOnlyDenom1(assetInfos[0]) === ORAI ? assetInfos[1] : assetInfos[0]; +}; + export const pairWithStakingAsset = pairs.map((pair) => { return { ...pair, - stakingAssetInfo: getStakingAssetInfo(pair.asset_infos) + stakingAssetInfo: getStakingAssetInfo1(pair.asset_infos) }; }); diff --git a/packages/oraidex-sync/src/pool-helper.ts b/packages/oraidex-sync/src/pool-helper.ts index d66e0ae0..beb0f45e 100644 --- a/packages/oraidex-sync/src/pool-helper.ts +++ b/packages/oraidex-sync/src/pool-helper.ts @@ -13,12 +13,12 @@ import { ORAI, ORAIXOCH_INFO, SEC_PER_YEAR, atomic, network, oraiInfo, usdtInfo import { calculatePriceByPool, getCosmwasmClient, - isAssetInfoPairReverse, parseAssetInfoOnlyDenom, - validateNumber + validateNumber, + isAssetInfoPairReverse } from "./helper"; -import { DuckDb } from "./index"; -import { pairs } from "./pairs"; +import { DuckDb, concatAprHistoryToUniqueKey, getPairLiquidity, parsePairDenomToAssetInfo } from "./index"; +import { pairWithStakingAsset, pairs } from "./pairs"; import { fetchAllRewardPerSecInfos, fetchAllTokenAssetPools, @@ -26,7 +26,9 @@ import { queryAllPairInfos, queryPoolInfos } from "./query"; -import { PairInfoData, PairMapping } from "./types"; +import { PairInfoData, PairMapping, TxAnlysisResult } from "./types"; +import { Tx } from "@oraichain/cosmos-rpc-sync"; +import { processEventApr } from "./tx-parsing"; // use this type to determine the ratio of price of base to the quote or vice versa export type RatioDirection = "base_in_quote" | "quote_in_base"; @@ -85,7 +87,7 @@ export const getPriceByAsset = async ( const duckDb = DuckDb.instances; const poolInfo = await duckDb.getPoolByAssetInfos(assetInfos); const poolAmount = await duckDb.getLatestLpPoolAmount(poolInfo.pairAddr); - if (!poolAmount || !poolInfo.askPoolAmount || !poolInfo.offerPoolAmount) return 0; + if (!poolAmount || !poolAmount.askPoolAmount || !poolAmount.offerPoolAmount) return 0; // offer: orai, ask: usdt -> price offer in ask = calculatePriceByPool([ask, offer]) // offer: orai, ask: atom -> price ask in offer = calculatePriceByPool([offer, ask]) const basePrice = calculatePriceByPool( @@ -235,25 +237,6 @@ export const getStakingAssetInfo = (assetInfos: AssetInfo[]): AssetInfo => { return parseAssetInfoOnlyDenom(assetInfos[0]) === ORAI ? assetInfos[1] : assetInfos[0]; }; -// export const fetchAprResult = async (pairInfos: PairInfoData[], allLiquidities: number[]): Promise => { -// const assetTokens = pairInfos.map((pair) => -// getStakingAssetInfo([JSON.parse(pair.firstAssetInfo), JSON.parse(pair.secondAssetInfo)]) -// ); -// const liquidityAddrs = pairInfos.map((pair) => pair.liquidityAddr); -// try { -// const [allTokenInfo, allLpTokenAsset, allRewardPerSec] = await Promise.all([ -// fetchTokenInfos(liquidityAddrs), -// fetchAllTokenAssetPools(assetTokens), -// fetchAllRewardPerSecInfos(assetTokens) -// ]); -// const allTotalSupplies = allTokenInfo.map((info) => info.total_supply); -// const allBondAmounts = allLpTokenAsset.map((info) => info.total_bond_amount); -// return calculateAprResult(allLiquidities, allTotalSupplies, allBondAmounts, allRewardPerSec); -// } catch (error) { -// console.log({ errorFetchAprResult: error }); -// } -// }; - export const fetchAprResult = async (pairInfos: PairInfoData[], allLiquidities: number[]) => { const assetTokens = pairInfos.map((pair) => getStakingAssetInfo([JSON.parse(pair.firstAssetInfo), JSON.parse(pair.secondAssetInfo)]) @@ -267,10 +250,12 @@ export const fetchAprResult = async (pairInfos: PairInfoData[], allLiquidities: ]); const allTotalSupplies = allTokenInfo.map((info) => info.total_supply); const allBondAmounts = allLpTokenAsset.map((info) => info.total_bond_amount); + const allAprs = await calculateAprResult(allLiquidities, allTotalSupplies, allBondAmounts, allRewardPerSec); return { allTotalSupplies, allBondAmounts, - allRewardPerSec + allRewardPerSec, + allAprs }; } catch (error) { console.log({ errorFetchAprResult: error }); @@ -286,15 +271,48 @@ export const getAllPairInfos = async (): Promise => { return queryAllPairInfos(firstFactoryClient, secondFactoryClient); }; -export const triggerCalculateApr = async () => { - // TODO: get all infos relate to apr in duckdb from apr table -> call to calculateAprResult -}; +export const triggerCalculateApr = async (assetInfos: [AssetInfo, AssetInfo][], newOffset: number) => { + try { + // TODO: get all infos relate to apr in duckdb from apr table -> call to calculateAprResult + if (assetInfos.length === 0) return; + const duckDb = DuckDb.instances; + const pools = await Promise.all(assetInfos.map((infos) => duckDb.getPoolByAssetInfos(infos))); -export type SupplyAction = "mint" | "burn"; -export type BondAction = "bond" | "unbond" | "deposit_reward"; -export type RewardPerSecAction = "update_reward_per_sec"; + const allLiquidities = (await Promise.allSettled(pools.map((pair) => getPairLiquidity(pair)))).map((result) => { + if (result.status === "fulfilled") return result.value; + else console.error("error get allLiquidities: ", result.reason); + }); -export const refetchTokenInfos = async (assetInfos: [AssetInfo, AssetInfo][], height: number) => { + const poolAprInfos = await Promise.all(pools.map((pool) => duckDb.getLatestPoolApr(pool.pairAddr))); + const allTotalSupplies = poolAprInfos.map((item) => item.totalSupply); + const allBondAmounts = poolAprInfos.map((info) => info.totalBondAmount); + const allRewardPerSecs = poolAprInfos.map((info) => (info.rewardPerSec ? JSON.parse(info.rewardPerSec) : null)); + + const APRs = await calculateAprResult(allLiquidities, allTotalSupplies, allBondAmounts, allRewardPerSecs); + const latestPoolAprs = await Promise.all(pools.map((pool) => duckDb.getLatestPoolApr(pool.pairAddr))); + const newPoolAprs = latestPoolAprs.map((poolApr, index) => { + return { + ...poolApr, + height: newOffset, + apr: APRs[index], + uniqueKey: concatAprHistoryToUniqueKey({ + timestamp: Date.now(), + supply: allTotalSupplies[index], + bond: allBondAmounts[index], + reward: allRewardPerSecs[index], + apr: APRs[index], + pairAddr: pools[index].pairAddr + }) + }; + }); + await duckDb.insertPoolAprs(newPoolAprs); + } catch (error) { + console.log({ triggerCalculateApr: error }); + } +}; + +export const refetchTotalSupplies = async (assetInfos: [AssetInfo, AssetInfo][], height: number) => { + if (assetInfos.length === 0) return; const duckDb = DuckDb.instances; const pools = await Promise.all(assetInfos.map((assetInfo) => duckDb.getPoolByAssetInfos(assetInfo))); const liquidityAddrs = pools.map((pair) => pair.liquidityAddr); @@ -312,13 +330,14 @@ export const refetchTokenInfos = async (assetInfos: [AssetInfo, AssetInfo][], he await duckDb.insertPoolAprs(newPoolAprs); }; -export const refetchTokenAssetPools = async (assetInfos: [AssetInfo, AssetInfo][], height: number) => { +export const refetchTotalBond = async (assetInfos: [AssetInfo, AssetInfo][], height: number) => { + if (assetInfos.length === 0) return; const duckDb = DuckDb.instances; const pools = await Promise.all(assetInfos.map((assetInfo) => duckDb.getPoolByAssetInfos(assetInfo))); - const assetTokens = pools.map((pair) => + const stakingAssetInfo = pools.map((pair) => getStakingAssetInfo([JSON.parse(pair.firstAssetInfo), JSON.parse(pair.secondAssetInfo)]) ); - const tokenAssetPools = await fetchAllTokenAssetPools(assetTokens); + const tokenAssetPools = await fetchAllTokenAssetPools(stakingAssetInfo); const totalBondAmounts = tokenAssetPools.map((info) => info.total_bond_amount); const latestPoolAprs = await Promise.all(pools.map((pool) => duckDb.getLatestPoolApr(pool.pairAddr))); const newPoolAprs = latestPoolAprs.map((poolApr, index) => { @@ -332,6 +351,7 @@ export const refetchTokenAssetPools = async (assetInfos: [AssetInfo, AssetInfo][ }; export const refetchRewardPerSecInfos = async (assetInfos: [AssetInfo, AssetInfo][], height: number) => { + if (assetInfos.length === 0) return; const duckDb = DuckDb.instances; const pools = await Promise.all(assetInfos.map((assetInfo) => duckDb.getPoolByAssetInfos(assetInfo))); const assetTokens = pools.map((pair) => @@ -349,3 +369,41 @@ export const refetchRewardPerSecInfos = async (assetInfos: [AssetInfo, AssetInfo }); await duckDb.insertPoolAprs(newPoolAprs); }; + +export const handleEventApr = async (txs: Tx[], result: TxAnlysisResult, newOffset: number): Promise => { + let listAssetInfosPoolShouldRefetch = new Set<[AssetInfo, AssetInfo]>(); + // mint/burn trigger update total supply + const assetInfosTriggerTotalSupplies = Array.from( + [...result.provideLiquidityOpsData, ...result.withdrawLiquidityOpsData] + .map((op) => [op.baseTokenDenom, op.quoteTokenDenom] as [string, string]) + .reduce((accumulator, tokenDenoms) => { + const assetInfo = parsePairDenomToAssetInfo(tokenDenoms); + accumulator.add(assetInfo); + return accumulator; + }, new Set<[AssetInfo, AssetInfo]>()) + ); + assetInfosTriggerTotalSupplies.forEach((item) => listAssetInfosPoolShouldRefetch.add(item)); + + await refetchTotalSupplies(assetInfosTriggerTotalSupplies, newOffset); + + const { infoTokenAssetPools, isTriggerRewardPerSec } = await processEventApr(txs); + if (isTriggerRewardPerSec) { + // update_reward_per_sec trigger refetch all info + pairs.map((pair) => pair.asset_infos).forEach((assetInfos) => listAssetInfosPoolShouldRefetch.add(assetInfos)); + await refetchRewardPerSecInfos(Array.from(listAssetInfosPoolShouldRefetch), newOffset); + } + + // bond/unbond trigger refetch info token asset pools + const assetInfosTriggerTotalBond = Array.from(infoTokenAssetPools) + .map((stakingDenom) => { + return pairWithStakingAsset.find((pair) => parseAssetInfoOnlyDenom(pair.stakingAssetInfo) === stakingDenom) + ?.asset_infos; + }) + .filter(Boolean); + + !isTriggerRewardPerSec && + assetInfosTriggerTotalBond.forEach((assetInfo) => listAssetInfosPoolShouldRefetch.add(assetInfo)); + await refetchTotalBond(assetInfosTriggerTotalBond, newOffset); + + await triggerCalculateApr(Array.from(listAssetInfosPoolShouldRefetch), newOffset); +}; diff --git a/packages/oraidex-sync/src/tx-parsing.ts b/packages/oraidex-sync/src/tx-parsing.ts index 85cbe215..17ee14b2 100644 --- a/packages/oraidex-sync/src/tx-parsing.ts +++ b/packages/oraidex-sync/src/tx-parsing.ts @@ -1,8 +1,24 @@ import { Attribute, Event } from "@cosmjs/stargate"; -import { isEqual } from "lodash"; +import { Log } from "@cosmjs/stargate/build/logs"; import { Tx } from "@oraichain/cosmos-rpc-sync"; -import { MsgExecuteContract } from "cosmjs-types/cosmwasm/wasm/v1/tx"; import { Tx as CosmosTx } from "cosmjs-types/cosmos/tx/v1beta1/tx"; +import { MsgExecuteContract } from "cosmjs-types/cosmwasm/wasm/v1/tx"; +import { isEqual } from "lodash"; +import { DuckDb } from "./db"; +import { + buildOhlcv, + calculatePriceByPool, + concatDataToUniqueKey, + getSwapDirection, + groupByTime, + isAssetInfoPairReverse, + isoToTimestampNumber, + parseAssetInfo, + parseAssetInfoOnlyDenom, + removeOpsDuplication +} from "./helper"; +import { pairs } from "./pairs"; +import { calculateLiquidityFee, isPoolHasFee } from "./pool-helper"; import { AccountTx, BasicTxData, @@ -17,22 +33,6 @@ import { TxAnlysisResult, WithdrawLiquidityOperationData } from "./types"; -import { Log } from "@cosmjs/stargate/build/logs"; -import { - buildOhlcv, - calculatePriceByPool, - concatDataToUniqueKey, - getSwapDirection, - groupByTime, - isAssetInfoPairReverse, - isoToTimestampNumber, - parseAssetInfo, - parseAssetInfoOnlyDenom, - removeOpsDuplication -} from "./helper"; -import { pairs } from "./pairs"; -import { DuckDb } from "./db"; -import { calculateLiquidityFee, isPoolHasFee, refetchTokenInfos, triggerCalculateApr } from "./pool-helper"; function parseWasmEvents(events: readonly Event[]): (readonly Attribute[])[] { return events.filter((event) => event.type === "wasm").map((event) => event.attributes); @@ -294,10 +294,9 @@ function parseExecuteContractToOraidexMsgs(msgs: MsgExecuteContractWithLogs[]): "execute_swap_operation" in obj.msg || "bond" in obj.msg || "unbond" in obj.msg || - "update_reward_per_sec" in obj.msg || "mint" in obj.msg || "burn" in obj.msg || - "deposit_reward" in obj.msg + ("execute" in obj.msg && typeof obj.msg.execute === "object" && "proposal_id" in obj.msg.execute) ) objs.push(obj); if ("send" in obj.msg) { @@ -310,10 +309,8 @@ function parseExecuteContractToOraidexMsgs(msgs: MsgExecuteContractWithLogs[]): "withdraw_liquidity" in contractSendMsg || "bond" in contractSendMsg || "unbond" in contractSendMsg || - "update_reward_per_sec" in contractSendMsg || "mint" in contractSendMsg || - "burn" in contractSendMsg || - "deposit_reward" in contractSendMsg + "burn" in contractSendMsg ) { objs.push({ ...msg, msg: contractSendMsg }); } @@ -373,38 +370,26 @@ async function parseTxs(txs: Tx[]): Promise { export const processEventApr = async (txs: Tx[]) => { const assets = { - infoTokenAssetPools: new Set(), - infoRewardPerSec: new Set() + infoTokenAssetPools: new Set(), + isTriggerRewardPerSec: false }; for (let tx of txs) { + // guard code. Should refetch all token info if match event update_rewards_per_sec or length ofstaking asset equal to pairs length. + if (assets.isTriggerRewardPerSec || assets.infoTokenAssetPools.size === pairs.length) break; + const msgExecuteContracts = parseTxToMsgExecuteContractMsgs(tx); const msgs = parseExecuteContractToOraidexMsgs(msgExecuteContracts); for (let msg of msgs) { const wasmAttributes = parseWasmEvents(msg.logs.events); for (let attrs of wasmAttributes) { - // if (attrs.find((attr) => attr.key === "action" && (attr.value === "mint" || attr.value === "burn"))) { - // console.log("mint-burn"); - // console.table(attrs); - // // TODO: FIND ASSET THEN ADD TO LIST ASSET - // // assets.infoTokenAssetPools.add() - // } - - if ( - attrs.find( - (attr) => - attr.key === "action" && - (attr.value === "bond" || attr.value === "unbond" || attr.value === "deposit_reward") - ) - ) { - console.log("bond-unbond"); - console.table(attrs); - // TODO: FIND ASSET THEN ADD TO LIST ASSET - // assets.infoTokenAssetPools.add() + if (attrs.find((attr) => attr.key === "action" && (attr.value === "bond" || attr.value === "unbond"))) { + const stakingAssetDenom = attrs.find((attr) => attr.key === "asset_info")?.value; + assets.infoTokenAssetPools.add(stakingAssetDenom); } - if (attrs.find((attr) => attr.key === "action" && attr.value === "update_reward_per_sec")) { - console.log("update_reward_per_sec"); - console.table(attrs); + if (attrs.find((attr) => attr.key === "action" && attr.value === "update_rewards_per_sec")) { + assets.isTriggerRewardPerSec = true; + break; } } } @@ -412,4 +397,4 @@ export const processEventApr = async (txs: Tx[]) => { return assets; }; -export { parseAssetInfo, parseWasmEvents, parseTxs, parseWithdrawLiquidityAssets, parseTxToMsgExecuteContractMsgs }; +export { parseAssetInfo, parseTxToMsgExecuteContractMsgs, parseTxs, parseWasmEvents, parseWithdrawLiquidityAssets }; diff --git a/packages/oraidex-sync/src/types.ts b/packages/oraidex-sync/src/types.ts index 40d9891f..018a7c71 100644 --- a/packages/oraidex-sync/src/types.ts +++ b/packages/oraidex-sync/src/types.ts @@ -245,4 +245,5 @@ export type PoolApr = { totalBondAmount: string; rewardPerSec: string; apr: number; + createdAt?: string; }; diff --git a/packages/oraidex-sync/tests/helper.spec.ts b/packages/oraidex-sync/tests/helper.spec.ts index 1869cd1a..540911b0 100644 --- a/packages/oraidex-sync/tests/helper.spec.ts +++ b/packages/oraidex-sync/tests/helper.spec.ts @@ -26,7 +26,6 @@ import { findPairIndexFromDenoms, getSwapDirection, groupByTime, - isAssetInfoPairReverse, removeOpsDuplication, roundTime, toAmount, @@ -406,36 +405,45 @@ describe("test-helper", () => { baseTokenDenom: ORAI, quoteTokenAmount: 1, quoteTokenDenom: usdtCw20Address, - opType: "withdraw" + opType: "withdraw", + height: 1, + timestamp: 1 }, { baseTokenAmount: 2, baseTokenDenom: ORAI, quoteTokenAmount: 2, quoteTokenDenom: usdtCw20Address, - opType: "provide" + opType: "provide", + height: 1, + timestamp: 1 }, { baseTokenAmount: 1, baseTokenDenom: ORAI, quoteTokenAmount: -1, quoteTokenDenom: usdtCw20Address, - direction: "Buy" + direction: "Buy", + height: 1, + timestamp: 1 }, { baseTokenAmount: 1, baseTokenDenom: ORAI, quoteTokenAmount: -1, quoteTokenDenom: usdtCw20Address, - direction: "Sell" + direction: "Sell", + height: 1, + timestamp: 1 }, - { baseTokenAmount: 1, baseTokenDenom: ORAI, quoteTokenAmount: -1, quoteTokenDenom: atomIbcDenom, - direction: "Sell" + direction: "Sell", + height: 1, + timestamp: 1 } ]; duckDb = await DuckDb.create(":memory:"); @@ -475,8 +483,8 @@ describe("test-helper", () => { // assertion expect(accumulatedData).toStrictEqual({ - oraiUsdtPairAddr: { baseTokenAmount: 2n, quoteTokenAmount: 2n }, - oraiAtomPairAddr: { baseTokenAmount: 5n, quoteTokenAmount: 3n } + oraiUsdtPairAddr: { askPoolAmount: 2n, height: 1, offerPoolAmount: 2n, timestamp: 1 }, + oraiAtomPairAddr: { askPoolAmount: 3n, height: 1, offerPoolAmount: 5n, timestamp: 1 } }); }); @@ -685,16 +693,16 @@ describe("test-helper", () => { } ); - it.each([ - ["case-asset-info-pairs-is-NOT-reversed", [oraiInfo, usdtInfo], false], - ["case-asset-info-pairs-is-reversed", [usdtInfo, oraiInfo], true] - ])( - "test-isAssetInfoPairReverse-should-return-correctly", - (_caseName: string, assetInfos: AssetInfo[], expectedResult: boolean) => { - const result = isAssetInfoPairReverse(assetInfos); - expect(result).toBe(expectedResult); - } - ); + // it.each([ + // ["case-asset-info-pairs-is-NOT-reversed", [oraiInfo, usdtInfo], false], + // ["case-asset-info-pairs-is-reversed", [usdtInfo, oraiInfo], true] + // ])( + // "test-isAssetInfoPairReverse-should-return-correctly", + // (_caseName: string, assetInfos: AssetInfo[], expectedResult: boolean) => { + // const result = helper.isAssetInfoPairReverse(assetInfos); + // expect(result).toBe(expectedResult); + // } + // ); it("test-getSymbolFromAsset-should-throw-error-for-assetInfos-not-valid", () => { const asset_infos = [oraiInfo, { token: { contract_addr: "invalid-token" } }] as [AssetInfo, AssetInfo]; @@ -744,8 +752,19 @@ describe("test-helper", () => { [1n, 1n, 4] ])( "test-getPairLiquidity-should-return-correctly-liquidity-by-USDT", - async (offerAmount: bigint, askAmount: bigint, expectedResult: number) => { + async (offerPoolAmount: bigint, askPoolAmount: bigint, expectedResult: number) => { // setup + await duckDb.createPoolOpsTable(); + await duckDb.insertPoolAmountHistory([ + { + offerPoolAmount, + askPoolAmount, + timestamp: 1, + height: 1, + pairAddr: "oraiUsdtPairAddr", + uniqueKey: "1" + } + ]); const poolInfo: PairInfoData = { firstAssetInfo: JSON.stringify(oraiInfo), secondAssetInfo: JSON.stringify(usdtInfo), @@ -756,8 +775,8 @@ describe("test-helper", () => { symbols: "1", fromIconUrl: "1", toIconUrl: "1", - offerPoolAmount: offerAmount, - askPoolAmount: askAmount + offerPoolAmount, + askPoolAmount }; jest.spyOn(poolHelper, "getPriceAssetByUsdt").mockResolvedValue(2); diff --git a/packages/oraidex-sync/tests/pool-helper.spec.ts b/packages/oraidex-sync/tests/pool-helper.spec.ts index e2668121..efbef53c 100644 --- a/packages/oraidex-sync/tests/pool-helper.spec.ts +++ b/packages/oraidex-sync/tests/pool-helper.spec.ts @@ -401,7 +401,6 @@ describe("test-pool-helper", () => { allBondAmounts, allRewardPerSec ); - console.dir({ result }, { depth: null }); // assertion expect(result.length).toEqual(pairs.length); From ab7c27ec3b629dfd5149fa5e41149c0574ab3d00 Mon Sep 17 00:00:00 2001 From: trungbach Date: Thu, 14 Sep 2023 23:47:07 +0700 Subject: [PATCH 44/46] remove offer & ask pool column in table pair_infos --- packages/oraidex-sync/src/db.ts | 14 --------- packages/oraidex-sync/src/index.ts | 4 +-- packages/oraidex-sync/src/types.ts | 6 +--- packages/oraidex-sync/tests/db.spec.ts | 29 ------------------- packages/oraidex-sync/tests/helper.spec.ts | 16 +++------- .../oraidex-sync/tests/pool-helper.spec.ts | 16 +++------- 6 files changed, 10 insertions(+), 75 deletions(-) diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index c4fb667c..acdcb732 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -145,8 +145,6 @@ export class DuckDb { symbols VARCHAR, fromIconUrl VARCHAR, toIconUrl VARCHAR, - offerPoolAmount UBIGINT, - askPoolAmount UBIGINT, PRIMARY KEY (pairAddr) )` ); } @@ -155,18 +153,6 @@ export class DuckDb { await this.insertBulkData(ops, "pair_infos", true); } - async updatePairInfoAmount(offerPoolAmount: bigint, askPoolAmount: bigint, pairAddr: string) { - await this.conn.all( - `UPDATE pair_infos - SET offerPoolAmount = ?, askPoolAmount = ? - WHERE pairAddr = ? - `, - Number(offerPoolAmount), - Number(askPoolAmount), - pairAddr - ); - } - async insertPriceInfos(ops: PriceInfo[]) { await this.insertBulkData(ops, "price_infos", false, `price_infos-${Math.random() * 1000}`); } diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index 77744cfa..4cd4235f 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -164,9 +164,7 @@ class OraiDexSync { oracleAddr: pair.oracle_addr, symbols, fromIconUrl: "url1", - toIconUrl: "url2", - offerPoolAmount: 0n, - askPoolAmount: 0n + toIconUrl: "url2" } as PairInfoData; }) ); diff --git a/packages/oraidex-sync/src/types.ts b/packages/oraidex-sync/src/types.ts index 018a7c71..2913aef3 100644 --- a/packages/oraidex-sync/src/types.ts +++ b/packages/oraidex-sync/src/types.ts @@ -44,8 +44,6 @@ export type PairInfoData = { symbols: string; fromIconUrl: string; toIconUrl: string; - offerPoolAmount: bigint; - askPoolAmount: bigint; }; export type PairInfoDataResponse = PairInfoData & { @@ -229,13 +227,11 @@ export type PoolInfo = { }; export type PoolAmountHistory = { - offerPoolAmount: bigint; - askPoolAmount: bigint; timestamp: number; height: number; pairAddr: string; uniqueKey: string; -}; +} & PoolInfo; export type PoolApr = { uniqueKey: string; diff --git a/packages/oraidex-sync/tests/db.spec.ts b/packages/oraidex-sync/tests/db.spec.ts index c2a81763..8f60f0b6 100644 --- a/packages/oraidex-sync/tests/db.spec.ts +++ b/packages/oraidex-sync/tests/db.spec.ts @@ -271,35 +271,6 @@ describe("test-duckdb", () => { expect(queryResult.length).toEqual(2); }); - it("test-updatePairInfoAmount-should-success", async () => { - // setup - duckDb = await DuckDb.create(":memory:"); - await duckDb.createPairInfosTable(); - await duckDb.insertPairInfos([ - { - firstAssetInfo: JSON.stringify(oraiInfo), - secondAssetInfo: JSON.stringify(usdtInfo), - commissionRate: "", - pairAddr: "orai1c5s03c3l336dgesne7dylnmhszw8554tsyy9yt", - liquidityAddr: "", - oracleAddr: "", - symbols: "1", - fromIconUrl: "1", - toIconUrl: "1", - offerPoolAmount: 1n, - askPoolAmount: 1n - } as PairInfoData - ]); - - // act - await duckDb.updatePairInfoAmount(2n, 3n, "orai1c5s03c3l336dgesne7dylnmhszw8554tsyy9yt"); - - // assertion - const pairInfoAfterUpdate = await duckDb.getPoolByAssetInfos([oraiInfo, usdtInfo]); - expect(pairInfoAfterUpdate.offerPoolAmount).toEqual(2); - expect(pairInfoAfterUpdate.askPoolAmount).toEqual(3); - }); - it("test-getFeeSwap-should-return-correctly-fee-in-USDT", async () => { // setup duckDb = await DuckDb.create(":memory:"); diff --git a/packages/oraidex-sync/tests/helper.spec.ts b/packages/oraidex-sync/tests/helper.spec.ts index 540911b0..3fc255c2 100644 --- a/packages/oraidex-sync/tests/helper.spec.ts +++ b/packages/oraidex-sync/tests/helper.spec.ts @@ -202,9 +202,7 @@ describe("test-helper", () => { oracleAddr: "", symbols: "1", fromIconUrl: "1", - toIconUrl: "1", - offerPoolAmount: 1n, - askPoolAmount: 1n + toIconUrl: "1" } ]; let assetInfos: [AssetInfo, AssetInfo] = [{ native_token: { denom: ORAI } }, assetInfo]; @@ -459,9 +457,7 @@ describe("test-helper", () => { oracleAddr: "", symbols: "1", fromIconUrl: "1", - toIconUrl: "1", - offerPoolAmount: 1n, - askPoolAmount: 1n + toIconUrl: "1" }, { firstAssetInfo: JSON.stringify(oraiInfo), @@ -472,9 +468,7 @@ describe("test-helper", () => { oracleAddr: "", symbols: "1", fromIconUrl: "1", - toIconUrl: "1", - offerPoolAmount: 1n, - askPoolAmount: 1n + toIconUrl: "1" } ]); @@ -774,9 +768,7 @@ describe("test-helper", () => { oracleAddr: "", symbols: "1", fromIconUrl: "1", - toIconUrl: "1", - offerPoolAmount, - askPoolAmount + toIconUrl: "1" }; jest.spyOn(poolHelper, "getPriceAssetByUsdt").mockResolvedValue(2); diff --git a/packages/oraidex-sync/tests/pool-helper.spec.ts b/packages/oraidex-sync/tests/pool-helper.spec.ts index efbef53c..b9e2b49b 100644 --- a/packages/oraidex-sync/tests/pool-helper.spec.ts +++ b/packages/oraidex-sync/tests/pool-helper.spec.ts @@ -127,9 +127,7 @@ describe("test-pool-helper", () => { oracleAddr: "", symbols: "1", fromIconUrl: "1", - toIconUrl: "1", - offerPoolAmount: 1n, - askPoolAmount: 1n + toIconUrl: "1" } ]; await duckDb.insertPairInfos(pairInfoData); @@ -192,9 +190,7 @@ describe("test-pool-helper", () => { oracleAddr: "", symbols: "1", fromIconUrl: "1", - toIconUrl: "1", - offerPoolAmount: 1n, - askPoolAmount: 1n + toIconUrl: "1" }, { firstAssetInfo: JSON.stringify(oraiInfo as AssetInfo), @@ -205,9 +201,7 @@ describe("test-pool-helper", () => { oracleAddr: "", symbols: "1", fromIconUrl: "1", - toIconUrl: "1", - offerPoolAmount: 1n, - askPoolAmount: 1n + toIconUrl: "1" } ]; await duckDb.insertPairInfos(pairInfoData); @@ -292,9 +286,7 @@ describe("test-pool-helper", () => { oracleAddr: "1", symbols: "1", fromIconUrl: "1", - toIconUrl: "1", - offerPoolAmount: 1n, - askPoolAmount: 1n + toIconUrl: "1" }, 13344890, 1 From 530ffee389ff00a5b5ca050a6cc15cb3e9e37d38 Mon Sep 17 00:00:00 2001 From: trungbach Date: Fri, 15 Sep 2023 02:26:22 +0700 Subject: [PATCH 45/46] finished accumulate apr with testcase --- packages/oraidex-sync/src/db.ts | 42 ++-- packages/oraidex-sync/src/helper.ts | 25 +-- packages/oraidex-sync/src/index.ts | 23 +-- packages/oraidex-sync/src/pairs.ts | 4 +- packages/oraidex-sync/src/pool-helper.ts | 192 ++++++++++-------- packages/oraidex-sync/src/test-db.ts | 72 ------- packages/oraidex-sync/src/tx-parsing.ts | 2 +- packages/oraidex-sync/src/types.ts | 1 - packages/oraidex-sync/tests/db.spec.ts | 14 +- packages/oraidex-sync/tests/helper.spec.ts | 2 +- .../oraidex-sync/tests/pool-helper.spec.ts | 117 ++++++++++- 11 files changed, 255 insertions(+), 239 deletions(-) delete mode 100644 packages/oraidex-sync/src/test-db.ts diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index acdcb732..a25fb421 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -1,30 +1,23 @@ -import { Database, Connection } from "duckdb-async"; +import { AssetInfo } from "@oraichain/oraidex-contracts-sdk"; +import { Connection, Database } from "duckdb-async"; +import fs from "fs"; +import { isoToTimestampNumber, parseAssetInfo, renameKey, replaceAllNonAlphaBetChar, toObject } from "./helper"; import { + GetCandlesQuery, + GetFeeSwap, + GetVolumeQuery, Ohlcv, PairInfoData, + PoolAmountHistory, + PoolApr, PriceInfo, SwapOperationData, TokenVolumeData, TotalLiquidity, VolumeData, VolumeRange, - WithdrawLiquidityOperationData, - GetCandlesQuery, - GetFeeSwap, - GetVolumeQuery, - PoolAmountHistory, - PoolApr + WithdrawLiquidityOperationData } from "./types"; -import fs, { rename } from "fs"; -import { - isoToTimestampNumber, - parseAssetInfo, - parseAssetInfoOnlyDenom, - renameKey, - replaceAllNonAlphaBetChar, - toObject -} from "./helper"; -import { AssetInfo } from "@oraichain/oraidex-contracts-sdk"; export class DuckDb { static instances: DuckDb; @@ -487,7 +480,7 @@ export class DuckDb { return BigInt(result[0]?.totalVolume ?? 0); } - async createPoolOpsTable() { + async createLpAmountHistoryTable() { await this.conn.exec( `CREATE TABLE IF NOT EXISTS lp_amount_history ( offerPoolAmount ubigint, @@ -527,22 +520,13 @@ export class DuckDb { totalBondAmount varchar, rewardPerSec varchar, apr double, - createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) ` ); } async insertPoolAprs(poolAprs: PoolApr[]) { - await this.insertBulkData( - poolAprs.map((poolApr) => { - return { - ...poolApr, - createdAt: new Date() - }; - }), - "pool_apr" - ); + await this.insertBulkData(poolAprs, "pool_apr"); } async getLatestPoolApr(pairAddr: string): Promise { @@ -550,7 +534,7 @@ export class DuckDb { ` SELECT * FROM pool_apr WHERE pairAddr = ? - ORDER BY createdAt DESC + ORDER BY height DESC LIMIT 1 `, pairAddr diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index 7d95cd99..173a9b17 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -1,10 +1,4 @@ -import { - AssetInfo, - CosmWasmClient, - OraiswapPairQueryClient, - OraiswapPairTypes, - SwapOperation -} from "@oraichain/oraidex-contracts-sdk"; +import { AssetInfo, CosmWasmClient, OraiswapPairTypes, SwapOperation } from "@oraichain/oraidex-contracts-sdk"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; import { isEqual, maxBy, minBy } from "lodash"; import { ORAI, atomic, oraiInfo, tenAmountInDecimalSix, truncDecimals, usdtInfo } from "./constants"; @@ -17,7 +11,6 @@ import { OraiDexType, PairInfoData, PoolAmountHistory, - PoolInfo, SwapDirection, SwapOperationData } from "./types"; @@ -409,15 +402,6 @@ export const parsePoolAmount = (poolInfo: OraiswapPairTypes.PoolResponse, trueAs return BigInt(poolInfo.assets.find((asset) => isEqual(asset.info, trueAsset))?.amount || "0"); }; -async function fetchPoolInfoAmount(fromInfo: AssetInfo, toInfo: AssetInfo, pairAddr: string): Promise { - const client = await getCosmwasmClient(); - const pairContract = new OraiswapPairQueryClient(client, pairAddr); - const poolInfo = await pairContract.pool(); - const offerPoolAmount = parsePoolAmount(poolInfo, fromInfo); - const askPoolAmount = parsePoolAmount(poolInfo, toInfo); - return { offerPoolAmount, askPoolAmount }; -} - // get liquidity of pair from assetInfos export const getPairLiquidity = async (poolInfo: PairInfoData): Promise => { const duckDb = DuckDb.instances; @@ -535,10 +519,15 @@ export const parsePairDenomToAssetInfo = ([baseDenom, quoteDenom]: [string, stri return pair.asset_infos; }; +export function getDate24hBeforeNow(time: Date) { + const twentyFourHoursInMilliseconds = 24 * 60 * 60 * 1000; // 24 hours in milliseconds + const date24hBeforeNow = new Date(time.getTime() - twentyFourHoursInMilliseconds); + return date24hBeforeNow; +} + export { convertDateToSecond, delay, - fetchPoolInfoAmount, findAssetInfoPathToUsdt, findMappedTargetedAssetInfo, findPairAddress, diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index 4cd4235f..39a7a002 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -111,13 +111,11 @@ class WriteOrders extends WriteData { if (currentOffset === newOffset) return true; let result = await parseTxs(txs); + const lpOpsData = [...result.provideLiquidityOpsData, ...result.withdrawLiquidityOpsData]; // accumulate liquidity pool amount via provide/withdraw liquidity and swap ops - await this.accumulatePoolAmount( - [...result.provideLiquidityOpsData, ...result.withdrawLiquidityOpsData], - [...result.swapOpsData] - ); + await this.accumulatePoolAmount(lpOpsData, [...result.swapOpsData]); - await handleEventApr(txs, result, newOffset); + await handleEventApr(txs, lpOpsData, newOffset); // collect the latest offer & ask volume to accumulate the results // insert txs @@ -168,7 +166,6 @@ class OraiDexSync { } as PairInfoData; }) ); - console.timeEnd("timer-updateLatestPairInfos"); } catch (error) { console.log("error in updateLatestPairInfos: ", error); @@ -205,7 +202,7 @@ class OraiDexSync { this.duckDb.createSwapOpsTable(), this.duckDb.createPairInfosTable(), this.duckDb.createSwapOhlcv(), - this.duckDb.createPoolOpsTable(), + this.duckDb.createLpAmountHistoryTable(), this.duckDb.createAprInfoPair() ]); let currentInd = await this.duckDb.loadHeightSnapshot(); @@ -235,13 +232,13 @@ class OraiDexSync { } } -async function initSync() { - const duckDb = await DuckDb.create("oraidex-only-sync-data"); - const oraidexSync = await OraiDexSync.create(duckDb, "http://35.237.59.125:26657", process.env as any); - oraidexSync.sync(); -} +// async function initSync() { +// const duckDb = await DuckDb.create("oraidex-only-sync-data"); +// const oraidexSync = await OraiDexSync.create(duckDb, "http://35.237.59.125:26657", process.env as any); +// oraidexSync.sync(); +// } -initSync(); +// initSync(); export { OraiDexSync }; diff --git a/packages/oraidex-sync/src/pairs.ts b/packages/oraidex-sync/src/pairs.ts index f3bf5afe..6d6cdaee 100644 --- a/packages/oraidex-sync/src/pairs.ts +++ b/packages/oraidex-sync/src/pairs.ts @@ -114,13 +114,13 @@ function parseAssetInfoOnlyDenom1(info: AssetInfo): string { return info.token.contract_addr; } -export const getStakingAssetInfo1 = (assetInfos: AssetInfo[]): AssetInfo => { +const getStakingAssetInfo = (assetInfos: AssetInfo[]): AssetInfo => { return parseAssetInfoOnlyDenom1(assetInfos[0]) === ORAI ? assetInfos[1] : assetInfos[0]; }; export const pairWithStakingAsset = pairs.map((pair) => { return { ...pair, - stakingAssetInfo: getStakingAssetInfo1(pair.asset_infos) + stakingAssetInfo: getStakingAssetInfo(pair.asset_infos) }; }); diff --git a/packages/oraidex-sync/src/pool-helper.ts b/packages/oraidex-sync/src/pool-helper.ts index beb0f45e..5234fb62 100644 --- a/packages/oraidex-sync/src/pool-helper.ts +++ b/packages/oraidex-sync/src/pool-helper.ts @@ -1,4 +1,5 @@ import { MulticallQueryClient } from "@oraichain/common-contracts-sdk"; +import { Tx } from "@oraichain/cosmos-rpc-sync"; import { Asset, AssetInfo, @@ -13,9 +14,9 @@ import { ORAI, ORAIXOCH_INFO, SEC_PER_YEAR, atomic, network, oraiInfo, usdtInfo import { calculatePriceByPool, getCosmwasmClient, + isAssetInfoPairReverse, parseAssetInfoOnlyDenom, - validateNumber, - isAssetInfoPairReverse + validateNumber } from "./helper"; import { DuckDb, concatAprHistoryToUniqueKey, getPairLiquidity, parsePairDenomToAssetInfo } from "./index"; import { pairWithStakingAsset, pairs } from "./pairs"; @@ -26,10 +27,8 @@ import { queryAllPairInfos, queryPoolInfos } from "./query"; -import { PairInfoData, PairMapping, TxAnlysisResult } from "./types"; -import { Tx } from "@oraichain/cosmos-rpc-sync"; import { processEventApr } from "./tx-parsing"; - +import { PairInfoData, PairMapping, ProvideLiquidityOperationData } from "./types"; // use this type to determine the ratio of price of base to the quote or vice versa export type RatioDirection = "base_in_quote" | "quote_in_base"; @@ -86,6 +85,7 @@ export const getPriceByAsset = async ( ): Promise => { const duckDb = DuckDb.instances; const poolInfo = await duckDb.getPoolByAssetInfos(assetInfos); + if (!poolInfo) return 0; const poolAmount = await duckDb.getLatestLpPoolAmount(poolInfo.pairAddr); if (!poolAmount || !poolAmount.askPoolAmount || !poolAmount.offerPoolAmount) return 0; // offer: orai, ask: usdt -> price offer in ask = calculatePriceByPool([ask, offer]) @@ -272,109 +272,105 @@ export const getAllPairInfos = async (): Promise => { }; export const triggerCalculateApr = async (assetInfos: [AssetInfo, AssetInfo][], newOffset: number) => { - try { - // TODO: get all infos relate to apr in duckdb from apr table -> call to calculateAprResult - if (assetInfos.length === 0) return; - const duckDb = DuckDb.instances; - const pools = await Promise.all(assetInfos.map((infos) => duckDb.getPoolByAssetInfos(infos))); - - const allLiquidities = (await Promise.allSettled(pools.map((pair) => getPairLiquidity(pair)))).map((result) => { - if (result.status === "fulfilled") return result.value; - else console.error("error get allLiquidities: ", result.reason); - }); - - const poolAprInfos = await Promise.all(pools.map((pool) => duckDb.getLatestPoolApr(pool.pairAddr))); - const allTotalSupplies = poolAprInfos.map((item) => item.totalSupply); - const allBondAmounts = poolAprInfos.map((info) => info.totalBondAmount); - const allRewardPerSecs = poolAprInfos.map((info) => (info.rewardPerSec ? JSON.parse(info.rewardPerSec) : null)); - - const APRs = await calculateAprResult(allLiquidities, allTotalSupplies, allBondAmounts, allRewardPerSecs); - const latestPoolAprs = await Promise.all(pools.map((pool) => duckDb.getLatestPoolApr(pool.pairAddr))); - const newPoolAprs = latestPoolAprs.map((poolApr, index) => { - return { - ...poolApr, - height: newOffset, - apr: APRs[index], - uniqueKey: concatAprHistoryToUniqueKey({ - timestamp: Date.now(), - supply: allTotalSupplies[index], - bond: allBondAmounts[index], - reward: allRewardPerSecs[index], - apr: APRs[index], - pairAddr: pools[index].pairAddr - }) - }; - }); - await duckDb.insertPoolAprs(newPoolAprs); - } catch (error) { - console.log({ triggerCalculateApr: error }); - } -}; - -export const refetchTotalSupplies = async (assetInfos: [AssetInfo, AssetInfo][], height: number) => { + // get all infos relate to apr in duckdb from apr table -> call to calculateAprResult if (assetInfos.length === 0) return; const duckDb = DuckDb.instances; - const pools = await Promise.all(assetInfos.map((assetInfo) => duckDb.getPoolByAssetInfos(assetInfo))); - const liquidityAddrs = pools.map((pair) => pair.liquidityAddr); - const tokenInfos = await fetchTokenInfos(liquidityAddrs); - const totalSupplies = tokenInfos.map((info) => info.total_supply); + const pools = await Promise.all(assetInfos.map((infos) => duckDb.getPoolByAssetInfos(infos))); - const latestPoolAprs = await Promise.all(pools.map((pool) => duckDb.getLatestPoolApr(pool.pairAddr))); - const newPoolAprs = latestPoolAprs.map((poolApr, index) => { + const allLiquidities = (await Promise.allSettled(pools.map((pair) => getPairLiquidity(pair)))).map((result) => { + if (result.status === "fulfilled") return result.value; + else console.error("error get allLiquidities: ", result.reason); + }); + + const poolAprInfos = await Promise.all(pools.map((pool) => duckDb.getLatestPoolApr(pool.pairAddr))); + const allTotalSupplies = poolAprInfos.map((item) => item.totalSupply); + const allBondAmounts = poolAprInfos.map((info) => info.totalBondAmount); + const allRewardPerSecs = poolAprInfos.map((info) => (info.rewardPerSec ? JSON.parse(info.rewardPerSec) : null)); + + const APRs = await calculateAprResult(allLiquidities, allTotalSupplies, allBondAmounts, allRewardPerSecs); + console.dir({ APRs }, { depth: null }); + const newPoolAprs = poolAprInfos.map((poolApr, index) => { return { ...poolApr, - height, - totalSupply: totalSupplies[index] + height: newOffset, + apr: APRs[index], + uniqueKey: concatAprHistoryToUniqueKey({ + timestamp: Date.now(), + supply: allTotalSupplies[index], + bond: allBondAmounts[index], + reward: allRewardPerSecs[index], + apr: APRs[index], + pairAddr: pools[index].pairAddr + }) }; }); await duckDb.insertPoolAprs(newPoolAprs); }; -export const refetchTotalBond = async (assetInfos: [AssetInfo, AssetInfo][], height: number) => { +export type TypeInfoRelatedApr = "totalSupply" | "totalBondAmount" | "rewardPerSec"; +export const refetchInfoApr = async ( + type: TypeInfoRelatedApr, + assetInfos: [AssetInfo, AssetInfo][], + height: number +) => { if (assetInfos.length === 0) return; const duckDb = DuckDb.instances; const pools = await Promise.all(assetInfos.map((assetInfo) => duckDb.getPoolByAssetInfos(assetInfo))); const stakingAssetInfo = pools.map((pair) => getStakingAssetInfo([JSON.parse(pair.firstAssetInfo), JSON.parse(pair.secondAssetInfo)]) ); - const tokenAssetPools = await fetchAllTokenAssetPools(stakingAssetInfo); - const totalBondAmounts = tokenAssetPools.map((info) => info.total_bond_amount); + let newInfos; + switch (type) { + case "totalSupply": { + newInfos = await refetchTotalSupplies(pools); + break; + } + case "totalBondAmount": { + newInfos = await refetchTotalBond(stakingAssetInfo); + break; + } + case "rewardPerSec": { + newInfos = await refetchRewardPerSecInfos(stakingAssetInfo); + break; + } + default: + break; + } + const latestPoolAprs = await Promise.all(pools.map((pool) => duckDb.getLatestPoolApr(pool.pairAddr))); const newPoolAprs = latestPoolAprs.map((poolApr, index) => { return { ...poolApr, height, - totalBondAmount: totalBondAmounts[index] + [type]: newInfos[index] }; }); await duckDb.insertPoolAprs(newPoolAprs); }; -export const refetchRewardPerSecInfos = async (assetInfos: [AssetInfo, AssetInfo][], height: number) => { - if (assetInfos.length === 0) return; - const duckDb = DuckDb.instances; - const pools = await Promise.all(assetInfos.map((assetInfo) => duckDb.getPoolByAssetInfos(assetInfo))); - const assetTokens = pools.map((pair) => - getStakingAssetInfo([JSON.parse(pair.firstAssetInfo), JSON.parse(pair.secondAssetInfo)]) - ); - const rewardPerSecInfos = await fetchAllRewardPerSecInfos(assetTokens); +export const refetchTotalSupplies = async (pools: PairInfoData[]): Promise => { + const liquidityAddrs = pools.map((pair) => pair.liquidityAddr); + const tokenInfos = await fetchTokenInfos(liquidityAddrs); + const totalSupplies = tokenInfos.map((info) => info.total_supply); + return totalSupplies; +}; - const latestPoolAprs = await Promise.all(pools.map((pool) => duckDb.getLatestPoolApr(pool.pairAddr))); - const newPoolAprs = latestPoolAprs.map((poolApr, index) => { - return { - ...poolApr, - height, - rewardPerSec: JSON.stringify(rewardPerSecInfos[index]) - }; - }); - await duckDb.insertPoolAprs(newPoolAprs); +export const refetchTotalBond = async (stakingAssetInfo: AssetInfo[]): Promise => { + const tokenAssetPools = await fetchAllTokenAssetPools(stakingAssetInfo); + const totalBondAmounts = tokenAssetPools.map((info) => info.total_bond_amount); + return totalBondAmounts; +}; + +export const refetchRewardPerSecInfos = async (stakingAssetInfo: AssetInfo[]) => { + const rewardPerSecInfos = await fetchAllRewardPerSecInfos(stakingAssetInfo); + return rewardPerSecInfos.map((item) => JSON.stringify(item)); }; -export const handleEventApr = async (txs: Tx[], result: TxAnlysisResult, newOffset: number): Promise => { +export const getListAssetInfoShouldRefetchApr = async (txs: Tx[], lpOps: ProvideLiquidityOperationData[]) => { let listAssetInfosPoolShouldRefetch = new Set<[AssetInfo, AssetInfo]>(); // mint/burn trigger update total supply const assetInfosTriggerTotalSupplies = Array.from( - [...result.provideLiquidityOpsData, ...result.withdrawLiquidityOpsData] + lpOps .map((op) => [op.baseTokenDenom, op.quoteTokenDenom] as [string, string]) .reduce((accumulator, tokenDenoms) => { const assetInfo = parsePairDenomToAssetInfo(tokenDenoms); @@ -384,15 +380,7 @@ export const handleEventApr = async (txs: Tx[], result: TxAnlysisResult, newOffs ); assetInfosTriggerTotalSupplies.forEach((item) => listAssetInfosPoolShouldRefetch.add(item)); - await refetchTotalSupplies(assetInfosTriggerTotalSupplies, newOffset); - - const { infoTokenAssetPools, isTriggerRewardPerSec } = await processEventApr(txs); - if (isTriggerRewardPerSec) { - // update_reward_per_sec trigger refetch all info - pairs.map((pair) => pair.asset_infos).forEach((assetInfos) => listAssetInfosPoolShouldRefetch.add(assetInfos)); - await refetchRewardPerSecInfos(Array.from(listAssetInfosPoolShouldRefetch), newOffset); - } - + const { infoTokenAssetPools, isTriggerRewardPerSec } = processEventApr(txs); // bond/unbond trigger refetch info token asset pools const assetInfosTriggerTotalBond = Array.from(infoTokenAssetPools) .map((stakingDenom) => { @@ -401,9 +389,39 @@ export const handleEventApr = async (txs: Tx[], result: TxAnlysisResult, newOffs }) .filter(Boolean); - !isTriggerRewardPerSec && + if (isTriggerRewardPerSec) { + // update_reward_per_sec trigger refetch all info + listAssetInfosPoolShouldRefetch.clear(); + pairs.map((pair) => pair.asset_infos).forEach((assetInfos) => listAssetInfosPoolShouldRefetch.add(assetInfos)); + } else { assetInfosTriggerTotalBond.forEach((assetInfo) => listAssetInfosPoolShouldRefetch.add(assetInfo)); - await refetchTotalBond(assetInfosTriggerTotalBond, newOffset); + } + + return { + assetInfosTriggerTotalSupplies, + listAssetInfosPoolShouldRefetch: Array.from(listAssetInfosPoolShouldRefetch), + assetInfosTriggerTotalBond, + assetInfosTriggerRewardPerSec: isTriggerRewardPerSec ? Array.from(listAssetInfosPoolShouldRefetch) : [] + }; +}; + +export const handleEventApr = async ( + txs: Tx[], + result: ProvideLiquidityOperationData[], + newOffset: number +): Promise => { + const { + assetInfosTriggerTotalSupplies, + listAssetInfosPoolShouldRefetch, + assetInfosTriggerTotalBond, + assetInfosTriggerRewardPerSec + } = await getListAssetInfoShouldRefetchApr(txs, result); + + await Promise.allSettled([ + refetchInfoApr("totalSupply", assetInfosTriggerTotalSupplies, newOffset), + refetchInfoApr("rewardPerSec", assetInfosTriggerRewardPerSec, newOffset), + refetchInfoApr("totalBondAmount", assetInfosTriggerTotalBond, newOffset) + ]); await triggerCalculateApr(Array.from(listAssetInfosPoolShouldRefetch), newOffset); }; diff --git a/packages/oraidex-sync/src/test-db.ts b/packages/oraidex-sync/src/test-db.ts deleted file mode 100644 index b7d20c03..00000000 --- a/packages/oraidex-sync/src/test-db.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { CosmWasmClient, OraiswapRouterQueryClient, SwapOperation } from "@oraichain/oraidex-contracts-sdk"; -import { DuckDb } from "./db"; -import { SwapOperationData } from "./types"; -import { pairs, uniqueInfos } from "./pairs"; -import { parseAssetInfoOnlyDenom } from "./helper"; -import { simulateSwapPriceWithUsdt } from "./query"; -import "dotenv/config"; - -export function getDate24hBeforeNow(time: Date) { - const twentyFourHoursInMilliseconds = 24 * 60 * 60 * 1000; // 24 hours in milliseconds - const date24hBeforeNow = new Date(time.getTime() - twentyFourHoursInMilliseconds); - return date24hBeforeNow; -} - -const start = async () => { - const duckdb = await DuckDb.create("oraidex-sync-data"); - const tf = 86400; - const now = new Date(); - const then = getDate24hBeforeNow(now); - const firstTokenResult = await duckdb.conn.all("select * from swap_ohlcv limit 5"); - console.log(firstTokenResult); - - // let swapTokenMap = []; - // const baseVolume = 1000000000; - // for (let i = 0; i < swapOps.length; i++) { - // const indexOf = swapTokenMap.findIndex( - // (swapMap) => new Date(swapMap.timestamp).toISOString() === new Date(swapOps[i].timestamp).toISOString() - // ); - // console.log("index of: ", indexOf); - // if (indexOf === -1) { - // swapTokenMap.push({ - // timestamp: swapOps[i].timestamp, - // tokenData: uniqueInfos.map((info) => { - // if (parseAssetInfoOnlyDenom(info) === swapOps[i].offerDenom) { - // return { denom: swapOps[i].offerDenom, amount: swapOps[i].offerAmount }; - // } - // if (parseAssetInfoOnlyDenom(info) === swapOps[i].askDenom) { - // return { denom: swapOps[i].askDenom, amount: swapOps[i].returnAmount }; - // } - // return { denom: parseAssetInfoOnlyDenom(info), amount: 0 }; - // }) - // }); - // } else { - // swapTokenMap[indexOf] = { - // ...swapTokenMap[indexOf], - // tokenData: swapTokenMap[indexOf].tokenData.map((tokenData) => { - // if (tokenData.denom === swapOps[i].offerDenom) - // return { ...tokenData, amount: tokenData.amount + swapOps[i].offerAmount }; - // if (tokenData.denom === swapOps[i].askDenom) - // return { ...tokenData, amount: tokenData.amount + swapOps[i].returnAmount }; - // return tokenData; - // }) - // }; - // } - // } - // console.log( - // swapTokenMap.map((tokenMap) => ({ - // ...tokenMap, - // tokenData: tokenMap.tokenData.reduce((acc, item) => { - // acc[item.denom] = item.amount; - // return acc; - // }, {}) - // })) - // ); - // const newData = calculatePrefixSum( - // 100000000000, - // result.map((res) => ({ denom: "", amount: res.liquidity })) - // ); - // console.log("new data: ", newData); -}; - -start(); diff --git a/packages/oraidex-sync/src/tx-parsing.ts b/packages/oraidex-sync/src/tx-parsing.ts index 17ee14b2..2edecb34 100644 --- a/packages/oraidex-sync/src/tx-parsing.ts +++ b/packages/oraidex-sync/src/tx-parsing.ts @@ -368,7 +368,7 @@ async function parseTxs(txs: Tx[]): Promise { }; } -export const processEventApr = async (txs: Tx[]) => { +export const processEventApr = (txs: Tx[]) => { const assets = { infoTokenAssetPools: new Set(), isTriggerRewardPerSec: false diff --git a/packages/oraidex-sync/src/types.ts b/packages/oraidex-sync/src/types.ts index 2913aef3..64248d39 100644 --- a/packages/oraidex-sync/src/types.ts +++ b/packages/oraidex-sync/src/types.ts @@ -241,5 +241,4 @@ export type PoolApr = { totalBondAmount: string; rewardPerSec: string; apr: number; - createdAt?: string; }; diff --git a/packages/oraidex-sync/tests/db.spec.ts b/packages/oraidex-sync/tests/db.spec.ts index 8f60f0b6..12b78c24 100644 --- a/packages/oraidex-sync/tests/db.spec.ts +++ b/packages/oraidex-sync/tests/db.spec.ts @@ -1,8 +1,6 @@ -import fs from "fs"; -import { oraiInfo, usdtInfo } from "../src"; import { DuckDb } from "../src/db"; import { isoToTimestampNumber } from "../src/helper"; -import { GetFeeSwap, GetVolumeQuery, PairInfoData, PoolApr, ProvideLiquidityOperationData } from "../src/types"; +import { GetFeeSwap, GetVolumeQuery, ProvideLiquidityOperationData } from "../src/types"; describe("test-duckdb", () => { let duckDb: DuckDb; afterAll(jest.resetModules); @@ -461,7 +459,7 @@ describe("test-duckdb", () => { totalBondAmount: "1", rewardPerSec: "1", apr: 2 - } as PoolApr, + }, { uniqueKey: "orai_usdt_4", pairAddr: "orai_usdt", @@ -470,7 +468,7 @@ describe("test-duckdb", () => { totalBondAmount: "1", rewardPerSec: "1", apr: 4 - } as PoolApr, + }, { uniqueKey: "orai_usdt_3", pairAddr: "orai_usdt", @@ -479,7 +477,7 @@ describe("test-duckdb", () => { totalBondAmount: "1", rewardPerSec: "1", apr: 3 - } as PoolApr, + }, { uniqueKey: "orai_atom", pairAddr: "orai_atom", @@ -488,7 +486,7 @@ describe("test-duckdb", () => { totalBondAmount: "1", rewardPerSec: "1", apr: 2 - } as PoolApr + } ]); }); @@ -508,7 +506,7 @@ describe("test-duckdb", () => { const result = await duckDb.getLatestPoolApr("orai_usdt"); // assertion - expect(result).toEqual({ + expect(result).toMatchObject({ uniqueKey: "orai_usdt_4", pairAddr: "orai_usdt", height: 4, diff --git a/packages/oraidex-sync/tests/helper.spec.ts b/packages/oraidex-sync/tests/helper.spec.ts index 3fc255c2..24b1046e 100644 --- a/packages/oraidex-sync/tests/helper.spec.ts +++ b/packages/oraidex-sync/tests/helper.spec.ts @@ -748,7 +748,7 @@ describe("test-helper", () => { "test-getPairLiquidity-should-return-correctly-liquidity-by-USDT", async (offerPoolAmount: bigint, askPoolAmount: bigint, expectedResult: number) => { // setup - await duckDb.createPoolOpsTable(); + await duckDb.createLpAmountHistoryTable(); await duckDb.insertPoolAmountHistory([ { offerPoolAmount, diff --git a/packages/oraidex-sync/tests/pool-helper.spec.ts b/packages/oraidex-sync/tests/pool-helper.spec.ts index b9e2b49b..ffadf021 100644 --- a/packages/oraidex-sync/tests/pool-helper.spec.ts +++ b/packages/oraidex-sync/tests/pool-helper.spec.ts @@ -14,6 +14,9 @@ import * as helper from "../src/helper"; import { DuckDb, pairs } from "../src/index"; import * as poolHelper from "../src/pool-helper"; import { PairInfoData, PairMapping, ProvideLiquidityOperationData } from "../src/types"; +import { Tx } from "@oraichain/cosmos-rpc-sync"; +import { Tx as CosmosTx } from "cosmjs-types/cosmos/tx/v1beta1/tx"; +import * as txParsing from "../src/tx-parsing"; describe("test-pool-helper", () => { let duckDb: DuckDb; @@ -116,13 +119,15 @@ describe("test-pool-helper", () => { ])("test-getPriceByAsset-should-return-correctly-price", async (assetInfos, ratioDirection, expectedPrice) => { // setup duckDb = await DuckDb.create(":memory:"); - await duckDb.createPairInfosTable(); + await Promise.all([duckDb.createPairInfosTable(), duckDb.createLpAmountHistoryTable()]); + + const pairAddr = "orai1c5s03c3l336dgesne7dylnmhszw8554tsyy9yt"; let pairInfoData: PairInfoData[] = [ { firstAssetInfo: JSON.stringify({ native_token: { denom: ORAI } } as AssetInfo), secondAssetInfo: JSON.stringify({ token: { contract_addr: usdtCw20Address } } as AssetInfo), commissionRate: "", - pairAddr: "orai1c5s03c3l336dgesne7dylnmhszw8554tsyy9yt", + pairAddr, liquidityAddr: "", oracleAddr: "", symbols: "1", @@ -131,12 +136,17 @@ describe("test-pool-helper", () => { } ]; await duckDb.insertPairInfos(pairInfoData); + await duckDb.insertPoolAmountHistory([ + { + offerPoolAmount: 1n, + askPoolAmount: 1n, + height: 1, + timestamp: 1, + pairAddr, + uniqueKey: "1" + } + ]); - // mock result of nested function implemented inside getPriceByAsset. - jest.spyOn(helper, "fetchPoolInfoAmount").mockResolvedValue({ - askPoolAmount: 1000n, - offerPoolAmount: 2000n - }); jest.spyOn(helper, "calculatePriceByPool").mockReturnValue(0.5); // assert @@ -398,4 +408,97 @@ describe("test-pool-helper", () => { expect(result.length).toEqual(pairs.length); expect(result).toStrictEqual(Array(pairs.length).fill(315360000)); }); + + it.each([ + [true, pairs.length, pairs.length], + [false, 4, 0] + ])( + "test-getListAssetInfoShouldRefetchApr-with-is-isTriggerRewardPerSec-%p-shoud-return-listAssetInfosPoolShouldRefetch-length-%p-and-assetInfosTriggerRewardPerSec-length-%p", + async ( + isTriggerRewardPerSec: boolean, + expectedListAssetInfosPoolShouldRefetch: number, + expectedAssetInfosTriggerRewardPerSec: number + ) => { + // setup + const cosmosTx = CosmosTx.encode( + CosmosTx.fromPartial({ body: { messages: [{ typeUrl: "/cosmwasm.wasm.v1.MsgExecuteContract" }] } }) + ).finish(); + const txs: Tx[] = [ + { + hash: "", + height: 1, + code: 1, + txIndex: 0, + tx: cosmosTx, + timestamp: new Date().toISOString(), + rawLog: JSON.stringify({ events: [] }), + events: [], + msgResponses: [{ typeUrl: "", value: Buffer.from("") }], + gasUsed: 1, + gasWanted: 1 + } + ]; + + const ops: ProvideLiquidityOperationData[] = [ + { + basePrice: 1, + baseTokenAmount: 1, + baseTokenDenom: ORAI, + quoteTokenAmount: 1, + quoteTokenDenom: usdtCw20Address, + baseTokenReserve: 1, + quoteTokenReserve: 1, + opType: "provide", + uniqueKey: "1", + timestamp: 1, + txCreator: "a", + txhash: "a", + txheight: 1, + taxRate: 1n + }, + { + basePrice: 1, + baseTokenAmount: 1, + baseTokenDenom: ORAI, + quoteTokenAmount: 1, + quoteTokenDenom: usdtCw20Address, + baseTokenReserve: 1, + quoteTokenReserve: 1, + opType: "withdraw", + uniqueKey: "2", + timestamp: 1, + txCreator: "a", + txhash: "a", + txheight: 1, + taxRate: 1n + }, + { + basePrice: 1, + baseTokenAmount: 1, + baseTokenDenom: ORAI, + quoteTokenAmount: 1, + quoteTokenDenom: atomIbcDenom, + baseTokenReserve: 1, + quoteTokenReserve: 1, + opType: "withdraw", + uniqueKey: "1", + timestamp: 1, + txCreator: "a", + txhash: "a", + txheight: 1, + taxRate: 1n + } + ]; + jest.spyOn(txParsing, "processEventApr").mockReturnValue({ + isTriggerRewardPerSec, + infoTokenAssetPools: new Set([airiCw20Adress, scAtomCw20Address]) + }); + + const result = await poolHelper.getListAssetInfoShouldRefetchApr(txs, ops); + expect(result.assetInfosTriggerTotalSupplies.length).toEqual(2); + expect(result.assetInfosTriggerTotalBond.length).toEqual(2); + expect(result.listAssetInfosPoolShouldRefetch.length).toEqual(expectedListAssetInfosPoolShouldRefetch); + expect(result.assetInfosTriggerRewardPerSec.length).toEqual(expectedAssetInfosTriggerRewardPerSec); + } + ); }); From b177df99c4344a78559ea949d144705adea666e9 Mon Sep 17 00:00:00 2001 From: trungbach Date: Fri, 15 Sep 2023 02:37:12 +0700 Subject: [PATCH 46/46] use promise all to minimize time blocking --- packages/oraidex-server/src/index.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/oraidex-server/src/index.ts b/packages/oraidex-server/src/index.ts index 05aac142..7a478b42 100644 --- a/packages/oraidex-server/src/index.ts +++ b/packages/oraidex-server/src/index.ts @@ -236,14 +236,13 @@ app.get("/v1/candles/", async (req: Request<{}, {}, {}, GetCandlesQuery>, res) = app.get("/v1/pools/", async (_req, res) => { try { - const [volumes, allFee7Days] = await Promise.all([getAllVolume24h(), getAllFees()]); - - const pools = await duckDb.getPools(); - const allPoolApr = await duckDb.getApr(); - const allLiquidities = (await Promise.allSettled(pools.map((pair) => getPairLiquidity(pair)))).map((result) => { - if (result.status === "fulfilled") return result.value; - else console.error("error get allLiquidities: ", result.reason); - }); + const [volumes, allFee7Days, pools, allPoolApr] = await Promise.all([ + getAllVolume24h(), + getAllFees(), + duckDb.getPools(), + duckDb.getApr() + ]); + const allLiquidities = await Promise.all(pools.map((pair) => getPairLiquidity(pair))); res.status(200).send( pools.map((pool, index) => {