From 87bf0ef3bbd4e889d65cefb715a3e3cf6aed1091 Mon Sep 17 00:00:00 2001 From: Bernardo Garces Chapero Date: Tue, 24 Sep 2024 18:06:53 +0100 Subject: [PATCH] injectable unwrap cache (#323) --- .../adapters-library/src/core/getProfits.ts | 15 ++- .../adapters-library/src/core/utils/unwrap.ts | 27 +++- packages/adapters-library/src/defiProvider.ts | 27 +++- packages/adapters-library/src/index.ts | 1 + .../adapters-library/src/unwrapCache.test.ts | 116 ++++++++++++++++++ packages/adapters-library/src/unwrapCache.ts | 77 ++++++++++++ 6 files changed, 251 insertions(+), 12 deletions(-) create mode 100644 packages/adapters-library/src/unwrapCache.test.ts create mode 100644 packages/adapters-library/src/unwrapCache.ts diff --git a/packages/adapters-library/src/core/getProfits.ts b/packages/adapters-library/src/core/getProfits.ts index 9c7ef942b..99f00c8e6 100644 --- a/packages/adapters-library/src/core/getProfits.ts +++ b/packages/adapters-library/src/core/getProfits.ts @@ -12,6 +12,7 @@ import { Underlying, } from '../types/adapter' import { Erc20Metadata } from '../types/erc20Metadata' +import { IUnwrapCache } from '../unwrapCache' import { aggregateFiatBalances } from './utils/aggregateFiatBalances' import { aggregateFiatBalancesFromMovements } from './utils/aggregateFiatBalancesFromMovements' import { calculateDeFiAttributionPerformance } from './utils/calculateDeFiAttributionPerformance' @@ -25,6 +26,7 @@ export async function getProfits({ protocolTokenAddresses, tokenIds, includeRawValues, + unwrapCache, }: { adapter: IProtocolAdapter userAddress: string @@ -33,6 +35,7 @@ export async function getProfits({ protocolTokenAddresses?: string[] tokenIds?: string[] includeRawValues?: boolean + unwrapCache: IUnwrapCache }): Promise { let endPositionValues: ReturnType let startPositionValues: ReturnType @@ -52,7 +55,7 @@ export async function getProfits({ tokenIds, }) .then(async (result) => { - await unwrap(adapter, toBlock, result, 'balanceRaw') + await unwrap(adapter, toBlock, result, 'balanceRaw', unwrapCache) return result }) .then((result) => { @@ -67,7 +70,7 @@ export async function getProfits({ tokenIds, }) .then(async (result) => { - await unwrap(adapter, fromBlock, result, 'balanceRaw') + await unwrap(adapter, fromBlock, result, 'balanceRaw', unwrapCache) return result }) .then((result) => { @@ -83,7 +86,7 @@ export async function getProfits({ blockNumber: toBlock, }) .then(async (result) => { - await unwrap(adapter, toBlock, result, 'balanceRaw') + await unwrap(adapter, toBlock, result, 'balanceRaw', unwrapCache) return result }) .then((result) => { @@ -99,7 +102,7 @@ export async function getProfits({ tokenIds: Object.keys(endPositionValues), // endPositionValues is indexed by tokenId ?? protocolTokenAddress }) .then(async (result) => { - await unwrap(adapter, fromBlock, result, 'balanceRaw') + await unwrap(adapter, fromBlock, result, 'balanceRaw', unwrapCache) return result }) .then((result) => { @@ -140,6 +143,7 @@ export async function getProfits({ positionMovements.blockNumber, positionMovements.tokens, 'balanceRaw', + unwrapCache, ) }), ) @@ -169,6 +173,7 @@ export async function getProfits({ positionMovements.blockNumber, positionMovements.tokens, 'balanceRaw', + unwrapCache, ) }), ) @@ -198,6 +203,7 @@ export async function getProfits({ positionMovements.blockNumber, positionMovements.tokens, 'balanceRaw', + unwrapCache, ) }), ) @@ -227,6 +233,7 @@ export async function getProfits({ positionMovements.blockNumber, positionMovements.tokens, 'balanceRaw', + unwrapCache, ) }), ) diff --git a/packages/adapters-library/src/core/utils/unwrap.ts b/packages/adapters-library/src/core/utils/unwrap.ts index 55b9ebdab..72ca7a285 100644 --- a/packages/adapters-library/src/core/utils/unwrap.ts +++ b/packages/adapters-library/src/core/utils/unwrap.ts @@ -2,6 +2,7 @@ import { Protocol } from '../../adapters/protocols' import { IProtocolAdapter } from '../../types/IProtocolAdapter' import { TokenType, UnderlyingTokenTypeMap } from '../../types/adapter' import { Erc20Metadata } from '../../types/erc20Metadata' +import { IUnwrapCache } from '../../unwrapCache' import { AdapterMissingError, NotImplementedError, @@ -20,10 +21,11 @@ export async function unwrap( blockNumber: number | undefined, tokens: Token[], fieldToUpdate: string, + unwrapCache: IUnwrapCache, ) { return await Promise.all( tokens.map(async (token) => { - await unwrapToken(adapter, blockNumber, token, fieldToUpdate) + await unwrapToken(adapter, blockNumber, token, fieldToUpdate, unwrapCache) }), ) } @@ -33,6 +35,7 @@ async function unwrapToken( blockNumber: number | undefined, token: Token, fieldToUpdate: string, + unwrapCache: IUnwrapCache, ) { const underlyingProtocolTokenAdapter = await adapter.adaptersController.fetchTokenAdapter( @@ -45,7 +48,12 @@ async function unwrapToken( if (!underlyingProtocolTokenAdapter) { // Try to fetch prices if there is no tokens and no adapter to resolve - const tokenPriceRaw = await fetchPrice(adapter, token, blockNumber) + const tokenPriceRaw = await fetchPrice( + adapter, + token, + blockNumber, + unwrapCache, + ) if (tokenPriceRaw) { token.priceRaw = tokenPriceRaw } @@ -55,6 +63,7 @@ async function unwrapToken( underlyingProtocolTokenAdapter, token, blockNumber, + unwrapCache, ) if (unwrapExchangeRates?.tokens) { @@ -82,7 +91,13 @@ async function unwrapToken( await Promise.all( token.tokens?.map(async (underlyingToken) => { - await unwrapToken(adapter, blockNumber, underlyingToken, fieldToUpdate) + await unwrapToken( + adapter, + blockNumber, + underlyingToken, + fieldToUpdate, + unwrapCache, + ) }) ?? [], ) } @@ -91,9 +106,10 @@ async function fetchUnwrapExchangeRates( underlyingProtocolTokenAdapter: IProtocolAdapter, underlyingProtocolTokenPosition: Token, blockNumber: number | undefined, + unwrapCache: IUnwrapCache, ) { try { - return await underlyingProtocolTokenAdapter.unwrap({ + return await unwrapCache.fetchWithCache(underlyingProtocolTokenAdapter, { protocolTokenAddress: underlyingProtocolTokenPosition.address, blockNumber, }) @@ -114,6 +130,7 @@ async function fetchPrice( adapter: IProtocolAdapter, token: Erc20Metadata & { priceRaw?: bigint }, blockNumber: number | undefined, + unwrapCache: IUnwrapCache, ) { let priceAdapter: IProtocolAdapter try { @@ -131,7 +148,7 @@ async function fetchPrice( } try { - const price = await priceAdapter.unwrap({ + const price = await unwrapCache.fetchWithCache(priceAdapter, { protocolTokenAddress: token.address, blockNumber, }) diff --git a/packages/adapters-library/src/defiProvider.ts b/packages/adapters-library/src/defiProvider.ts index fa5275134..8857ec1d8 100644 --- a/packages/adapters-library/src/defiProvider.ts +++ b/packages/adapters-library/src/defiProvider.ts @@ -46,6 +46,7 @@ import { } from './types/response' import { existsSync } from 'node:fs' +import { IUnwrapCache, IUnwrapCacheProvider, UnwrapCache } from './unwrapCache' function buildMetadataProviders(): Record { return Object.values(Chain).reduce( @@ -77,12 +78,15 @@ export class DefiProvider { private adaptersControllerWithoutPrices: AdaptersController private metadataProviders: Record + private unwrapCache: IUnwrapCache constructor( config?: DeepPartial, metadataProviders?: Record, + unwrapCacheProvider?: IUnwrapCacheProvider, ) { this.metadataProviders = metadataProviders ?? buildMetadataProviders() + this.unwrapCache = new UnwrapCache(unwrapCacheProvider) this.parsedConfig = new Config(config) this.chainProvider = new ChainProvider(this.parsedConfig.values) @@ -195,7 +199,13 @@ export class DefiProvider { const getRewardTime = Date.now() - await unwrap(adapter, blockNumber, protocolPositions, 'balanceRaw') + await unwrap( + adapter, + blockNumber, + protocolPositions, + 'balanceRaw', + this.unwrapCache, + ) const unwrapTime = Date.now() @@ -392,6 +402,7 @@ export class DefiProvider { protocolTokenAddresses, tokenIds: filterTokenIds, includeRawValues, + unwrapCache: this.unwrapCache, }) const endTime = Date.now() @@ -461,7 +472,7 @@ export class DefiProvider { protocolTokens.map(async (address) => { const startTime = Date.now() - const unwrap = await adapter.unwrap({ + const unwrap = await this.unwrapCache.fetchWithCache(adapter, { protocolTokenAddress: getAddress(address), blockNumber, }) @@ -571,6 +582,7 @@ export class DefiProvider { positionMovements.blockNumber, positionMovements.tokens, 'balanceRaw', + this.unwrapCache, ) }), ) @@ -654,6 +666,7 @@ export class DefiProvider { positionMovements.blockNumber, positionMovements.tokens, 'balanceRaw', + this.unwrapCache, ) }), ) @@ -707,6 +720,7 @@ export class DefiProvider { positionMovements.blockNumber, positionMovements.tokens, 'balanceRaw', + this.unwrapCache, ) }), ) @@ -760,6 +774,7 @@ export class DefiProvider { positionMovements.blockNumber, positionMovements.tokens, 'balanceRaw', + this.unwrapCache, ) }), ) @@ -793,7 +808,13 @@ export class DefiProvider { blockNumber, }) - await unwrap(adapter, blockNumber, tokens, 'totalSupplyRaw') + await unwrap( + adapter, + blockNumber, + tokens, + 'totalSupplyRaw', + this.unwrapCache, + ) return { tokens: tokens.map((value) => diff --git a/packages/adapters-library/src/index.ts b/packages/adapters-library/src/index.ts index d910fb7ad..3dce52264 100644 --- a/packages/adapters-library/src/index.ts +++ b/packages/adapters-library/src/index.ts @@ -17,3 +17,4 @@ export { } from './adapters/supportedProtocols' export { SQLiteMetadataProvider } from './SQLiteMetadataProvider' +export type { IUnwrapCacheProvider } from './unwrapCache' diff --git a/packages/adapters-library/src/unwrapCache.test.ts b/packages/adapters-library/src/unwrapCache.test.ts new file mode 100644 index 000000000..1d412a73a --- /dev/null +++ b/packages/adapters-library/src/unwrapCache.test.ts @@ -0,0 +1,116 @@ +import { AVERAGE_BLOCKS_PER_10_MINUTES } from './core/constants/AVERAGE_BLOCKS_PER_10_MINS' +import { Chain } from './core/constants/chains' +import { IProtocolAdapter } from './types/IProtocolAdapter' +import { + IUnwrapCacheProvider, + UnwrapCache, + getTenMinuteKeyByBlock, +} from './unwrapCache' + +describe('UnwrapCache', () => { + describe('fetchWithCache', () => { + const unwrapResult = {} + const mockAdapter = { + chainId: Chain.Ethereum, + unwrap: jest.fn().mockResolvedValue(unwrapResult), + } as unknown as IProtocolAdapter + + const unwrapInput = { + protocolTokenAddress: '0x123', + blockNumber: 123, + } + + beforeEach(() => { + jest.clearAllMocks() + }) + + it('returns immediatelly if there is no provider', async () => { + const unwrapCache = new UnwrapCache() + + const result = await unwrapCache.fetchWithCache(mockAdapter, unwrapInput) + + expect(result).toEqual(unwrapResult) + expect(mockAdapter.unwrap).toHaveBeenCalledWith(unwrapInput) + }) + + it('misses cache if no value is provided', async () => { + const unwrapCacheProvider = { + getFromDb: jest.fn(), + setToDb: jest.fn(), + } as IUnwrapCacheProvider + const unwrapCache = new UnwrapCache(unwrapCacheProvider) + + const result = await unwrapCache.fetchWithCache(mockAdapter, unwrapInput) + + expect(result).toEqual(unwrapResult) + expect(unwrapCacheProvider.getFromDb).toHaveBeenCalled() + expect(unwrapCacheProvider.setToDb).toHaveBeenCalled() + expect(mockAdapter.unwrap).toHaveBeenCalledWith(unwrapInput) + }) + + it('hits cache if a value is provided', async () => { + const unwrapCacheProvider = { + getFromDb: jest.fn().mockResolvedValueOnce({}), + setToDb: jest.fn(), + } as IUnwrapCacheProvider + const unwrapCache = new UnwrapCache(unwrapCacheProvider) + + const result = await unwrapCache.fetchWithCache(mockAdapter, unwrapInput) + + expect(result).toEqual(unwrapResult) + expect(unwrapCacheProvider.getFromDb).toHaveBeenCalled() + expect(unwrapCacheProvider.setToDb).not.toHaveBeenCalled() + expect(mockAdapter.unwrap).not.toHaveBeenCalledWith(unwrapInput) + }) + }) +}) + +describe('getTenMinuteKeyByBlock', () => { + it('returns same key with blocks within the same range', async () => { + const blockNumber1 = AVERAGE_BLOCKS_PER_10_MINUTES[Chain.Ethereum] * 10 + const blockNumber2 = AVERAGE_BLOCKS_PER_10_MINUTES[Chain.Ethereum] * 11 - 1 + + const result1 = getTenMinuteKeyByBlock(blockNumber1, Chain.Ethereum) + const result2 = getTenMinuteKeyByBlock(blockNumber2, Chain.Ethereum) + + expect(result1).toEqual(result2) + }) + + it('returns different key with blocks that are in different ranges', async () => { + const blockNumber1 = AVERAGE_BLOCKS_PER_10_MINUTES[Chain.Ethereum] * 10 + const blockNumber2 = AVERAGE_BLOCKS_PER_10_MINUTES[Chain.Ethereum] * 11 + + const result1 = getTenMinuteKeyByBlock(blockNumber1, Chain.Ethereum) + const result2 = getTenMinuteKeyByBlock(blockNumber2, Chain.Ethereum) + + expect(result1).not.toEqual(result2) + }) + + it('returns same key with timestamps within the same 10 minutes range', async () => { + const mockDateNow = jest + .spyOn(Date, 'now') + .mockImplementationOnce(() => new Date('2024-09-01T00:00:00Z').getTime()) + .mockImplementationOnce(() => new Date('2024-09-01T00:01:00Z').getTime()) + + const result1 = getTenMinuteKeyByBlock(undefined, Chain.Ethereum) + const result2 = getTenMinuteKeyByBlock(undefined, Chain.Ethereum) + + expect(result1).toEqual(result2) + + mockDateNow.mockRestore() + }) + + it('returns same key with timestamps that are in different ranges', async () => { + const mockDateNow = jest + .spyOn(Date, 'now') + .mockImplementationOnce(() => new Date('2024-09-01T00:00:00Z').getTime()) + .mockImplementationOnce(() => new Date('2024-09-01T00:10:00Z').getTime()) + + const result1 = getTenMinuteKeyByBlock(undefined, Chain.Ethereum) + const result2 = getTenMinuteKeyByBlock(undefined, Chain.Ethereum) + + expect(result1).not.toEqual(result2) + + mockDateNow.mockRestore() + }) +}) diff --git a/packages/adapters-library/src/unwrapCache.ts b/packages/adapters-library/src/unwrapCache.ts new file mode 100644 index 000000000..526520784 --- /dev/null +++ b/packages/adapters-library/src/unwrapCache.ts @@ -0,0 +1,77 @@ +import { AVERAGE_BLOCKS_PER_10_MINUTES } from './core/constants/AVERAGE_BLOCKS_PER_10_MINS' +import { Chain } from './core/constants/chains' +import { logger } from './core/utils/logger' +import { IProtocolAdapter } from './types/IProtocolAdapter' +import { UnwrapExchangeRate, UnwrapInput } from './types/adapter' + +const TEN_MINUTES_IN_MS = 10 * 60 * 1000 + +export interface IUnwrapCacheProvider { + getFromDb(key: string): Promise + setToDb(key: string, value: UnwrapExchangeRate): Promise +} + +export interface IUnwrapCache { + fetchWithCache( + adapter: IProtocolAdapter, + input: UnwrapInput, + ): Promise +} + +export class UnwrapCache implements IUnwrapCache { + constructor(private readonly unwrapCacheProvider?: IUnwrapCacheProvider) {} + + async fetchWithCache( + adapter: IProtocolAdapter, + input: UnwrapInput, + ): Promise { + if (!this.unwrapCacheProvider) { + return adapter.unwrap(input) + } + + const chainId = adapter.chainId + const key = `${chainId}:${ + input.protocolTokenAddress + }:${getTenMinuteKeyByBlock(input.blockNumber, chainId)}` + + const dbValue = await this.unwrapCacheProvider.getFromDb(key) + + if (dbValue) { + logger.debug({ key }, 'Unwrap cache hit') + return dbValue + } + + logger.debug({ key }, 'Unwrap cache miss') + + const value = await adapter.unwrap(input) + + await this.unwrapCacheProvider.setToDb(key, value) + + return value + } +} + +export function getTenMinuteKeyByBlock( + blockNumber: number | undefined, + chainId: Chain, +): string { + if (!blockNumber) { + const timestampKey = roundDownToNearestMultiple( + Date.now(), + TEN_MINUTES_IN_MS, + ) + + return `T${timestampKey}` + } + + const blockKey = roundDownToNearestMultiple( + blockNumber, + AVERAGE_BLOCKS_PER_10_MINUTES[chainId], + ) + + return `B${blockKey}` +} + +function roundDownToNearestMultiple(value: number, multiple: number): number { + return Math.floor(value / multiple) * multiple +}