From 9b1586df7eb8ff7fdd8475a3e5498786953d53c2 Mon Sep 17 00:00:00 2001 From: iamnovichek Date: Fri, 15 Nov 2024 21:14:57 +0100 Subject: [PATCH] updating data code replaced --- apps/dashboard_app/data_conector.py | 4 +- apps/dashboard_app/load_data.py | 171 ++++++++++ .../handlers/loan_states/zklend/run.py | 2 +- apps/shared/helpers.py | 64 ++++ apps/shared/loans_table.py | 72 ++++ apps/shared/protocol_initializers/__init__.py | 0 .../protocol_initializers/zklend.py} | 0 .../shared/protocol_loan_entities/__init__.py | 0 apps/shared/protocol_loan_entities/zklend.py | 102 ++++++ apps/shared/protocol_states/__init__.py | 0 apps/shared/protocol_states/values.py | 22 ++ apps/shared/protocol_states/zklend.py | 320 ++++++++++++++++++ apps/shared/state.py | 34 ++ apps/shared/types/__init__.py | 1 + apps/shared/types/base.py | 85 +++++ apps/shared/types/zklend.py | 5 + 16 files changed, 880 insertions(+), 2 deletions(-) create mode 100644 apps/dashboard_app/load_data.py create mode 100644 apps/shared/loans_table.py create mode 100644 apps/shared/protocol_initializers/__init__.py rename apps/{data_handler/handlers/loan_states/zklend/utils.py => shared/protocol_initializers/zklend.py} (100%) create mode 100644 apps/shared/protocol_loan_entities/__init__.py create mode 100644 apps/shared/protocol_loan_entities/zklend.py create mode 100644 apps/shared/protocol_states/__init__.py create mode 100644 apps/shared/protocol_states/values.py create mode 100644 apps/shared/protocol_states/zklend.py diff --git a/apps/dashboard_app/data_conector.py b/apps/dashboard_app/data_conector.py index 9c01496a..4f9e671f 100644 --- a/apps/dashboard_app/data_conector.py +++ b/apps/dashboard_app/data_conector.py @@ -4,12 +4,14 @@ import sqlalchemy from dotenv import load_dotenv +from shared.constants import ZKLEND + load_dotenv() class DataConnector: REQUIRED_VARS = ("DB_USER", "DB_PASSWORD", "DB_HOST", "DB_PORT", "DB_NAME") - SQL_QUERY = "SELECT * FROM %s WHERE protocol_id = 'zkLend'" + SQL_QUERY = f"SELECT * FROM %s WHERE protocol_id = '{ZKLEND}'" def __init__(self): """ diff --git a/apps/dashboard_app/load_data.py b/apps/dashboard_app/load_data.py new file mode 100644 index 00000000..14b0362d --- /dev/null +++ b/apps/dashboard_app/load_data.py @@ -0,0 +1,171 @@ +import asyncio +import itertools +import logging +import math +import time + +import pandas + + + +import src.loans_table +import src.main_chart +import src.persistent_state +import src.protocol_stats +import src.protocol_parameters +import src.settings +import src.swap_amm +import src.zklend + +from data_conector import DataConnector +from shared.constants import ZKLEND +from shared.protocol_states.zklend import ZkLendState +from shared.protocol_initializers.zklend import ZkLendInitializer +from helpers.tools import get_prices, GS_BUCKET_NAME + +def update_data(zklend_state: src.zklend.ZkLendState): + logging.info(f"Updating SQL data from {zklend_state.last_block_number}...") + time_end = time.time() + # TODO: parallelize per protocol + # TODO: stream the data, don't wait until we get all events + zklend_events = src.zklend.zklend_get_events( + start_block_number=zklend_state.last_block_number + 1 + ) + + logging.info( + f"got = {len(zklend_events)} events in {time.time() - time_end}s" + ) # TODO: this log will become obsolete + + # Iterate over ordered events to obtain the final state of each user. + t1 = time.time() + for _, zklend_event in zklend_events.iterrows(): + zklend_state.process_event(event=zklend_event) + + logging.info(f"updated state in {time.time() - t1}s") + + # TODO: move this to state inits above? + # # Collect token parameters. + t2 = time.time() + asyncio.run(zklend_state.collect_token_parameters()) + logging.info(f"collected token parameters in {time.time() - t2}s") + + # Get prices of the underlying tokens. + t_prices = time.time() + states = [ + zklend_state, + ] + underlying_addresses_to_decimals = {} + for state in states: + underlying_addresses_to_decimals.update( + { + x.underlying_address: x.decimals + for x in state.token_parameters.collateral.values() + } + ) + underlying_addresses_to_decimals.update( + { + x.underlying_address: x.decimals + for x in state.token_parameters.debt.values() + } + ) + underlying_addresses_to_decimals.update( + { + x.address: int(math.log10(x.decimal_factor)) + for x in src.settings.TOKEN_SETTINGS.values() + } + ) + prices = get_prices(token_decimals=underlying_addresses_to_decimals) + logging.info(f"prices in {time.time() - t_prices}s") + + t_swap = time.time() + swap_amms = src.swap_amm.SwapAmm() + asyncio.run(swap_amms.init()) + logging.info(f"swap in {time.time() - t_swap}s") + + t3 = time.time() + for pair, state in itertools.product(src.settings.PAIRS, states): + protocol = src.protocol_parameters.get_protocol(state=state) + logging.info( + f"Preparing main chart data for protocol = {protocol} and pair = {pair}." + ) + # TODO: Decipher `pair` in a smarter way. + collateral_token_underlying_symbol, debt_token_underlying_symbol = pair.split( + "-" + ) + _ = src.main_chart.get_main_chart_data( + state=state, + prices=prices, + swap_amms=swap_amms, + collateral_token_underlying_symbol=collateral_token_underlying_symbol, + debt_token_underlying_symbol=debt_token_underlying_symbol, + save_data=True, + ) + logging.info( + f"Main chart data for protocol = {protocol} and pair = {pair} prepared in {time.time() - t3}s" + ) + logging.info(f"updated graphs in {time.time() - t3}s") + + loan_stats = {} + for state in states: + protocol = src.protocol_parameters.get_protocol(state=state) + loan_stats[protocol] = src.loans_table.get_loans_table_data( + state=state, prices=prices, save_data=True + ) + + general_stats = src.protocol_stats.get_general_stats( + states=states, loan_stats=loan_stats, save_data=True + ) + supply_stats = src.protocol_stats.get_supply_stats( + states=states, + prices=prices, + save_data=True, + ) + _ = src.protocol_stats.get_collateral_stats(states=states, save_data=True) + debt_stats = src.protocol_stats.get_debt_stats(states=states, save_data=True) + _ = src.protocol_stats.get_utilization_stats( + general_stats=general_stats, + supply_stats=supply_stats, + debt_stats=debt_stats, + save_data=True, + ) + + max_block_number = zklend_events["block_number"].max() + max_timestamp = zklend_events["timestamp"].max() + last_update = { + "timestamp": str(max_timestamp), + "block_number": str(max_block_number), + } + src.persistent_state.upload_object_as_pickle( + last_update, path=src.persistent_state.LAST_UPDATE_FILENAME + ) + zklend_state.save_loan_entities( + path=src.persistent_state.PERSISTENT_STATE_LOAN_ENTITIES_FILENAME + ) + zklend_state.clear_loan_entities() + src.persistent_state.upload_object_as_pickle( + zklend_state, path=src.persistent_state.PERSISTENT_STATE_FILENAME + ) + loan_entities = pandas.read_parquet( + f"gs://{GS_BUCKET_NAME}/{src.persistent_state.PERSISTENT_STATE_LOAN_ENTITIES_FILENAME}", + engine="fastparquet", + ) + zklend_state.set_loan_entities(loan_entities=loan_entities) + logging.info(f"Updated CSV data in {time.time() - time_end}s") + + return zklend_state + + +if __name__ == "__name__": + # Fetching data from DB + connector = DataConnector() + loan_states_data_frame = connector.fetch_data("loan_state", ZKLEND) + + # Initializing ZkLend state + zklend_state = ZkLendState() + zklend_initializer = ZkLendInitializer(zklend_state) + user_ids = zklend_initializer.get_user_ids_from_df(loan_states_data_frame) + zklend_initializer.set_last_loan_states_per_users(user_ids) + + # Updating data + zklend_initializer.zklend_state = update_data(zklend_initializer.zklend_state) + print(zklend_initializer.zklend_state) diff --git a/apps/data_handler/handlers/loan_states/zklend/run.py b/apps/data_handler/handlers/loan_states/zklend/run.py index e687ee96..5cb43e42 100644 --- a/apps/data_handler/handlers/loan_states/zklend/run.py +++ b/apps/data_handler/handlers/loan_states/zklend/run.py @@ -6,7 +6,7 @@ from data_handler.handler_tools.constants import ProtocolAddresses from data_handler.handlers.loan_states.abstractions import LoanStateComputationBase from data_handler.handlers.loan_states.zklend.events import ZkLendState -from data_handler.handlers.loan_states.zklend.utils import ZkLendInitializer +from shared.protocol_initializers.zklend import ZkLendInitializer from shared.constants import ProtocolIDs from shared.state import State diff --git a/apps/shared/helpers.py b/apps/shared/helpers.py index 7eda8b62..0e738987 100644 --- a/apps/shared/helpers.py +++ b/apps/shared/helpers.py @@ -1,6 +1,70 @@ +import starknet_py + +from .blockchain_call import func_call +from .state import State +from shared.protocol_states.zklend import ZkLendState + def add_leading_zeros(hash: str) -> str: """ Converts e.g. `0x436d8d078de345c11493bd91512eae60cd2713e05bcaa0bb9f0cba90358c6e` to `0x00436d8d078de345c11493bd91512eae60cd2713e05bcaa0bb9f0cba90358c6e`. """ return "0x" + hash[2:].zfill(64) + + +async def get_symbol(token_address: str) -> str: + # DAI V2's symbol is `DAI` but we don't want to mix it with DAI = DAI V1. + if ( + token_address + == "0x05574eb6b8789a91466f902c380d978e472db68170ff82a5b650b95a58ddf4ad" + ): + return "DAI V2" + symbol = await func_call( + addr=token_address, + selector="symbol", + calldata=[], + ) + # For some Nostra Mainnet tokens, a list of length 3 is returned. + if len(symbol) > 1: + return starknet_py.cairo.felt.decode_shortstring(symbol[1]) + return starknet_py.cairo.felt.decode_shortstring(symbol[0]) + + +def get_protocol(state: State) -> str: + # TODO: Improve the inference. + if isinstance(state, ZkLendState): + return "zkLend" + + # We'll add it later + + # if isinstance(state, src.hashstack_v0.HashstackV0State): + # return "Hashstack V0" + # if isinstance(state, src.hashstack_v1.HashstackV1State): + # return "Hashstack V1" + # if isinstance(state, src.nostra_alpha.NostraAlphaState) and not isinstance( + # state, src.nostra_mainnet.NostraMainnetState + # ): + # return "Nostra Alpha" + # if isinstance(state, src.nostra_mainnet.NostraMainnetState): + # return "Nostra Mainnet" + raise ValueError + + +def get_directory(state: State) -> str: + # TODO: Improve the inference. + if isinstance(state, ZkLendState): + return "zklend_data" + + # We'll add it later + + # if isinstance(state, src.hashstack_v0.HashstackV0State): + # return "hashstack_v0_data" + # if isinstance(state, src.hashstack_v1.HashstackV1State): + # return "hashstack_v1_data" + # if isinstance(state, src.nostra_alpha.NostraAlphaState) and not isinstance( + # state, src.nostra_mainnet.NostraMainnetState + # ): + # return "nostra_alpha_data" + # if isinstance(state, src.nostra_mainnet.NostraMainnetState): + # return "nostra_mainnet_data" + raise ValueError diff --git a/apps/shared/loans_table.py b/apps/shared/loans_table.py new file mode 100644 index 00000000..09e70f29 --- /dev/null +++ b/apps/shared/loans_table.py @@ -0,0 +1,72 @@ +import pandas + +from shared.types import Prices +from shared.state import State + +from shared.helpers import get_protocol, get_directory + + +def get_loans_table_data( + state: State, + prices: Prices, + save_data: bool = False, +) -> pandas.DataFrame: + data = [] + for loan_entity_id, loan_entity in state.loan_entities.items(): + collateral_usd = loan_entity.compute_collateral_usd( + risk_adjusted=False, + collateral_token_parameters=state.token_parameters.collateral, + collateral_interest_rate_model=state.interest_rate_models.collateral, + prices=prices, + ) + risk_adjusted_collateral_usd = loan_entity.compute_collateral_usd( + risk_adjusted=True, + collateral_token_parameters=state.token_parameters.collateral, + collateral_interest_rate_model=state.interest_rate_models.collateral, + prices=prices, + ) + debt_usd = loan_entity.compute_debt_usd( + risk_adjusted=False, + debt_token_parameters=state.token_parameters.debt, + debt_interest_rate_model=state.interest_rate_models.debt, + prices=prices, + ) + + health_factor = loan_entity.compute_health_factor( + standardized=False, + risk_adjusted_collateral_usd=risk_adjusted_collateral_usd, + debt_usd=debt_usd, + ) + standardized_health_factor = loan_entity.compute_health_factor( + standardized=True, + risk_adjusted_collateral_usd=risk_adjusted_collateral_usd, + debt_usd=debt_usd, + ) + + data.append( + { + "User": ( + loan_entity_id + ), + "Protocol": get_protocol(state=state), + "Collateral (USD)": collateral_usd, + "Risk-adjusted collateral (USD)": risk_adjusted_collateral_usd, + "Debt (USD)": debt_usd, + "Health factor": health_factor, + "Standardized health factor": standardized_health_factor, + "Collateral": loan_entity.get_collateral_str( + collateral_token_parameters=state.token_parameters.collateral, + collateral_interest_rate_model=state.interest_rate_models.collateral, + ), + "Debt": loan_entity.get_debt_str( + debt_token_parameters=state.token_parameters.debt, + debt_interest_rate_model=state.interest_rate_models.debt, + ), + } + ) + data = pandas.DataFrame(data) + if save_data: + directory = get_directory(state=state) + path = f"{directory}/loans.parquet" + src.helpers.save_dataframe(data=data, path=path) + return data \ No newline at end of file diff --git a/apps/shared/protocol_initializers/__init__.py b/apps/shared/protocol_initializers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/data_handler/handlers/loan_states/zklend/utils.py b/apps/shared/protocol_initializers/zklend.py similarity index 100% rename from apps/data_handler/handlers/loan_states/zklend/utils.py rename to apps/shared/protocol_initializers/zklend.py diff --git a/apps/shared/protocol_loan_entities/__init__.py b/apps/shared/protocol_loan_entities/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/shared/protocol_loan_entities/zklend.py b/apps/shared/protocol_loan_entities/zklend.py new file mode 100644 index 00000000..39e11a5a --- /dev/null +++ b/apps/shared/protocol_loan_entities/zklend.py @@ -0,0 +1,102 @@ +from shared.types import ZkLendCollateralEnabled, Portfolio, InterestRateModels, TokenParameters, Prices, LoanEntity + +class ZkLendLoanEntity(LoanEntity): + """ + A class that describes the zkLend loan entity. On top of the abstract `LoanEntity`, it implements the `deposit` and + `collateral_enabled` attributes in order to help with accounting for the changes in collateral. This is because + under zkLend, collateral is the amount deposited that is specificaly flagged with `collateral_enabled` set to True + for the given token. To properly account for the changes in collateral, we must hold the information about the + given token's deposits being enabled as collateral or not and the amount of the deposits. We keep all balances in raw + amounts. + """ + + def __init__(self) -> None: + super().__init__() + self.deposit: Portfolio = Portfolio() + self.collateral_enabled: ZkLendCollateralEnabled = ZkLendCollateralEnabled() + + def compute_health_factor( + self, + standardized: bool, + collateral_token_parameters: TokenParameters | None = None, + collateral_interest_rate_model: InterestRateModels | None = None, + debt_token_parameters: TokenParameters | None = None, + debt_interest_rate_model: InterestRateModels | None = None, + prices: Prices | None = None, + risk_adjusted_collateral_usd: float | None = None, + debt_usd: float | None = None, + ) -> float: + if risk_adjusted_collateral_usd is None: + risk_adjusted_collateral_usd = self.compute_collateral_usd( + risk_adjusted=True, + collateral_token_parameters=collateral_token_parameters, + collateral_interest_rate_model=collateral_interest_rate_model, + prices=prices, + ) + if debt_usd is None: + debt_usd = self.compute_debt_usd( + risk_adjusted=False, + collateral_token_parameters=debt_token_parameters, + debt_interest_rate_model=debt_interest_rate_model, + prices=prices, + ) + + if standardized: + # Denominator is the value of (risk-adjusted) collateral at which the loan entity can be liquidated. + # TODO: denominator = debt_usd * liquidation_threshold?? + denominator = debt_usd + else: + denominator = debt_usd + + if denominator == 0.0: + # TODO: Assumes collateral is positive. + return float("inf") + return risk_adjusted_collateral_usd / denominator + + def compute_debt_to_be_liquidated( + self, + collateral_token_underlying_address: str, + debt_token_underlying_address: str, + prices: Prices, + collateral_token_parameters: TokenParameters, + collateral_interest_rate_model: InterestRateModels | None = None, + debt_token_parameters: TokenParameters | None = None, + debt_interest_rate_model: InterestRateModels | None = None, + risk_adjusted_collateral_usd: float | None = None, + debt_usd: float | None = None, + ) -> float: + if risk_adjusted_collateral_usd is None: + risk_adjusted_collateral_usd = self.compute_collateral_usd( + risk_adjusted=True, + collateral_token_parameters=collateral_token_parameters, + collateral_interest_rate_model=collateral_interest_rate_model, + prices=prices, + ) + if debt_usd is None: + debt_usd = self.compute_debt_usd( + risk_adjusted=False, + collateral_token_parameters=debt_token_parameters, + debt_interest_rate_model=debt_interest_rate_model, + prices=prices, + ) + + # TODO: Commit a PDF with the derivation of the formula? + numerator = debt_usd - risk_adjusted_collateral_usd + denominator = prices[debt_token_underlying_address] * ( + 1 + - collateral_token_parameters[ + collateral_token_underlying_address + ].collateral_factor + * ( + 1 + + collateral_token_parameters[ + collateral_token_underlying_address + ].liquidation_bonus + ) + ) + max_debt_to_be_liquidated = numerator / denominator + # The liquidator can't liquidate more debt than what is available. + debt_to_be_liquidated = min( + float(self.debt[debt_token_underlying_address]), max_debt_to_be_liquidated + ) + return debt_to_be_liquidated \ No newline at end of file diff --git a/apps/shared/protocol_states/__init__.py b/apps/shared/protocol_states/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/shared/protocol_states/values.py b/apps/shared/protocol_states/values.py new file mode 100644 index 00000000..5bb7726f --- /dev/null +++ b/apps/shared/protocol_states/values.py @@ -0,0 +1,22 @@ +ZKLEND_EVENTS_TO_METHODS: dict[str, str] = { + "AccumulatorsSync": "process_accumulators_sync_event", + "zklend::market::Market::AccumulatorsSync": "process_accumulators_sync_event", + "Deposit": "process_deposit_event", + "zklend::market::Market::Deposit": "process_deposit_event", + "CollateralEnabled": "process_collateral_enabled_event", + "zklend::market::Market::CollateralEnabled": "process_collateral_enabled_event", + "CollateralDisabled": "process_collateral_disabled_event", + "zklend::market::Market::CollateralDisabled": "process_collateral_disabled_event", + "Withdrawal": "process_withdrawal_event", + "zklend::market::Market::Withdrawal": "process_withdrawal_event", + "Borrowing": "process_borrowing_event", + "zklend::market::Market::Borrowing": "process_borrowing_event", + "Repayment": "process_repayment_event", + "zklend::market::Market::Repayment": "process_repayment_event", + "Liquidation": "process_liquidation_event", + "zklend::market::Market::Liquidation": "process_liquidation_event", +} + +ZKLEND_MARKET: str = ( + "0x04c0a5193d58f74fbace4b74dcf65481e734ed1714121bdc571da345540efa05" +) diff --git a/apps/shared/protocol_states/zklend.py b/apps/shared/protocol_states/zklend.py new file mode 100644 index 00000000..11e7ff99 --- /dev/null +++ b/apps/shared/protocol_states/zklend.py @@ -0,0 +1,320 @@ +import decimal +import copy +import logging +import pandas as pd + +from shared.state import State +from shared.protocol_loan_entities.zklend import ZkLendLoanEntity +from shared.helpers import add_leading_zeros, get_symbol +from shared.blockchain_call import func_call +from shared.types import Prices, ZkLendCollateralTokenParameters, ZkLendDebtTokenParameters + +from .values import ZKLEND_EVENTS_TO_METHODS, ZKLEND_MARKET + +class ZkLendState(State): + """ + A class that describes the state of all zkLend loan entities. It implements methods for correct processing of every + relevant event. + """ + + EVENTS_TO_METHODS: dict[str, str] = ZKLEND_EVENTS_TO_METHODS + + def __init__( + self, + verbose_user: str | None = None, + ) -> None: + super().__init__( + loan_entity_class=ZkLendLoanEntity, + verbose_user=verbose_user, + ) + + def process_accumulators_sync_event(self, event: pd.Series) -> None: + # The order of the values in the `data` column is: `token`, `lending_accumulator`, `debt_accumulator`. + # Example: https://starkscan.co/event/0x029628b89875a98c1c64ae206e7eb65669cb478a24449f3485f5e98aba6204dc_0. + token = add_leading_zeros(event["data"][0]) + collateral_interest_rate_index = decimal.Decimal( + str(int(event["data"][1], base=16)) + ) / decimal.Decimal("1e27") + debt_interest_rate_index = decimal.Decimal( + str(int(event["data"][2], base=16)) + ) / decimal.Decimal("1e27") + self.interest_rate_models.collateral[token] = collateral_interest_rate_index + self.interest_rate_models.debt[token] = debt_interest_rate_index + + def process_deposit_event(self, event: pd.Series) -> None: + # The order of the values in the `data` column is: `user`, `token`, `face_amount`. + # Example: https://starkscan.co/event/0x036185142bb51e2c1f5bfdb1e6cef81f8ea87fd4d777990014249bf5435fd31b_3. + user = add_leading_zeros(event["data"][0]) + token = add_leading_zeros(event["data"][1]) + face_amount = decimal.Decimal(str(int(event["data"][2], base=16))) + raw_amount = face_amount / self.interest_rate_models.collateral[token] + self.loan_entities[user].deposit.increase_value(token=token, value=raw_amount) + if self.loan_entities[user].collateral_enabled[token]: + self.loan_entities[user].collateral.increase_value( + token=token, value=raw_amount + ) + if user == self.verbose_user: + logging.info( + "In block number = {}, raw amount = {} of token = {} was deposited.".format( + event["block_number"], + raw_amount, + token, + ) + ) + + def process_collateral_enabled_event(self, event: pd.Series) -> None: + # The order of the values in the `data` column is: `user`, `token`. + # Example: https://starkscan.co/event/0x036185142bb51e2c1f5bfdb1e6cef81f8ea87fd4d777990014249bf5435fd31b_6. + user = add_leading_zeros(event["data"][0]) + token = add_leading_zeros(event["data"][1]) + self.loan_entities[user].collateral_enabled[token] = True + self.loan_entities[user].collateral.set_value( + token=token, + value=self.loan_entities[user].deposit[token], + ) + if user == self.verbose_user: + logging.info( + "In block number = {}, collateral was enabled for token = {}.".format( + event["block_number"], + token, + ) + ) + + def process_collateral_disabled_event(self, event: pd.Series) -> None: + # The order of the values in the `data` column is: `user`, `token`. + # Example: https://starkscan.co/event/0x0049b445bed84e0118795dbd22d76610ccac2ad626f8f04a1fc7e38113c2afe7_0. + user = add_leading_zeros(event["data"][0]) + token = add_leading_zeros(event["data"][1]) + self.loan_entities[user].collateral_enabled[token] = False + self.loan_entities[user].collateral.set_value( + token=token, value=decimal.Decimal("0") + ) + if user == self.verbose_user: + logging.info( + "In block number = {}, collateral was disabled for token = {}.".format( + event["block_number"], + token, + ) + ) + + def process_withdrawal_event(self, event: pd.Series) -> None: + # The order of the values in the `data` column is: `user`, `token`, `face_amount`. + # Example: https://starkscan.co/event/0x03472cf7511687a55bc7247f8765c4bbd2c18b70e09b2a10a77c61f567bfd2cb_4. + user = add_leading_zeros(event["data"][0]) + token = add_leading_zeros(event["data"][1]) + face_amount = decimal.Decimal(str(int(event["data"][2], base=16))) + raw_amount = face_amount / self.interest_rate_models.collateral[token] + self.loan_entities[user].deposit.increase_value(token=token, value=-raw_amount) + if self.loan_entities[user].collateral_enabled[token]: + self.loan_entities[user].collateral.increase_value( + token=token, value=-raw_amount + ) + if user == self.verbose_user: + logging.info( + "In block number = {}, raw amount = {} of token = {} was withdrawn.".format( + event["block_number"], + raw_amount, + token, + ) + ) + + def process_borrowing_event(self, event: pd.Series) -> None: + # The order of the values in the `data` column is: `user`, `token`, `raw_amount`, `face_amount`. + # Example: https://starkscan.co/event/0x076b1615750528635cf0b63ca80986b185acbd20fa37f0f2b5368a4f743931f8_3. + user = add_leading_zeros(event["data"][0]) + token = add_leading_zeros(event["data"][1]) + raw_amount = decimal.Decimal(str(int(event["data"][2], base=16))) + self.loan_entities[user].debt.increase_value(token=token, value=raw_amount) + if user == self.verbose_user: + logging.info( + "In block number = {}, raw amount = {} of token = {} was borrowed.".format( + event["block_number"], + raw_amount, + token, + ) + ) + + def process_repayment_event(self, event: pd.Series) -> None: + # The order of the values in the `data` column is: `repayer`, `beneficiary`, `token`, `raw_amount`, + # `face_amount`. + # Example: https://starkscan.co/event/0x06fa3dd6e12c9a66aeacd2eefa5a2ff2915dd1bb4207596de29bd0e8cdeeae66_5. + user = add_leading_zeros(event["data"][1]) + token = add_leading_zeros(event["data"][2]) + raw_amount = decimal.Decimal(str(int(event["data"][3], base=16))) + self.loan_entities[user].debt.increase_value(token=token, value=-raw_amount) + if user == self.verbose_user: + logging.info( + "In block number = {}, raw amount = {} of token = {} was repaid.".format( + event["block_number"], + raw_amount, + token, + ) + ) + + def process_liquidation_event(self, event: pd.Series) -> None: + # The order of the arguments is: `liquidator`, `user`, `debt_token`, `debt_raw_amount`, `debt_face_amount`, + # `collateral_token`, `collateral_amount`. + # Example: https://starkscan.co/event/0x07b8ec709df1066d9334d56b426c45440ca1f1bb841285a5d7b33f9d1008f256_5. + user = add_leading_zeros(event["data"][1]) + debt_token = add_leading_zeros(event["data"][2]) + debt_raw_amount = decimal.Decimal(str(int(event["data"][3], base=16))) + collateral_token = add_leading_zeros(event["data"][5]) + collateral_face_amount = decimal.Decimal(str(int(event["data"][6], base=16))) + collateral_raw_amount = ( + collateral_face_amount + / self.interest_rate_models.collateral[collateral_token] + ) + self.loan_entities[user].debt.increase_value( + token=debt_token, value=-debt_raw_amount + ) + self.loan_entities[user].deposit.increase_value( + token=collateral_token, value=-collateral_raw_amount + ) + if self.loan_entities[user].collateral_enabled[collateral_token]: + self.loan_entities[user].collateral.increase_value( + token=collateral_token, value=-collateral_raw_amount + ) + if user == self.verbose_user: + logging.info( + "In block number = {}, debt of raw amount = {} of token = {} and collateral of raw amount = {} of " + "token = {} were liquidated.".format( + event["block_number"], + debt_raw_amount, + debt_token, + collateral_raw_amount, + collateral_token, + ) + ) + + async def collect_token_parameters(self) -> None: + # Get the sets of unique collateral and debt tokens. + collateral_tokens = { + y for x in self.loan_entities.values() for y in x.collateral.keys() + } + debt_tokens = {y for x in self.loan_entities.values() for y in x.debt.keys()} + + # Get parameters for each collateral and debt token. Under zkLend, the collateral token in the events data is + # the underlying token directly. + for underlying_collateral_token_address in collateral_tokens: + underlying_collateral_token_symbol = await get_symbol( + token_address=underlying_collateral_token_address + ) + # The order of the arguments is: `enabled`, `decimals`, `z_token_address`, `interest_rate_model`, + # `collateral_factor`, `borrow_factor`, `reserve_factor`, `last_update_timestamp`, `lending_accumulator`, + # `debt_accumulator`, `current_lending_rate`, `current_borrowing_rate`, `raw_total_debt`, `flash_loan_fee`, + # `liquidation_bonus`, `debt_limit`. + reserve_data = await func_call( + addr=ZKLEND_MARKET, + selector="get_reserve_data", + calldata=[underlying_collateral_token_address], + ) + collateral_token_address = add_leading_zeros( + hex(reserve_data[2]) + ) + collateral_token_symbol = await get_symbol( + token_address=collateral_token_address + ) + self.token_parameters.collateral[ + underlying_collateral_token_address + ] = ZkLendCollateralTokenParameters( + address=collateral_token_address, + decimals=int(reserve_data[1]), + symbol=collateral_token_symbol, + underlying_symbol=underlying_collateral_token_symbol, + underlying_address=underlying_collateral_token_address, + collateral_factor=reserve_data[4] / 1e27, + liquidation_bonus=reserve_data[14] / 1e27, + ) + for underlying_debt_token_address in debt_tokens: + underlying_debt_token_symbol = await get_symbol( + token_address=underlying_debt_token_address + ) + # The order of the arguments is: `enabled`, `decimals`, `z_token_address`, `interest_rate_model`, + # `collateral_factor`, `borrow_factor`, `reserve_factor`, `last_update_timestamp`, `lending_accumulator`, + # `debt_accumulator`, `current_lending_rate`, `current_borrowing_rate`, `raw_total_debt`, `flash_loan_fee`, + # `liquidation_bonus`, `debt_limit`. + reserve_data = await func_call( + addr=ZKLEND_MARKET, + selector="get_reserve_data", + calldata=[underlying_debt_token_address], + ) + debt_token_address = add_leading_zeros(hex(reserve_data[2])) + debt_token_symbol = await get_symbol( + token_address=debt_token_address + ) + self.token_parameters.debt[ + underlying_debt_token_address + ] = ZkLendDebtTokenParameters( + address=debt_token_address, + decimals=int(reserve_data[1]), + symbol=debt_token_symbol, + underlying_symbol=underlying_debt_token_symbol, + underlying_address=underlying_debt_token_address, + debt_factor=reserve_data[5] / 1e27, + ) + + def compute_liquidable_debt_at_price( + self, + prices: Prices, + collateral_token_underlying_address: str, + collateral_token_price: float, + debt_token_underlying_address: str, + ) -> float: + changed_prices = copy.deepcopy(prices) + changed_prices[collateral_token_underlying_address] = collateral_token_price + max_liquidated_amount = 0.0 + for loan_entity in self.loan_entities.values(): + # Filter out entities where the collateral token of interest is deposited as collateral. + collateral_token_underlying_addresses = { + token # TODO: this assumes that `token` is the underlying address + for token, token_amount in loan_entity.collateral.items() + if token_amount > decimal.Decimal("0") + } + if ( + not collateral_token_underlying_address + in collateral_token_underlying_addresses + ): + continue + + # Filter out entities where the debt token of interest is borowed. + debt_token_underlying_addresses = { + token # TODO: this assumes that `token` is the underlying address + for token, token_amount in loan_entity.debt.items() + if token_amount > decimal.Decimal("0") + } + if not debt_token_underlying_address in debt_token_underlying_addresses: + continue + + # Filter out entities with health factor below 1. + risk_adjusted_collateral_usd = loan_entity.compute_collateral_usd( + risk_adjusted=True, + collateral_token_parameters=self.token_parameters.collateral, + collateral_interest_rate_model=self.interest_rate_models.collateral, + prices=changed_prices, + ) + debt_usd = loan_entity.compute_debt_usd( + risk_adjusted=False, + debt_token_parameters=self.token_parameters.debt, + debt_interest_rate_model=self.interest_rate_models.debt, + prices=changed_prices, + ) + health_factor = loan_entity.compute_health_factor( + standardized=False, + risk_adjusted_collateral_usd=risk_adjusted_collateral_usd, + debt_usd=debt_usd, + ) + # TODO: `health_factor` < 0 should not be possible if the data is right. Should we keep the filter? + if health_factor >= 1.0 or health_factor <= 0.0: + continue + + # Find out how much of the `debt_token` will be liquidated. We assume that the liquidator receives the + # collateral token of interest even though it might not be the most optimal choice for the liquidator. + max_liquidated_amount += loan_entity.compute_debt_to_be_liquidated( + debt_token_underlying_address=debt_token_underlying_address, + collateral_token_underlying_address=collateral_token_underlying_address, + prices=changed_prices, + collateral_token_parameters=self.token_parameters.collateral, + risk_adjusted_collateral_usd=risk_adjusted_collateral_usd, + debt_usd=debt_usd, + ) + return max_liquidated_amount \ No newline at end of file diff --git a/apps/shared/state.py b/apps/shared/state.py index 5a1bb6e2..848b54b5 100644 --- a/apps/shared/state.py +++ b/apps/shared/state.py @@ -43,6 +43,40 @@ def __init__( self.last_block_number: int = 0 self.last_interest_rate_block_number: int = 0 + def set_loan_entities(self, loan_entities: pandas.DataFrame) -> None: # MB we won't need it + # When we're processing the data for the first time, we call this method with empty `loan_entities`. In that + # case, there's nothing to be set. + if loan_entities.empty: + return + + # Clear `self.loan entities` in case they were not empty. + if len(self.loan_entities) > 0: + self.clear_loan_entities() + + # Fill up `self.loan_entities` with `loan_entities`. + for _, loan_entity in loan_entities.iterrows(): + user = loan_entity["user"] + for collateral_token, collateral_amount in json.loads( + loan_entity["collateral"].decode("utf-8") + ).items(): + if collateral_amount: + self.loan_entities[user].collateral[ + collateral_token + ] = decimal.Decimal(str(collateral_amount)) + for debt_token, debt_amount in json.loads( + loan_entity["debt"].decode("utf-8") + ).items(): + if debt_amount: + self.loan_entities[user].debt[debt_token] = decimal.Decimal( + str(debt_amount) + ) + logging.info( + "Set = {} non-zero loan entities out of the former = {} loan entities.".format( + len(self.loan_entities), + len(loan_entities), + ) + ) + def process_event(self, method_name: str, event: pd.Series) -> None: # TODO: Save the timestamp of each update? if event["block_number"] >= self.last_block_number: diff --git a/apps/shared/types/__init__.py b/apps/shared/types/__init__.py index d37c6fd6..e240d406 100644 --- a/apps/shared/types/__init__.py +++ b/apps/shared/types/__init__.py @@ -9,6 +9,7 @@ TokenParameters, TokenSettings, TokenValues, + LoanEntity ) from shared.types.nostra import ( NostraAlphaCollateralTokenParameters, diff --git a/apps/shared/types/base.py b/apps/shared/types/base.py index 9daabd3b..a8cb22b7 100644 --- a/apps/shared/types/base.py +++ b/apps/shared/types/base.py @@ -1,3 +1,4 @@ +from abc import ABC, abstractmethod from collections import defaultdict from dataclasses import dataclass from decimal import Decimal @@ -327,3 +328,87 @@ def __init__( self.values: dict[str, Decimal] = { token: init_value for token in TOKEN_SETTINGS } + +class LoanEntity(ABC): + """ + A class that describes and entity which can hold collateral, borrow debt and be liquidable. For example, on + Starknet, such an entity is the user in case of zkLend, Nostra Alpha and Nostra Mainnet, or an individual loan in + case od Hashstack V0 and Hashstack V1. + """ + + def __init__(self) -> None: + self.collateral: Portfolio = Portfolio() + self.debt: Portfolio = Portfolio() + + def compute_collateral_usd( + self, + risk_adjusted: bool, + collateral_token_parameters: TokenParameters, + collateral_interest_rate_model: InterestRateModels, + prices: Prices, + ) -> float: + return sum( + float(token_amount) + / (10 ** collateral_token_parameters[token].decimals) + * ( + collateral_token_parameters[token].collateral_factor + if risk_adjusted + else 1.0 + ) + * float(collateral_interest_rate_model[token]) + * prices[collateral_token_parameters[token].underlying_address] + for token, token_amount in self.collateral.items() + ) + + def compute_debt_usd( + self, + risk_adjusted: bool, + debt_token_parameters: TokenParameters, + debt_interest_rate_model: InterestRateModels, + prices: Prices, + ) -> float: + return sum( + float(token_amount) + / (10 ** debt_token_parameters[token].decimals) + / (debt_token_parameters[token].debt_factor if risk_adjusted else 1.0) + * float(debt_interest_rate_model[token]) + * prices[debt_token_parameters[token].underlying_address] + for token, token_amount in self.debt.items() + ) + + @abstractmethod + def compute_health_factor(self): + pass + + # TODO: rename to liquidable debt? + @abstractmethod + def compute_debt_to_be_liquidated(self): + pass + + def get_collateral_str( + self, + collateral_token_parameters: TokenParameters, + collateral_interest_rate_model: InterestRateModels, + ) -> str: + return ", ".join( + f"{token}: {round(token_amount / (10 ** collateral_token_parameters[token].decimals) * collateral_interest_rate_model[token], 4)}" + for token, token_amount in self.collateral.items() + if token_amount > Decimal("0") + ) + + def get_debt_str( + self, + debt_token_parameters: TokenParameters, + debt_interest_rate_model: InterestRateModels, + ) -> str: + return ", ".join( + f"{token}: {round(token_amount / (10 ** debt_token_parameters[token].decimals) * debt_interest_rate_model[token], 4)}" + for token, token_amount in self.debt.items() + if token_amount > Decimal("0") + ) + + def has_collateral(self) -> bool: + return any(token_amount for token_amount in self.collateral.values()) + + def has_debt(self) -> bool: + return any(token_amount for token_amount in self.debt.values()) diff --git a/apps/shared/types/zklend.py b/apps/shared/types/zklend.py index c0460cff..8ac524d0 100644 --- a/apps/shared/types/zklend.py +++ b/apps/shared/types/zklend.py @@ -20,3 +20,8 @@ class ZkLendCollateralTokenParameters(BaseTokenParameters): @dataclass class ZkLendDebtTokenParameters(BaseTokenParameters): debt_factor: float + + +@dataclass +class ZkLendDebtTokenParameters(BaseTokenParameters): + debt_factor: float