From b2a092f75e94f3634b20f676f7f48b8b1915bd6e Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Fri, 10 Mar 2023 17:29:59 +0000 Subject: [PATCH] feat(liquidator): attempt at graceful termination on api disconnect --- bots/bridge-tester/src/issue.ts | 2 +- bots/bridge-tester/src/utils.ts | 9 -- .../test/_initialization/initialize.test.ts | 21 ++-- bots/lending-liquidator/src/liquidate.ts | 114 ++++++++++-------- bots/lending-liquidator/src/utils.ts | 45 +++++++ .../test/integration/liquidate.test.ts | 9 +- bots/lending-liquidator/test/utils.ts | 5 +- 7 files changed, 132 insertions(+), 73 deletions(-) create mode 100644 bots/lending-liquidator/src/utils.ts diff --git a/bots/bridge-tester/src/issue.ts b/bots/bridge-tester/src/issue.ts index 28cdc15..f893a43 100644 --- a/bots/bridge-tester/src/issue.ts +++ b/bots/bridge-tester/src/issue.ts @@ -15,7 +15,7 @@ import _ from "underscore"; import { LOAD_TEST_ISSUE_AMOUNT } from "./consts"; import logger from "./logger"; -import { sleep, waitForEmptyMempool } from "./utils"; +import { sleep } from "./utils"; export class Issue { interBtcApi: InterBtcApi; diff --git a/bots/bridge-tester/src/utils.ts b/bots/bridge-tester/src/utils.ts index 3ab6de9..ebcb9be 100644 --- a/bots/bridge-tester/src/utils.ts +++ b/bots/bridge-tester/src/utils.ts @@ -1,12 +1,3 @@ -import { BitcoinCoreClient } from "@interlay/interbtc-api"; - -export async function waitForEmptyMempool( - bitcoinCoreClient: BitcoinCoreClient -): Promise { - while ((await bitcoinCoreClient.getMempoolInfo()).size === 0) { - await sleep(1000); - } -} export async function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); diff --git a/bots/bridge-tester/test/_initialization/initialize.test.ts b/bots/bridge-tester/test/_initialization/initialize.test.ts index 821e5d8..8944021 100644 --- a/bots/bridge-tester/test/_initialization/initialize.test.ts +++ b/bots/bridge-tester/test/_initialization/initialize.test.ts @@ -4,7 +4,6 @@ import { assert } from "chai"; import { BitcoinCoreClient, - initializeStableConfirmations, issueSingle, InterBtcApi, DefaultInterBtcApi, @@ -14,6 +13,7 @@ import { InterbtcPrimitivesVaultId, newVaultId, CollateralCurrencyExt, + setRawStorage, } from "@interlay/interbtc-api"; import { DEFAULT_PARACHAIN_ENDPOINT, @@ -87,15 +87,20 @@ describe.skip("Initialize parachain state", () => { ]); if (stableBitcoinConfirmations != 0 || stableParachainConfirmations != 0) { - await initializeStableConfirmations( + console.log("Initializing stable block confirmations..."); + await setRawStorage( api, - { - bitcoinConfirmations: stableBitcoinConfirmationsToSet, - parachainConfirmations: stableParachainConfirmationsToSet, - }, - sudoAccount, - bitcoinCoreClient + api.query.btcRelay.stableBitcoinConfirmations.key(), + api.createType("u32", stableBitcoinConfirmationsToSet), + sudoAccount ); + await setRawStorage( + api, + api.query.btcRelay.stableParachainConfirmations.key(), + api.createType("u32", stableParachainConfirmationsToSet), + sudoAccount + ); + await bitcoinCoreClient.mineBlocks(3); [stableBitcoinConfirmations, stableParachainConfirmations] = await Promise.all([ userInterBtcApi.btcRelay.getStableBitcoinConfirmations(), userInterBtcApi.btcRelay.getStableParachainConfirmations(), diff --git a/bots/lending-liquidator/src/liquidate.ts b/bots/lending-liquidator/src/liquidate.ts index 672f8ee..d7e2767 100644 --- a/bots/lending-liquidator/src/liquidate.ts +++ b/bots/lending-liquidator/src/liquidate.ts @@ -2,6 +2,7 @@ import { ChainBalance, CollateralPosition, CurrencyExt, InterBtcApi, LoansMarket import { Currency, ExchangeRate, MonetaryAmount } from "@interlay/monetary-js"; import { AccountId } from "@polkadot/types/interfaces"; import { APPROX_BLOCK_TIME_MS } from "./consts"; +import { waitForEvent } from "./utils"; type CollateralAndValue = { collateral: MonetaryAmount, @@ -78,54 +79,71 @@ function liquidationStrategy( }) }); return result; -} - -export async function startLiquidator(interBtcApi: InterBtcApi): Promise { - console.log("Starting lending liquidator..."); - const foreignAssets = await interBtcApi.assetRegistry.getForeignAssets(); - - let nativeCurrency = [interBtcApi.getWrappedCurrency(), interBtcApi.getGovernanceCurrency(), interBtcApi.getRelayChainCurrency()] - let chainAssets = new Set([...nativeCurrency, ...foreignAssets]); - if (interBtcApi.account == undefined) { - return Promise.reject("No account set for the lending-liquidator"); } - const accountId = addressOrPairAsAccountId(interBtcApi.api, interBtcApi.account); - console.log("Listening to new blocks..."); - await interBtcApi.api.rpc.chain.subscribeNewHeads(async (header) => { - console.log(`Scanning block: #${header.number}`); - const liquidatorBalance: Map = new Map(); - const oracleRates: Map> = new Map(); - try { - const [_balancesPromise, _oraclePromise, undercollateralizedBorrowers, marketsArray, foreignAssets] = await Promise.all([ - Promise.all([...chainAssets].map((asset) => interBtcApi.tokens.balance(asset, accountId).then((balance) => liquidatorBalance.set(asset, balance)))), - Promise.all([...chainAssets].map((asset) => interBtcApi.oracle.getExchangeRate(asset).then((rate) => oracleRates.set(asset, rate)))), - interBtcApi.loans.getUndercollateralizedBorrowers(), - interBtcApi.loans.getLoansMarkets(), - interBtcApi.assetRegistry.getForeignAssets() - ]); - - console.log(`undercollateralized borrowers: ${undercollateralizedBorrowers.length}`); - const potentialLiquidation = liquidationStrategy( - interBtcApi, - liquidatorBalance, - oracleRates, - undercollateralizedBorrowers, - new Map(marketsArray) - ) as [MonetaryAmount, CurrencyExt, AccountId]; - if (potentialLiquidation) { - const [amountToRepay, collateralToLiquidate, borrower] = potentialLiquidation; - console.log(`Liquidating ${borrower.toString()} with ${amountToRepay.toHuman()} ${amountToRepay.currency.ticker}, collateral: ${collateralToLiquidate.ticker}`); - // Either our liquidation will go through, or someone else's will - await Promise.all([ - DefaultTransactionAPI.waitForEvent(interBtcApi.api, interBtcApi.api.events.loans.ActivatedMarket, 10 * APPROX_BLOCK_TIME_MS), - interBtcApi.loans.liquidateBorrowPosition(borrower, amountToRepay.currency, amountToRepay, collateralToLiquidate) - ]); - } - - // Add any new foreign assets to `chainAssets` - chainAssets = new Set([...Array.from(chainAssets), ...foreignAssets]); - } catch (error) { - console.log("found an error: ", error); + + export async function startLiquidator(interBtcApi: InterBtcApi): Promise { + console.log("Starting lending liquidator..."); + const foreignAssets = await interBtcApi.assetRegistry.getForeignAssets(); + + let nativeCurrency = [interBtcApi.getWrappedCurrency(), interBtcApi.getGovernanceCurrency(), interBtcApi.getRelayChainCurrency()] + let chainAssets = new Set([...nativeCurrency, ...foreignAssets]); + if (interBtcApi.account == undefined) { + return Promise.reject("No account set for the lending-liquidator"); } - }); + const accountId = addressOrPairAsAccountId(interBtcApi.api, interBtcApi.account); + console.log("Listening to new blocks..."); + let flagPromiseResolve: () => void; + const flagPromise = new Promise((resolve) => flagPromiseResolve = () => { resolve(); }) + // The block subscription is a Promise that never resolves + const subscriptionPromise = new Promise(() => { + interBtcApi.api.rpc.chain.subscribeNewHeads(async (header) => { + console.log(`Scanning block: #${header.number}`); + const liquidatorBalance: Map = new Map(); + const oracleRates: Map> = new Map(); + try { + const [_balancesPromise, _oraclePromise, undercollateralizedBorrowers, marketsArray, foreignAssets] = await Promise.all([ + Promise.all([...chainAssets].map((asset) => interBtcApi.tokens.balance(asset, accountId).then((balance) => liquidatorBalance.set(asset, balance)))), + Promise.all([...chainAssets].map((asset) => interBtcApi.oracle.getExchangeRate(asset).then((rate) => oracleRates.set(asset, rate)))), + interBtcApi.loans.getUndercollateralizedBorrowers(), + interBtcApi.loans.getLoansMarkets(), + interBtcApi.assetRegistry.getForeignAssets() + ]); + + console.log(`undercollateralized borrowers: ${undercollateralizedBorrowers.length}`); + const potentialLiquidation = liquidationStrategy( + interBtcApi, + liquidatorBalance, + oracleRates, + undercollateralizedBorrowers, + new Map(marketsArray) + ) as [MonetaryAmount, CurrencyExt, AccountId]; + if (potentialLiquidation) { + const [amountToRepay, collateralToLiquidate, borrower] = potentialLiquidation; + console.log(`Liquidating ${borrower.toString()} with ${amountToRepay.toHuman()} ${amountToRepay.currency.ticker}, collateral: ${collateralToLiquidate.ticker}`); + // Either our liquidation will go through, or someone else's will + await Promise.all([ + waitForEvent(interBtcApi.api, interBtcApi.api.events.loans.ActivatedMarket, 10 * APPROX_BLOCK_TIME_MS), + interBtcApi.loans.liquidateBorrowPosition(borrower, amountToRepay.currency, amountToRepay, collateralToLiquidate) + ]); + } + + // Add any new foreign assets to `chainAssets` + chainAssets = new Set([...Array.from(chainAssets), ...foreignAssets]); + } catch (error) { + if (interBtcApi.api.isConnected) { + console.log(error); + } else { + flagPromiseResolve(); + } + } + }); + }); + await Promise.race([ + flagPromise, + subscriptionPromise + ]); + // TODO: investigate why the process doesn't gracefully terminate here even though `Promise.race` finished + // Likely because of a `setTimeout` created by polkadot-js + // Kill the process for now + process.exit(0); } diff --git a/bots/lending-liquidator/src/utils.ts b/bots/lending-liquidator/src/utils.ts new file mode 100644 index 0000000..391f300 --- /dev/null +++ b/bots/lending-liquidator/src/utils.ts @@ -0,0 +1,45 @@ +import type { AnyTuple } from "@polkadot/types/types"; +import { ApiPromise } from "@polkadot/api"; +import { AugmentedEvent, ApiTypes } from "@polkadot/api/types"; +import { EventRecord } from "@polkadot/types/interfaces/system"; + +export async function waitForEvent( + api: ApiPromise, + event: AugmentedEvent, + timeoutMs: number +): Promise { + // Use this function with a timeout. + // Unless the awaited event occurs, this Promise will never resolve. + let timeoutHandle: NodeJS.Timeout; + const timeoutPromise = new Promise((_, reject) => { + timeoutHandle = setTimeout(() => reject(), timeoutMs); + }); + + await Promise.race([ + new Promise((resolve, _reject) => { + api.query.system.events((eventsVec) => { + const events = eventsVec.toArray(); + if (doesArrayContainEvent(events, event)) { + resolve(); + } + }); + }), + timeoutPromise, + ]).then((_) => { + clearTimeout(timeoutHandle); + }); + + return true; +} + +function doesArrayContainEvent( + events: EventRecord[], + eventType: AugmentedEvent +): boolean { + for (const { event } of events) { + if (eventType.is(event)) { + return true; + } + } + return false; +} \ No newline at end of file diff --git a/bots/lending-liquidator/test/integration/liquidate.test.ts b/bots/lending-liquidator/test/integration/liquidate.test.ts index dae9458..f8eeb77 100644 --- a/bots/lending-liquidator/test/integration/liquidate.test.ts +++ b/bots/lending-liquidator/test/integration/liquidate.test.ts @@ -8,6 +8,7 @@ import { DEFAULT_PARACHAIN_ENDPOINT, DEFAULT_SUDO_URI, DEFAULT_USER_1_URI, DEFAU import { setExchangeRate, waitRegisteredLendingMarkets } from "../utils"; import { startLiquidator } from "../../src"; import { APPROX_BLOCK_TIME_MS } from "../../src/consts"; +import { waitForEvent } from "../../src/utils"; describe("liquidate", () => { const approx10Blocks = 10 * APPROX_BLOCK_TIME_MS; @@ -104,7 +105,7 @@ describe("liquidate", () => { ]); const [eventFound] = await Promise.all([ - DefaultTransactionAPI.waitForEvent(sudoInterBtcAPI.api, sudoInterBtcAPI.api.events.loans.ActivatedMarket, approx10Blocks), + waitForEvent(sudoInterBtcAPI.api, sudoInterBtcAPI.api.events.loans.ActivatedMarket, approx10Blocks), api.tx.sudo.sudo(addMarkets).signAndSend(sudoAccount), ]); expect( @@ -115,10 +116,10 @@ describe("liquidate", () => { }); after(async () => { - api.disconnect(); + await api.disconnect(); }); - it("should lend expected amount of currency to protocol", async function () { + it("should liquidate undercollateralized borrower", async function () { this.timeout(20 * approx10Blocks); const depositAmount = newMonetaryAmount(1000, underlyingCurrency1, true); @@ -146,7 +147,7 @@ describe("liquidate", () => { // Do not `await` so it runs in the background startLiquidator(sudoInterBtcAPI); - const liquidationEventFoundPromise = DefaultTransactionAPI.waitForEvent(sudoInterBtcAPI.api, sudoInterBtcAPI.api.events.loans.LiquidatedBorrow, approx10Blocks); + const liquidationEventFoundPromise = waitForEvent(sudoInterBtcAPI.api, sudoInterBtcAPI.api.events.loans.LiquidatedBorrow, approx10Blocks); // crash the collateral exchange rate const newExchangeRate = "0x00000000000000000000100000000000"; diff --git a/bots/lending-liquidator/test/utils.ts b/bots/lending-liquidator/test/utils.ts index c070a90..8747fff 100644 --- a/bots/lending-liquidator/test/utils.ts +++ b/bots/lending-liquidator/test/utils.ts @@ -1,5 +1,5 @@ -import { CurrencyExt, InterBtcApi, storageKeyToNthInner, getStorageMapItemKey, createExchangeRateOracleKey, setStorageAtKey, sleep, SLEEP_TIME_MS } from "@interlay/interbtc-api"; +import { CurrencyExt, InterBtcApi, storageKeyToNthInner, createExchangeRateOracleKey, setStorageAtKey, sleep, SLEEP_TIME_MS } from "@interlay/interbtc-api"; import { ApiPromise } from "@polkadot/api"; @@ -22,8 +22,7 @@ export async function setExchangeRate( // Change Exchange rate storage for currency. const exchangeRateOracleKey = createExchangeRateOracleKey(api, currency); - - const exchangeRateStorageKey = getStorageMapItemKey("Oracle", "Aggregate", exchangeRateOracleKey.toHex()); + const exchangeRateStorageKey = sudoInterBtcAPI.api.query.oracle.aggregate.key(exchangeRateOracleKey); await setStorageAtKey(sudoInterBtcAPI.api, exchangeRateStorageKey, newExchangeRateHex, sudoAccount); }