feat(liquidator): attempt at graceful termination on api disconnect

bots/bridge-tester/src/issue.ts

@@ -15,7 +15,7 @@ import _ from "underscore";
 
 import { LOAD_TEST_ISSUE_AMOUNT } from "./consts";
 import logger from "./logger";
-import { sleep, waitForEmptyMempool } from "./utils";
+import { sleep } from "./utils";
 
 export class Issue {
   interBtcApi: InterBtcApi;

bots/bridge-tester/src/utils.ts

@@ -1,12 +1,3 @@
-import { BitcoinCoreClient } from "@interlay/interbtc-api";
-
-export async function waitForEmptyMempool(
-  bitcoinCoreClient: BitcoinCoreClient
-): Promise<void> {
-  while ((await bitcoinCoreClient.getMempoolInfo()).size === 0) {
-    await sleep(1000);
-  }
-}
 
 export async function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));

bots/bridge-tester/test/_initialization/initialize.test.ts

@@ -4,7 +4,6 @@ import { assert } from "chai";
 
 import {
   BitcoinCoreClient,
-  initializeStableConfirmations,
   issueSingle,
   InterBtcApi,
   DefaultInterBtcApi,
@@ -14,6 +13,7 @@ import {
   InterbtcPrimitivesVaultId,
   newVaultId,
   CollateralCurrencyExt,
+  setRawStorage,
 } from "@interlay/interbtc-api";
 import {
   DEFAULT_PARACHAIN_ENDPOINT,
@@ -87,15 +87,20 @@ describe.skip("Initialize parachain state", () => {
         ]);
 
         if (stableBitcoinConfirmations != 0 || stableParachainConfirmations != 0) {
-            await initializeStableConfirmations(
+            console.log("Initializing stable block confirmations...");
+            await setRawStorage(
                 api,
-                {
-                    bitcoinConfirmations: stableBitcoinConfirmationsToSet,
-                    parachainConfirmations: stableParachainConfirmationsToSet,
-                },
-                sudoAccount,
-                bitcoinCoreClient
+                api.query.btcRelay.stableBitcoinConfirmations.key(),
+                api.createType("u32", stableBitcoinConfirmationsToSet),
+                sudoAccount
             );
+            await setRawStorage(
+                api,
+                api.query.btcRelay.stableParachainConfirmations.key(),
+                api.createType("u32", stableParachainConfirmationsToSet),
+                sudoAccount
+            );
+            await bitcoinCoreClient.mineBlocks(3);
             [stableBitcoinConfirmations, stableParachainConfirmations] = await Promise.all([
                 userInterBtcApi.btcRelay.getStableBitcoinConfirmations(),
                 userInterBtcApi.btcRelay.getStableParachainConfirmations(),

bots/lending-liquidator/src/liquidate.ts

@@ -2,6 +2,7 @@ import { ChainBalance, CollateralPosition, CurrencyExt, InterBtcApi, LoansMarket
 import { Currency, ExchangeRate, MonetaryAmount } from "@interlay/monetary-js";
 import { AccountId } from "@polkadot/types/interfaces";
 import { APPROX_BLOCK_TIME_MS } from "./consts";
+import { waitForEvent } from "./utils";
 
 type CollateralAndValue = {
     collateral: MonetaryAmount<Currency>,
@@ -78,54 +79,71 @@ function liquidationStrategy(
             })
         });
         return result;
-}
-
-export async function startLiquidator(interBtcApi: InterBtcApi): Promise<void> {
-    console.log("Starting lending liquidator...");
-    const foreignAssets = await interBtcApi.assetRegistry.getForeignAssets();
-
-    let nativeCurrency = [interBtcApi.getWrappedCurrency(), interBtcApi.getGovernanceCurrency(), interBtcApi.getRelayChainCurrency()]
-    let chainAssets = new Set([...nativeCurrency, ...foreignAssets]);
-    if (interBtcApi.account == undefined) {
-        return Promise.reject("No account set for the lending-liquidator");
     }
-    const accountId = addressOrPairAsAccountId(interBtcApi.api, interBtcApi.account);
-    console.log("Listening to new blocks...");
-    await interBtcApi.api.rpc.chain.subscribeNewHeads(async (header) => {
-        console.log(`Scanning block: #${header.number}`);
-        const liquidatorBalance: Map<Currency, ChainBalance> = new Map();
-        const oracleRates: Map<Currency, ExchangeRate<Currency, CurrencyExt>> = new Map();
-        try {
-            const [_balancesPromise, _oraclePromise, undercollateralizedBorrowers, marketsArray, foreignAssets] = await Promise.all([
-                Promise.all([...chainAssets].map((asset) => interBtcApi.tokens.balance(asset, accountId).then((balance) => liquidatorBalance.set(asset, balance)))),
-                Promise.all([...chainAssets].map((asset) => interBtcApi.oracle.getExchangeRate(asset).then((rate) => oracleRates.set(asset, rate)))),
-                interBtcApi.loans.getUndercollateralizedBorrowers(),
-                interBtcApi.loans.getLoansMarkets(),
-                interBtcApi.assetRegistry.getForeignAssets()
-            ]);
-
-            console.log(`undercollateralized borrowers: ${undercollateralizedBorrowers.length}`);
-            const potentialLiquidation = liquidationStrategy(
-                interBtcApi,
-                liquidatorBalance,
-                oracleRates,
-                undercollateralizedBorrowers,
-                new Map(marketsArray)
-            ) as [MonetaryAmount<CurrencyExt>, CurrencyExt, AccountId];
-            if (potentialLiquidation) {
-                const [amountToRepay, collateralToLiquidate, borrower] = potentialLiquidation;
-                console.log(`Liquidating ${borrower.toString()} with ${amountToRepay.toHuman()} ${amountToRepay.currency.ticker}, collateral: ${collateralToLiquidate.ticker}`);
-                // Either our liquidation will go through, or someone else's will
-                await Promise.all([
-                    DefaultTransactionAPI.waitForEvent(interBtcApi.api, interBtcApi.api.events.loans.ActivatedMarket, 10 * APPROX_BLOCK_TIME_MS),
-                    interBtcApi.loans.liquidateBorrowPosition(borrower, amountToRepay.currency, amountToRepay, collateralToLiquidate)
-                ]);
-            }
-
-            // Add any new foreign assets to `chainAssets`
-            chainAssets = new Set([...Array.from(chainAssets), ...foreignAssets]);
-        } catch (error) {
-            console.log("found an error: ", error);   
+
+    export async function startLiquidator(interBtcApi: InterBtcApi): Promise<void> {
+        console.log("Starting lending liquidator...");
+        const foreignAssets = await interBtcApi.assetRegistry.getForeignAssets();
+
+        let nativeCurrency = [interBtcApi.getWrappedCurrency(), interBtcApi.getGovernanceCurrency(), interBtcApi.getRelayChainCurrency()]
+        let chainAssets = new Set([...nativeCurrency, ...foreignAssets]);
+        if (interBtcApi.account == undefined) {
+            return Promise.reject("No account set for the lending-liquidator");
         }
-    });
+        const accountId = addressOrPairAsAccountId(interBtcApi.api, interBtcApi.account);
+        console.log("Listening to new blocks...");
+        let flagPromiseResolve: () => void;
+        const flagPromise = new Promise<void>((resolve) => flagPromiseResolve = () => { resolve(); })
+        // The block subscription is a Promise that never resolves
+        const subscriptionPromise = new Promise(() => {
+            interBtcApi.api.rpc.chain.subscribeNewHeads(async (header) => {
+                console.log(`Scanning block: #${header.number}`);
+                const liquidatorBalance: Map<Currency, ChainBalance> = new Map();
+                const oracleRates: Map<Currency, ExchangeRate<Currency, CurrencyExt>> = new Map();
+                try {
+                    const [_balancesPromise, _oraclePromise, undercollateralizedBorrowers, marketsArray, foreignAssets] = await Promise.all([
+                        Promise.all([...chainAssets].map((asset) => interBtcApi.tokens.balance(asset, accountId).then((balance) => liquidatorBalance.set(asset, balance)))),
+                        Promise.all([...chainAssets].map((asset) => interBtcApi.oracle.getExchangeRate(asset).then((rate) => oracleRates.set(asset, rate)))),
+                        interBtcApi.loans.getUndercollateralizedBorrowers(),
+                        interBtcApi.loans.getLoansMarkets(),
+                        interBtcApi.assetRegistry.getForeignAssets()
+                    ]);
+
+                    console.log(`undercollateralized borrowers: ${undercollateralizedBorrowers.length}`);
+                    const potentialLiquidation = liquidationStrategy(
+                        interBtcApi,
+                        liquidatorBalance,
+                        oracleRates,
+                        undercollateralizedBorrowers,
+                        new Map(marketsArray)
+                        ) as [MonetaryAmount<CurrencyExt>, CurrencyExt, AccountId];
+                        if (potentialLiquidation) {
+                            const [amountToRepay, collateralToLiquidate, borrower] = potentialLiquidation;
+                            console.log(`Liquidating ${borrower.toString()} with ${amountToRepay.toHuman()} ${amountToRepay.currency.ticker}, collateral: ${collateralToLiquidate.ticker}`);
+                            // Either our liquidation will go through, or someone else's will
+                            await Promise.all([
+                                waitForEvent(interBtcApi.api, interBtcApi.api.events.loans.ActivatedMarket, 10 * APPROX_BLOCK_TIME_MS),
+                                interBtcApi.loans.liquidateBorrowPosition(borrower, amountToRepay.currency, amountToRepay, collateralToLiquidate)
+                            ]);
+                        }
+
+                    // Add any new foreign assets to `chainAssets`
+                    chainAssets = new Set([...Array.from(chainAssets), ...foreignAssets]);
+                } catch (error) {
+                    if (interBtcApi.api.isConnected) {
+                        console.log(error);
+                    } else {
+                        flagPromiseResolve();
+                    }
+                }
+            });
+        });
+        await Promise.race([
+            flagPromise,
+            subscriptionPromise
+        ]);
+        // TODO: investigate why the process doesn't gracefully terminate here even though `Promise.race` finished
+        // Likely because of a `setTimeout` created by polkadot-js
+        // Kill the process for now
+        process.exit(0);
 }

bots/lending-liquidator/src/utils.ts

@@ -0,0 +1,45 @@
+import type { AnyTuple } from "@polkadot/types/types";
+import { ApiPromise } from "@polkadot/api";
+import { AugmentedEvent, ApiTypes } from "@polkadot/api/types";
+import { EventRecord } from "@polkadot/types/interfaces/system";
+
+export async function waitForEvent<T extends AnyTuple>(
+    api: ApiPromise,
+    event: AugmentedEvent<ApiTypes, T>,
+    timeoutMs: number
+): Promise<boolean> {
+    // Use this function with a timeout.
+    // Unless the awaited event occurs, this Promise will never resolve.
+    let timeoutHandle: NodeJS.Timeout;
+    const timeoutPromise = new Promise((_, reject) => {
+        timeoutHandle = setTimeout(() => reject(), timeoutMs);
+    });
+
+    await Promise.race([
+        new Promise<void>((resolve, _reject) => {
+            api.query.system.events((eventsVec) => {
+                const events = eventsVec.toArray();
+                if (doesArrayContainEvent(events, event)) {
+                    resolve();
+                }
+            });
+        }),
+        timeoutPromise,
+    ]).then((_) => {
+        clearTimeout(timeoutHandle);
+    });
+
+    return true;
+}
+
+function doesArrayContainEvent<T extends AnyTuple>(
+    events: EventRecord[],
+    eventType: AugmentedEvent<ApiTypes, T>
+): boolean {
+    for (const { event } of events) {
+        if (eventType.is(event)) {
+            return true;
+        }
+    }
+    return false;
+}

bots/lending-liquidator/test/integration/liquidate.test.ts

@@ -8,6 +8,7 @@ import { DEFAULT_PARACHAIN_ENDPOINT, DEFAULT_SUDO_URI, DEFAULT_USER_1_URI, DEFAU
 import { setExchangeRate, waitRegisteredLendingMarkets } from "../utils";
 import { startLiquidator } from "../../src";
 import { APPROX_BLOCK_TIME_MS } from "../../src/consts";
+import { waitForEvent } from "../../src/utils";
 
 describe("liquidate", () => {
     const approx10Blocks = 10 * APPROX_BLOCK_TIME_MS;
@@ -104,7 +105,7 @@ describe("liquidate", () => {
         ]);
 
         const [eventFound] = await Promise.all([
-            DefaultTransactionAPI.waitForEvent(sudoInterBtcAPI.api, sudoInterBtcAPI.api.events.loans.ActivatedMarket, approx10Blocks),
+            waitForEvent(sudoInterBtcAPI.api, sudoInterBtcAPI.api.events.loans.ActivatedMarket, approx10Blocks),
             api.tx.sudo.sudo(addMarkets).signAndSend(sudoAccount),
         ]);
         expect(
@@ -115,10 +116,10 @@ describe("liquidate", () => {
     });
 
     after(async () => {
-        api.disconnect();
+        await api.disconnect();
     });
 
-    it("should lend expected amount of currency to protocol", async function () {
+    it("should liquidate undercollateralized borrower", async function () {
         this.timeout(20 * approx10Blocks);
 
         const depositAmount = newMonetaryAmount(1000, underlyingCurrency1, true);
@@ -146,7 +147,7 @@ describe("liquidate", () => {
         // Do not `await` so it runs in the background
         startLiquidator(sudoInterBtcAPI);
 
-        const liquidationEventFoundPromise = DefaultTransactionAPI.waitForEvent(sudoInterBtcAPI.api, sudoInterBtcAPI.api.events.loans.LiquidatedBorrow, approx10Blocks);
+        const liquidationEventFoundPromise = waitForEvent(sudoInterBtcAPI.api, sudoInterBtcAPI.api.events.loans.LiquidatedBorrow, approx10Blocks);
 
         // crash the collateral exchange rate
         const newExchangeRate = "0x00000000000000000000100000000000";

bots/lending-liquidator/test/utils.ts

@@ -1,5 +1,5 @@
 
-import { CurrencyExt, InterBtcApi, storageKeyToNthInner, getStorageMapItemKey, createExchangeRateOracleKey, setStorageAtKey, sleep, SLEEP_TIME_MS } from "@interlay/interbtc-api";
+import { CurrencyExt, InterBtcApi, storageKeyToNthInner, createExchangeRateOracleKey, setStorageAtKey, sleep, SLEEP_TIME_MS } from "@interlay/interbtc-api";
 
 import { ApiPromise } from "@polkadot/api";
 
@@ -22,8 +22,7 @@ export async function setExchangeRate(
 
     // Change Exchange rate storage for currency.
     const exchangeRateOracleKey = createExchangeRateOracleKey(api, currency);
-
-    const exchangeRateStorageKey = getStorageMapItemKey("Oracle", "Aggregate", exchangeRateOracleKey.toHex());
+    const exchangeRateStorageKey = sudoInterBtcAPI.api.query.oracle.aggregate.key(exchangeRateOracleKey);
     await setStorageAtKey(sudoInterBtcAPI.api, exchangeRateStorageKey, newExchangeRateHex, sudoAccount);
 }