improve logic for arbitrum network (#71)

powerpool-finance · Dec 10, 2023 · 181de25 · 181de25
1 parent fb0d846
commit 181de25
Show file tree

Hide file tree

Showing 12 changed files with 148 additions and 82 deletions.
diff --git a/app/ConfigGetters.ts b/app/ConfigGetters.ts
@@ -76,14 +76,23 @@ export function getDefaultExecutorConfig() {
 
 export function getDefaultNetworkConfig(name) {
   const avgBlockTime = AVERAGE_BLOCK_TIME_SECONDS[name];
+  const max_new_block_delay = avgBlockTime ? avgBlockTime * 10 : 15;
   return {
     max_block_delay: 60,
-    max_new_block_delay: avgBlockTime ? avgBlockTime * 3 : 10,
+    max_new_block_delay: max_new_block_delay < 15 ? 15 : max_new_block_delay,
     resolve_min_success_count: 3,
     block_logs_mode: false,
   };
 }
 
+export function getMaxBlocksSubgraphDelay(networkName) {
+  return (
+    {
+      arbitrumOne: 1000,
+    }[networkName] || 10
+  );
+}
+
 export function setConfigDefaultValues(config, defaultValues) {
   Object.keys(defaultValues).forEach(name => {
     if (typeof config[name] === 'undefined') {

diff --git a/app/Network.ts b/app/Network.ts
@@ -55,7 +55,6 @@ export class Network {
   private flashbotsAddress: string;
   private flashbotsPass: string;
   private flashbotsRpc: string;
-  private newBlockNotifications: Map<number, Set<string>>;
   private contractWrapperFactory: ContractWrapperFactory;
   private newBlockEventEmitter: EventEmitter;
   private contractEventsEmitter: ContractEventsEmitter;
@@ -98,8 +97,6 @@ export class Network {
     this.newBlockEventEmitter = new EventEmitter();
     this.contractEventsEmitter = new ContractEventsEmitter(networkConfig.block_logs_mode);
 
-    this.newBlockNotifications = new Map();
-
     if (!this.rpc && !this.rpc.startsWith('ws')) {
       throw this.err(
         `Only WebSockets RPC endpoints are supported. The current value for '${this.getName()}' is '${this.rpc}'.`,
@@ -338,6 +335,12 @@ export class Network {
   private async _onNewBlockCallback(blockNumber) {
     blockNumber = BigInt(blockNumber.toString());
     const before = this.nowMs();
+
+    if (this.latestBlockNumber && blockNumber <= this.latestBlockNumber) {
+      return null;
+    }
+    this.latestBlockNumber = blockNumber;
+
     const block = await this.queryBlock(blockNumber);
     if (!block) {
       setTimeout(() => {
@@ -353,27 +356,13 @@ export class Network {
         `🧱 New block: (number=${blockNumber},timestamp=${block.timestamp},hash=${block.hash},txCount=${block.transactions.length},baseFee=${block.baseFeePerGas},fetchDelayMs=${fetchBlockDelay})`,
       );
     }
-
-    if (this.latestBlockNumber && blockNumber <= this.latestBlockNumber) {
-      return null;
-    }
-    this.latestBlockNumber = blockNumber;
     this.latestBaseFee = BigInt(block.baseFeePerGas.toString());
     this.latestBlockTimestamp = BigInt(block.timestamp.toString());
     this.currentBlockDelay = this.nowS() - parseInt(block.timestamp.toString());
 
     this.newBlockEventEmitter.emit('newBlock', block.timestamp, blockNumber);
 
-    if (this.newBlockNotifications.has(blockNumber)) {
-      const emittedBlockHashes = this.newBlockNotifications.get(blockNumber);
-      if (emittedBlockHashes && !emittedBlockHashes.has(block.hash)) {
-        emittedBlockHashes.add(block.hash);
-        this.walkThroughTheJobs(blockNumber, block.timestamp);
-      }
-    } else {
-      this.newBlockNotifications.set(blockNumber, new Set([block.hash]));
-      this.walkThroughTheJobs(blockNumber, block.timestamp);
-    }
+    this.walkThroughTheJobs(blockNumber, block.timestamp);
 
     setTimeout(async () => {
       if (this.latestBlockNumber > blockNumber) {

diff --git a/app/Types.ts b/app/Types.ts
@@ -359,6 +359,8 @@ export interface IRandaoAgent extends IAgent {
 }
 
 export interface IDataSource {
+  getType(): string;
+  getBlocksDelay(): Promise<bigint>;
   getRegisteredJobs(_context): Promise<Map<string, RandaoJob | LightJob>>;
   getOwnersBalances(context, jobOwnersSet: Set<string>): Promise<Map<string, BigNumber>>;
   addLensFieldsToOneJob(newJobs: RandaoJob | LightJob): void;

diff --git a/app/Utils.ts b/app/Utils.ts
@@ -61,11 +61,11 @@ export function jsonStringify(obj: any): string {
   );
 }
 
-export function prepareTx(tx: UnsignedTransaction) {
+export function prepareTx(tx: UnsignedTransaction, isEstimate = false) {
   const resTx = {
     ...tx,
     value: bigintToHex(tx.value),
-    gasLimit: bigintToHex(tx.gasLimit),
+    gasLimit: bigintToHex(isEstimate ? tx.gasLimit : 5_000_000n),
     gasPrice: bigintToHex(tx.gasPrice),
     maxPriorityFeePerGas: bigintToHex(tx.maxPriorityFeePerGas),
     maxFeePerGas: bigintToHex(tx.maxFeePerGas),

diff --git a/app/agents/AbstractAgent.ts b/app/agents/AbstractAgent.ts
@@ -16,7 +16,7 @@ import {
 } from '../Types.js';
 import { BigNumber, ethers, Wallet } from 'ethers';
 import { getEncryptedJson } from '../services/KeyService.js';
-import { BN_ZERO, DEFAULT_SYNC_FROM_CHAINS } from '../Constants.js';
+import { AVERAGE_BLOCK_TIME_SECONDS, BN_ZERO, DEFAULT_SYNC_FROM_CHAINS } from '../Constants.js';
 import { numberToBigInt, toChecksummedAddress, weiValueToEth } from '../Utils.js';
 import { FlashbotsExecutor } from '../executors/FlashbotsExecutor.js';
 import { PGAExecutor } from '../executors/PGAExecutor.js';
@@ -427,7 +427,8 @@ export abstract class AbstractAgent implements IAgent {
    * 4. Handle SetJobResolver events
    * @private
    */
-  private async resyncAllJobs(): Promise<number> {
+  private async resyncAllJobs(skipRepeat = false): Promise<number> {
+    this.clog('info', 'resyncAllJobs start');
     const latestBock = this.network.getLatestBlockNumber();
     // 1. init jobs
     let newJobs = new Map<string, RandaoJob | LightJob>();
@@ -455,6 +456,17 @@ export abstract class AbstractAgent implements IAgent {
 
     await this.startAllJobs();
 
+    if (!skipRepeat && this.dataSource.getType() === 'subgraph' && this.networkName !== 'testnet') {
+      const blockDelay = await this.dataSource.getBlocksDelay().catch(() => null);
+      if (blockDelay > 10) {
+        setTimeout(() => {
+          this.resyncAllJobs(true);
+        }, parseInt(blockDelay.toString()) * AVERAGE_BLOCK_TIME_SECONDS[this.networkName]);
+      }
+    }
+
+    this.clog('info', `resyncAllJobs end (${Array.from(this.jobs.keys()).length} synced)`);
+
     return Number(latestBock);
   }
   abstract _buildNewJob(event): LightJob | RandaoJob;
@@ -591,8 +603,13 @@ export abstract class AbstractAgent implements IAgent {
   private parseAndSetUnrecognizedErrorMessage(err) {
     try {
       let decodedError;
-      const reason = err.reason || (err.message && err.message.toString());
-      if (reason && reason.includes('unrecognized custom error')) {
+      const reason =
+        err.reason && err.reason !== 'execution reverted' ? err.reason : err.message && err.message.toString();
+      if (reason && reason.includes('response":"')) {
+        decodedError = this.contract.decodeError(
+          JSON.parse(JSON.parse(`"${reason.split('response":"')[1].split('"},')[0]}"`)).error.data,
+        );
+      } else if (reason && reason.includes('unrecognized custom error')) {
         decodedError = this.contract.decodeError(reason.split('data: ')[1].slice(0, -1));
       } else if (reason && reason.includes('error={"code":3')) {
         // 'cannot estimate gas; transaction may fail or may require manual gas limit [ See: https://links.ethers.org/v5-errors-UNPREDICTABLE_GAS_LIMIT ] (reason="execution reverted", method="estimateGas", transaction={"from":"0x779bEfe2b4C43cD1F87924defd13c8b9d3B1E1d8","maxPriorityFeePerGas":{"type":"BigNumber","hex":"0x05196259dd"},"maxFeePerGas":{"type":"BigNumber","hex":"0x05196259ed"},"to":"0x071412e301C2087A4DAA055CF4aFa2683cE1e499","data":"0x00000000ef0b5a45ff9b79d4b9162130bf0cd44dcf68b90d0000010200003066f23ebc0000000000000000000000000000000000000000000000000000000000000000","type":2,"accessList":null}, error={"code":3,"response":"{\"jsonrpc\":\"2.0\",\"id\":20442,\"error\":{\"code\":3,\"message\":\"execution reverted\",\"data\":\"0xbe32c0ad\"}}\n"}, code=UNPREDICTABLE_GAS_LIMIT, version=providers/5.7.2)'
@@ -609,7 +626,9 @@ export abstract class AbstractAgent implements IAgent {
       err.message =
         `Error: VM Exception while processing transaction: reverted with ${decodedError.name} ` +
         `decoded error and ${JSON.stringify(decodedError.args)} args`;
-    } catch (_) {}
+    } catch (_) {
+      console.error('decode error', _);
+    }
   }
 
   private async trySendExecuteEnvelope(envelope: TxEnvelope) {

diff --git a/app/clients/EthersContract.ts b/app/clients/EthersContract.ts
@@ -109,7 +109,7 @@ export class EthersContract implements ContractWrapper {
     return {
       name: decoded.name,
       signature: decoded.signature,
-      args: filterFunctionResultObject(decoded.args),
+      args: filterFunctionResultObject(decoded.args, true),
     };
   }
   public decodeTxData(data: string): TxDataWrapper {
@@ -159,39 +159,51 @@ export class EthersContract implements ContractWrapper {
     if (!(method in this.contract)) {
       throw this.err(`Contract ${this.address} doesn't have method '${method}' in the provided abi.`);
     }
-    let errorCounter = this.attempts;
 
+    let timeout,
+      tries = 0;
     do {
-      const timeout = setTimeout(() => {
-        throw new Error(
-          `Call execution took more than
-           ${Math.ceil(this.wsCallTimeout / 1000)}
-           seconds: method=${method},args=${JSON.stringify(args)}.`,
-        );
-      }, this.wsCallTimeout);
-      try {
-        let res;
+      const res = await new Promise(async (resolve, reject) => {
+        let callRes;
+        timeout = setTimeout(() => {
+          console.log('callRes', callRes);
+          reject(
+            new Error(
+              `${Math.round(new Date().getTime() / 1000)}: Call execution took more than ` +
+                `${Math.ceil(this.wsCallTimeout / 1000)} seconds: ` +
+                `method=${method},args=${JSON.stringify(args)}.`,
+            ),
+          );
+        }, this.wsCallTimeout);
+
         if (callStatic) {
-          res = await this.contract.callStatic[method](...args);
+          callRes = await this.contract.callStatic[method](...args);
         } else {
-          res = await this.contract[method](...args);
+          callRes = await this.contract[method](...args);
+        }
+        resolve(filterFunctionResultObject(callRes));
+      }).catch(async e => {
+        if (e.message && e.message.includes('Call execution took more than')) {
+          this.clog('error', `${e.message} (attempt=${tries}/${this.attempts})`);
+        } else {
+          this.clog(
+            'error',
+            `Error executing a ethCall(): (attempt=${tries}/${this.attempts}): ` +
+              `querying method '${method}' with arguments ${JSON.stringify(args)} and overrides ` +
+              `${JSON.stringify(overrides)}: ${e.message}: ${Error().stack}`,
+          );
         }
-        clearTimeout(timeout);
-        return filterFunctionResultObject(res);
-      } catch (e) {
-        this.clog(
-          'error',
-          `Error executing a ethCall(): (attempt=${this.attempts - errorCounter}/${
-            this.attempts
-          }): querying method '${method}' with arguments ${JSON.stringify(args)} and overrides ${JSON.stringify(
-            overrides,
-          )}:
-${e.message}: ${Error().stack}`,
-        );
-        clearTimeout(timeout);
         await sleep(this.attemptTimeoutSeconds * 1000);
+        if (tries >= this.attempts) {
+          throw e;
+        }
+      });
+
+      clearTimeout(timeout);
+      if (res) {
+        return res;
       }
-    } while (errorCounter-- > 0);
+    } while (tries++ < this.attempts);
   }
 
   public async getPastEvents(eventName: string, from: number, to: number): Promise<EventWrapper[]> {
@@ -246,8 +258,20 @@ ${e.message}: ${Error().stack}`,
   }
 }
 
-function filterFunctionResultObject(res: Result): { [key: string]: any } {
+function filterFunctionResultObject(res: Result, numberToString = false): { [key: string]: any } {
   if (!Array.isArray(res)) {
+    if (typeof res === 'object' && numberToString) {
+      const clone = { ...res };
+      console.log('clone', clone);
+      Object.keys(clone).map(key => {
+        if (clone[key] && clone[key].hex) {
+          console.log('clone[key]', clone[key]);
+          console.log('BigInt(clone[key].hex)', BigInt(clone[key].hex));
+          clone[key] = BigInt(clone[key].hex).toString(10);
+        }
+      });
+      return clone;
+    }
     return res;
   }
 

diff --git a/app/dataSources/AbstractSource.ts b/app/dataSources/AbstractSource.ts
@@ -48,4 +48,12 @@ export abstract class AbstractSource implements IDataSource {
       return objectKey ? value[objectKey] : value;
     }
   }
+
+  public getType(): string {
+    return this.type;
+  }
+
+  async getBlocksDelay(): Promise<bigint> {
+    return null;
+  }
 }
diff --git a/app/dataSources/SubgraphSource.ts b/app/dataSources/SubgraphSource.ts
@@ -8,6 +8,7 @@ import { IAgent } from '../Types';
 import { BigNumber, utils } from 'ethers';
 import { toChecksummedAddress } from '../Utils.js';
 import logger from '../services/Logger.js';
+import { getMaxBlocksSubgraphDelay } from '../ConfigGetters.js';
 
 export const QUERY_ALL_JOBS = `{
   jobs(first: 1000) {
@@ -105,13 +106,8 @@ export class SubgraphSource extends AbstractSource {
    */
   async isGraphOk(): Promise<boolean> {
     try {
-      const [latestBock, { _meta }] = await Promise.all([
-        this.network.getLatestBlockNumber(),
-        this.query(this.subgraphUrl, QUERY_META),
-      ]);
-
-      const diff = latestBock - BigInt(_meta.block.number);
-      const isSynced = diff <= 10; // Our graph is desynced if its behind for more than 10 blocks
+      const diff = await this.getBlocksDelay();
+      const isSynced = diff <= getMaxBlocksSubgraphDelay(this.network.getName()); // Our graph is desynced if its behind for more than 10 blocks
       if (!isSynced) {
         this.clog('error', `Subgraph is ${diff} blocks behind.`);
       }
@@ -122,6 +118,14 @@ export class SubgraphSource extends AbstractSource {
     }
   }
 
+  async getBlocksDelay(): Promise<bigint> {
+    const [latestBock, { _meta }] = await Promise.all([
+      this.network.getLatestBlockNumber(),
+      this.query(this.subgraphUrl, QUERY_META),
+    ]);
+    return latestBock - BigInt(_meta.block.number);
+  }
+
   /**
    * Getting a list of jobs from subgraph and initialise job.
    * Returns Map structure which key is jobKey and value is instance of RandaoJob or LightJob. Await is required.
@@ -132,8 +136,8 @@ export class SubgraphSource extends AbstractSource {
    */
   async getRegisteredJobs(context): Promise<Map<string, RandaoJob | LightJob>> {
     let newJobs = new Map<string, RandaoJob | LightJob>();
-    const graphIsFine = await this.isGraphOk();
-    if (!graphIsFine) {
+    const isSynced = await this.isGraphOk();
+    if (!isSynced) {
       this.clog('warn', 'Subgraph is not ok, falling back to the blockchain datasource.');
       newJobs = await this.blockchainSource.getRegisteredJobs(context);
       return newJobs;
@@ -225,8 +229,8 @@ export class SubgraphSource extends AbstractSource {
   public async getOwnersBalances(context, jobOwnersSet: Set<string>): Promise<Map<string, BigNumber>> {
     let result = new Map<string, BigNumber>();
     try {
-      const graphIsFine = await this.isGraphOk();
-      if (!graphIsFine) {
+      const isSynced = await this.isGraphOk();
+      if (!isSynced) {
         this.clog('warn', 'Subgraph is not ok, falling back to the blockchain datasource.');
         result = await this.blockchainSource.getOwnersBalances(context, jobOwnersSet);
         return result;

diff --git a/app/executors/PGAExecutor.ts b/app/executors/PGAExecutor.ts
@@ -58,9 +58,16 @@ export class PGAExecutor extends AbstractExecutor implements Executor {
       let txSimulation;
       try {
         txSimulation = await this.genericProvider.call(prepareTx(tx));
-      } catch (e) {
-        envelope.executorCallbacks.txEstimationFailed(e, tx.data as string);
-        return callback(this.err(`gasLimitEstimation failed with error: ${e.message}`));
+      } catch (_e) {
+        envelope.executorCallbacks.txEstimationFailed(_e, tx.data as string);
+        return callback(this.err(`gasLimitEstimation failed with error: ${_e.message}`));
+      }
+      if (e.message && e.message.includes('insufficient funds')) {
+        try {
+          await this.genericProvider.estimateGas(prepareTx(tx, true));
+        } catch (_e) {
+          e = _e;
+        }
       }
       printSolidityCustomError(
         this.clog.bind(this),

diff --git a/app/jobs/RandaoJob.ts b/app/jobs/RandaoJob.ts
@@ -301,7 +301,12 @@ export class RandaoJob extends AbstractJob {
 
   protected _executeTxEstimationFailedRewatch(err: Error, _txData: string) {
     if (this._getCurrentPeriod() === 3) {
-      this.clog('info', 'Scheduling self-unassign since the current period is #3...');
+      this.clog(
+        'info',
+        `Scheduling self-unassign since the current period is #3 and transaction failed: ${
+          err ? err.message : 'Unknown error'
+        }`,
+      );
       this._selfUnassign();
       this.watch();
       return;

diff --git a/package.json b/package.json
@@ -11,6 +11,7 @@
     "lint": "eslint --ext .ts app/ tests/",
     "lint:fix": "eslint --fix --ext .ts app/ tests/",
     "test": "NODE_ENV=test LOG_LEVEL=error mocha tests/**/*.test.ts",
+    "test:debug": "NODE_ENV=test LOG_LEVEL=debug mocha tests/**/*.test.ts",
     "write-version-data": "DIR=dist ts-node ./writeVersionData.ts"
   },
   "author": "PowerPool",