Max new block delay

app/Cli.ts

@@ -86,6 +86,9 @@ let app: App;
     const netConfig: NetworkConfig = {
       rpc: process.env.NETWORK_RPC,
       max_block_delay: process.env.NETWORK_MAX_BLOCK_DELAY ? parseInt(process.env.NETWORK_MAX_BLOCK_DELAY) : undefined,
+      max_new_block_delay: process.env.NETWORK_MAX_NEW_BLOCK_DELAY
+        ? parseInt(process.env.NETWORK_MAX_NEW_BLOCK_DELAY)
+        : undefined,
       resolve_min_success_count: process.env.NETWORK_MIN_SUCCESS_RESOLVE
         ? parseInt(process.env.NETWORK_MIN_SUCCESS_RESOLVE)
         : undefined,

app/ConfigGetters.ts

@@ -77,6 +77,7 @@ export function getDefaultExecutorConfig() {
 export function getDefaultNetworkConfig() {
   return {
     max_block_delay: 60,
+    max_new_block_delay: 10,
     resolve_min_success_count: 3,
   };
 }

app/Network.ts

@@ -39,6 +39,7 @@ export class Network {
   private readonly networkConfig: NetworkConfig;
   private readonly rpc: string;
   private readonly maxBlockDelay: number;
+  private readonly maxNewBlockDelay: number;
   private chainId: number;
   private provider: ethers.providers.WebSocketProvider | undefined;
   private agents: IAgent[];
@@ -80,6 +81,7 @@ export class Network {
     setConfigDefaultValues(networkConfig, getDefaultNetworkConfig());
     this.rpc = networkConfig.rpc;
     this.maxBlockDelay = networkConfig.max_block_delay;
+    this.maxNewBlockDelay = networkConfig.max_new_block_delay;
     this.networkConfig = networkConfig;
     this.agents = agents;
 
@@ -294,10 +296,23 @@ export class Network {
     blockNumber = BigInt(blockNumber.toString());
     const before = this.nowMs();
     const block = await this.queryBlock(blockNumber);
+    if (!block) {
+      setTimeout(() => {
+        this._onNewBlockCallback(blockNumber);
+      }, 1000);
+      this.clog('error', `⚠️ Block not found (number=${blockNumber},nowMs=${this.nowMs()})`);
+      return null;
+    }
     const fetchBlockDelay = this.nowMs() - before;
+    if (process.env.NODE_ENV !== 'test') {
+      this.clog(
+        'info',
+        `🧱 New block: (number=${blockNumber},timestamp=${block.timestamp},hash=${block.hash},txCount=${block.transactions.length},baseFee=${block.baseFeePerGas},fetchDelayMs=${fetchBlockDelay})`,
+      );
+    }
 
     if (this.latestBlockNumber && blockNumber <= this.latestBlockNumber) {
-      return;
+      return null;
     }
     this.latestBlockNumber = blockNumber;
     this.latestBaseFee = BigInt(block.baseFeePerGas.toString());
@@ -316,20 +331,37 @@ export class Network {
       this.newBlockNotifications.set(blockNumber, new Set([block.hash]));
       this.walkThroughTheJobs(blockNumber, block.timestamp);
     }
-    if (process.env.NODE_ENV !== 'test') {
+
+    setTimeout(async () => {
+      if (this.latestBlockNumber > blockNumber) {
+        return;
+      }
       this.clog(
-        'info',
-        `🧱 New block: (number=${blockNumber},timestamp=${block.timestamp},hash=${block.hash},txCount=${block.transactions.length},baseFee=${block.baseFeePerGas},fetchDelayMs=${fetchBlockDelay})`,
+        'error',
+        `⏲ New block timeout: (number=${blockNumber},before=${before},nowMs=${this.nowMs()},maxNewBlockDelay=${
+          this.maxNewBlockDelay
+        })`,
       );
-    }
+      this.newBlockEventEmitter.emit('newBlockDelay', blockNumber);
+      let block;
+      do {
+        block = await this._onNewBlockCallback(++blockNumber);
+      } while (block);
+    }, this.maxNewBlockDelay * 1000);
+
+    return block;
   }
 
   public isBlockDelayAboveMax() {
     return this.currentBlockDelay && this.currentBlockDelay > this.maxBlockDelay;
   }
 
   public blockDelay() {
-    return this.currentBlockDelay - this.maxBlockDelay;
+    return this.currentBlockDelay;
+  }
+
+  public getMaxNewBlockDelay() {
+    return this.maxNewBlockDelay;
   }
 
   public getNewBlockEventEmitter(): EventEmitter {

app/Types.ts

@@ -66,6 +66,7 @@ export interface NetworkConfig {
   rpc: string;
   ws_timeout?: number;
   max_block_delay?: number;
+  max_new_block_delay?: number;
   resolve_min_success_count?: number;
   flashbots?: {
     rpc: string;
@@ -229,6 +230,7 @@ export interface Executor {
   init();
   push(key: string, tx: TxEnvelope);
   sendBlockDelayLog(agent: IAgent, delay, blockNumber);
+  sendNewBlockDelayLog(agent: IAgent, delay, blockNumber);
 }
 
 export interface ClientWrapper {

app/agents/AbstractAgent.ts

@@ -179,6 +179,8 @@ export abstract class AbstractAgent implements IAgent {
 
     this.network.getNewBlockEventEmitter().on('newBlock', this.newBlockEventHandler.bind(this));
 
+    this.network.getNewBlockEventEmitter().on('newBlockDelay', this.newBlockDelayEventHandler.bind(this));
+
     // Ensure version matches
     // TODO: extract check
     const version = await this.queryContractVersion();
@@ -294,6 +296,11 @@ export abstract class AbstractAgent implements IAgent {
     }
   }
 
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  private newBlockDelayEventHandler(blockNumber) {
+    this.executor.sendNewBlockDelayLog(this, this.network.getMaxNewBlockDelay(), blockNumber);
+  }
+
   public exitIfStrictTopic(topic) {
     this.network.exitIfStrictTopic(topic);
   }
@@ -647,7 +654,7 @@ export abstract class AbstractAgent implements IAgent {
       'info',
       `Terminate agent, minKeeperCvp: ${ethers.utils.formatEther(
         this.minKeeperCvp,
-      )}, myStake: ${ethers.utils.formatEther(this.myStake)}`,
+      )}, myStake: ${ethers.utils.formatEther(this.myStake)}, delay: ${this.network.blockDelay()}`,
     );
     this.isAgentUp = false;
     this.stopAllJobs();

app/executors/AbstractExecutor.ts

@@ -142,14 +142,22 @@ export abstract class AbstractExecutor {
     }
     this.lastDelaySentAtMs = this.network.nowMs();
 
+    return this.logBlockDelay(agent, delay, blockNumber);
+  }
+
+  async sendNewBlockDelayLog(agent: IAgent, delay, blockNumber) {
+    return this.logBlockDelay(agent, delay, blockNumber);
+  }
+
+  async logBlockDelay(agent: IAgent, delay, blockNumber) {
     const types = {
       Mail: [{ name: 'metadataJson', type: 'string' }],
     };
-
     const networkStatusObj = this.network.getStatusObjectForApi();
     const blockData = {
       metadataJson: JSON.stringify({
         delay,
+        isNotEmitted: true,
         keeperId: agent.keeperId,
         rpc: networkStatusObj['rpc'],
         rpcClient: await this.network.getClientVersion(),