diff --git a/app/Cli.ts b/app/Cli.ts index 6d3c888..dde1ec9 100644 --- a/app/Cli.ts +++ b/app/Cli.ts @@ -86,6 +86,9 @@ let app: App; const netConfig: NetworkConfig = { rpc: process.env.NETWORK_RPC, max_block_delay: process.env.NETWORK_MAX_BLOCK_DELAY ? parseInt(process.env.NETWORK_MAX_BLOCK_DELAY) : undefined, + max_new_block_delay: process.env.NETWORK_MAX_NEW_BLOCK_DELAY + ? parseInt(process.env.NETWORK_MAX_NEW_BLOCK_DELAY) + : undefined, resolve_min_success_count: process.env.NETWORK_MIN_SUCCESS_RESOLVE ? parseInt(process.env.NETWORK_MIN_SUCCESS_RESOLVE) : undefined, diff --git a/app/ConfigGetters.ts b/app/ConfigGetters.ts index ced51fa..7d444b5 100644 --- a/app/ConfigGetters.ts +++ b/app/ConfigGetters.ts @@ -77,6 +77,7 @@ export function getDefaultExecutorConfig() { export function getDefaultNetworkConfig() { return { max_block_delay: 60, + max_new_block_delay: 10, resolve_min_success_count: 3, }; } diff --git a/app/Network.ts b/app/Network.ts index 9338c89..5830b5c 100644 --- a/app/Network.ts +++ b/app/Network.ts @@ -39,6 +39,7 @@ export class Network { private readonly networkConfig: NetworkConfig; private readonly rpc: string; private readonly maxBlockDelay: number; + private readonly maxNewBlockDelay: number; private chainId: number; private provider: ethers.providers.WebSocketProvider | undefined; private agents: IAgent[]; @@ -80,6 +81,7 @@ export class Network { setConfigDefaultValues(networkConfig, getDefaultNetworkConfig()); this.rpc = networkConfig.rpc; this.maxBlockDelay = networkConfig.max_block_delay; + this.maxNewBlockDelay = networkConfig.max_new_block_delay; this.networkConfig = networkConfig; this.agents = agents; @@ -294,10 +296,23 @@ export class Network { blockNumber = BigInt(blockNumber.toString()); const before = this.nowMs(); const block = await this.queryBlock(blockNumber); + if (!block) { + setTimeout(() => { + this._onNewBlockCallback(blockNumber); + }, 1000); + this.clog('error', `⚠️ Block not found (number=${blockNumber},nowMs=${this.nowMs()})`); + return null; + } const fetchBlockDelay = this.nowMs() - before; + if (process.env.NODE_ENV !== 'test') { + this.clog( + 'info', + `🧱 New block: (number=${blockNumber},timestamp=${block.timestamp},hash=${block.hash},txCount=${block.transactions.length},baseFee=${block.baseFeePerGas},fetchDelayMs=${fetchBlockDelay})`, + ); + } if (this.latestBlockNumber && blockNumber <= this.latestBlockNumber) { - return; + return null; } this.latestBlockNumber = blockNumber; this.latestBaseFee = BigInt(block.baseFeePerGas.toString()); @@ -316,12 +331,25 @@ export class Network { this.newBlockNotifications.set(blockNumber, new Set([block.hash])); this.walkThroughTheJobs(blockNumber, block.timestamp); } - if (process.env.NODE_ENV !== 'test') { + + setTimeout(async () => { + if (this.latestBlockNumber > blockNumber) { + return; + } this.clog( - 'info', - `🧱 New block: (number=${blockNumber},timestamp=${block.timestamp},hash=${block.hash},txCount=${block.transactions.length},baseFee=${block.baseFeePerGas},fetchDelayMs=${fetchBlockDelay})`, + 'error', + `⏲ New block timeout: (number=${blockNumber},before=${before},nowMs=${this.nowMs()},maxNewBlockDelay=${ + this.maxNewBlockDelay + })`, ); - } + this.newBlockEventEmitter.emit('newBlockDelay', blockNumber); + let block; + do { + block = await this._onNewBlockCallback(++blockNumber); + } while (block); + }, this.maxNewBlockDelay * 1000); + + return block; } public isBlockDelayAboveMax() { @@ -329,7 +357,11 @@ export class Network { } public blockDelay() { - return this.currentBlockDelay - this.maxBlockDelay; + return this.currentBlockDelay; + } + + public getMaxNewBlockDelay() { + return this.maxNewBlockDelay; } public getNewBlockEventEmitter(): EventEmitter { diff --git a/app/Types.ts b/app/Types.ts index 88857be..74129d6 100644 --- a/app/Types.ts +++ b/app/Types.ts @@ -66,6 +66,7 @@ export interface NetworkConfig { rpc: string; ws_timeout?: number; max_block_delay?: number; + max_new_block_delay?: number; resolve_min_success_count?: number; flashbots?: { rpc: string; @@ -229,6 +230,7 @@ export interface Executor { init(); push(key: string, tx: TxEnvelope); sendBlockDelayLog(agent: IAgent, delay, blockNumber); + sendNewBlockDelayLog(agent: IAgent, delay, blockNumber); } export interface ClientWrapper { diff --git a/app/agents/AbstractAgent.ts b/app/agents/AbstractAgent.ts index 0f3501a..32531cf 100644 --- a/app/agents/AbstractAgent.ts +++ b/app/agents/AbstractAgent.ts @@ -179,6 +179,8 @@ export abstract class AbstractAgent implements IAgent { this.network.getNewBlockEventEmitter().on('newBlock', this.newBlockEventHandler.bind(this)); + this.network.getNewBlockEventEmitter().on('newBlockDelay', this.newBlockDelayEventHandler.bind(this)); + // Ensure version matches // TODO: extract check const version = await this.queryContractVersion(); @@ -294,6 +296,11 @@ export abstract class AbstractAgent implements IAgent { } } + // eslint-disable-next-line @typescript-eslint/no-unused-vars + private newBlockDelayEventHandler(blockNumber) { + this.executor.sendNewBlockDelayLog(this, this.network.getMaxNewBlockDelay(), blockNumber); + } + public exitIfStrictTopic(topic) { this.network.exitIfStrictTopic(topic); } @@ -647,7 +654,7 @@ export abstract class AbstractAgent implements IAgent { 'info', `Terminate agent, minKeeperCvp: ${ethers.utils.formatEther( this.minKeeperCvp, - )}, myStake: ${ethers.utils.formatEther(this.myStake)}`, + )}, myStake: ${ethers.utils.formatEther(this.myStake)}, delay: ${this.network.blockDelay()}`, ); this.isAgentUp = false; this.stopAllJobs(); diff --git a/app/executors/AbstractExecutor.ts b/app/executors/AbstractExecutor.ts index d849915..d1af81a 100644 --- a/app/executors/AbstractExecutor.ts +++ b/app/executors/AbstractExecutor.ts @@ -142,14 +142,22 @@ export abstract class AbstractExecutor { } this.lastDelaySentAtMs = this.network.nowMs(); + return this.logBlockDelay(agent, delay, blockNumber); + } + + async sendNewBlockDelayLog(agent: IAgent, delay, blockNumber) { + return this.logBlockDelay(agent, delay, blockNumber); + } + + async logBlockDelay(agent: IAgent, delay, blockNumber) { const types = { Mail: [{ name: 'metadataJson', type: 'string' }], }; - const networkStatusObj = this.network.getStatusObjectForApi(); const blockData = { metadataJson: JSON.stringify({ delay, + isNotEmitted: true, keeperId: agent.keeperId, rpc: networkStatusObj['rpc'], rpcClient: await this.network.getClientVersion(),