diff --git a/app/Types.ts b/app/Types.ts index 74129d6..ecb977d 100644 --- a/app/Types.ts +++ b/app/Types.ts @@ -231,6 +231,7 @@ export interface Executor { push(key: string, tx: TxEnvelope); sendBlockDelayLog(agent: IAgent, delay, blockNumber); sendNewBlockDelayLog(agent: IAgent, delay, blockNumber); + sendAddBlacklistedJob(agent: IAgent, jobKey, errMessage); } export interface ClientWrapper { @@ -400,7 +401,7 @@ export interface IAgent { exitIfStrictTopic(topic): void; - addJobToBlacklist(jobKey); + addJobToBlacklist(jobKey, errMessage); getIsAgentUp(): boolean; diff --git a/app/Utils.ts b/app/Utils.ts index 5ca7248..888fe10 100644 --- a/app/Utils.ts +++ b/app/Utils.ts @@ -118,7 +118,11 @@ export function buildSignature(abiItem: { name: string; inputs: object[] }): str } export function buildAbiSelector(signature: string): string { - return keccak256(utils.toUtf8Bytes(signature)).slice(0, 10); + return hashString(signature).slice(0, 10); +} + +export function hashString(signature: string): string { + return keccak256(utils.toUtf8Bytes(signature)); } export function parseRawJob(rawJob: string): ParsedRawJob { diff --git a/app/agents/AbstractAgent.ts b/app/agents/AbstractAgent.ts index 32531cf..f42713f 100644 --- a/app/agents/AbstractAgent.ts +++ b/app/agents/AbstractAgent.ts @@ -305,9 +305,10 @@ export abstract class AbstractAgent implements IAgent { this.network.exitIfStrictTopic(topic); } - public addJobToBlacklist(jobKey) { - this.clog('info', `addJobToBlacklist: ${jobKey}`); + public addJobToBlacklist(jobKey, errMessage) { + this.clog('info', `addJobToBlacklist: ${jobKey}, errMessage ${errMessage}`); this.blacklistedJobs.add(jobKey); + this.executor.sendAddBlacklistedJob(this, jobKey, errMessage); } public getJobOwnerBalance(address: string): BigNumber { @@ -585,12 +586,24 @@ export abstract class AbstractAgent implements IAgent { private parseAndSetUnrecognizedErrorMessage(err) { try { + let decodedError; if (err.reason && err.reason.includes('unrecognized custom error')) { - const decodedError = this.contract.decodeError(err.reason.split('data: ')[1].slice(0, -1)); - err.message = `Error: VM Exception while processing transaction: reverted with ${ - decodedError.name - } decoded error and ${JSON.stringify(decodedError.args)} args`; + decodedError = this.contract.decodeError(err.reason.split('data: ')[1].slice(0, -1)); + } else if (err.message && err.message.includes('error={"code":3')) { + // 'cannot estimate gas; transaction may fail or may require manual gas limit [ See: https://links.ethers.org/v5-errors-UNPREDICTABLE_GAS_LIMIT ] (reason="execution reverted", method="estimateGas", transaction={"from":"0x779bEfe2b4C43cD1F87924defd13c8b9d3B1E1d8","maxPriorityFeePerGas":{"type":"BigNumber","hex":"0x05196259dd"},"maxFeePerGas":{"type":"BigNumber","hex":"0x05196259ed"},"to":"0x071412e301C2087A4DAA055CF4aFa2683cE1e499","data":"0x00000000ef0b5a45ff9b79d4b9162130bf0cd44dcf68b90d0000010200003066f23ebc0000000000000000000000000000000000000000000000000000000000000000","type":2,"accessList":null}, error={"code":3,"response":"{\"jsonrpc\":\"2.0\",\"id\":20442,\"error\":{\"code\":3,\"message\":\"execution reverted\",\"data\":\"0xbe32c0ad\"}}\n"}, code=UNPREDICTABLE_GAS_LIMIT, version=providers/5.7.2)' + // -> + // '{"code":3,"response":{"jsonrpc":"2.0","id":20442,"error":{"code":3,"message":"execution reverted","data":"0xbe32c0ad"}}}' + const responseJson = err.message + .split('error=')[1] + .split(', code=UNPREDICTABLE_GAS_LIMIT')[0] + .replace('\n', '') + .replace('}"', '}') + .replace('"{', '{'); + decodedError = this.contract.decodeError(JSON.parse(responseJson).response.error.data); } + err.message = + `Error: VM Exception while processing transaction: reverted with ${decodedError.name} ` + + `decoded error and ${JSON.stringify(decodedError.args)} args`; } catch (_) {} } diff --git a/app/artifacts/PPAgentV2.3.0.randao.json b/app/artifacts/PPAgentV2.3.0.randao.json index fc374c9..aecfce9 100644 --- a/app/artifacts/PPAgentV2.3.0.randao.json +++ b/app/artifacts/PPAgentV2.3.0.randao.json @@ -10,6 +10,11 @@ "stateMutability": "nonpayable", "type": "constructor" }, + { + "inputs": [], + "name": "ActivationNotInitiated", + "type": "error" + }, { "inputs": [ { diff --git a/app/executors/AbstractExecutor.ts b/app/executors/AbstractExecutor.ts index d1af81a..2764dfb 100644 --- a/app/executors/AbstractExecutor.ts +++ b/app/executors/AbstractExecutor.ts @@ -1,6 +1,6 @@ import { ethers } from 'ethers'; import { ContractWrapper, ExecutorConfig, IAgent, TxEnvelope } from '../Types.js'; -import { getTxString } from '../Utils.js'; +import { getTxString, hashString, jsonStringify } from '../Utils.js'; import { Network } from '../Network'; import axios from 'axios'; @@ -142,22 +142,22 @@ export abstract class AbstractExecutor { } this.lastDelaySentAtMs = this.network.nowMs(); - return this.logBlockDelay(agent, delay, blockNumber); + return this.logBlockDelay(agent, delay, blockNumber, false); } async sendNewBlockDelayLog(agent: IAgent, delay, blockNumber) { - return this.logBlockDelay(agent, delay, blockNumber); + return this.logBlockDelay(agent, delay, blockNumber, true); } - async logBlockDelay(agent: IAgent, delay, blockNumber) { + async logBlockDelay(agent: IAgent, delay, blockNumber, isNotEmitted) { const types = { Mail: [{ name: 'metadataJson', type: 'string' }], }; const networkStatusObj = this.network.getStatusObjectForApi(); const blockData = { - metadataJson: JSON.stringify({ + metadataJson: jsonStringify({ delay, - isNotEmitted: true, + isNotEmitted, keeperId: agent.keeperId, rpc: networkStatusObj['rpc'], rpcClient: await this.network.getClientVersion(), @@ -172,4 +172,30 @@ export abstract class AbstractExecutor { const txLogEndpoint = process.env.TX_LOG_ENDPOINT || 'https://tx-log.powerpool.finance'; return axios.post(`${txLogEndpoint}/log-block-delay`, { blockData, signature, signatureVersion: 1 }); } + + async sendAddBlacklistedJob(agent: IAgent, jobKey, errorMessage) { + const types = { + Mail: [{ name: 'metadataHash', type: 'string' }], + }; + const networkStatusObj = this.network.getStatusObjectForApi(); + const blockData = { + metadataJson: jsonStringify({ + jobKey, + errorMessage, + keeperId: agent.keeperId, + rpc: networkStatusObj['rpc'], + rpcClient: await this.network.getClientVersion(), + blockNumber: networkStatusObj['latestBlockNumber'], + agent: agent.address.toLowerCase(), + chainId: networkStatusObj['chainId'], + appVersion: this.network.getAppVersion(), + appEnv: process.env.APP_ENV, + }), + }; + const signature = await this.workerSigner._signTypedData({}, types, { + metadataHash: hashString(blockData.metadataJson), + }); + const txLogEndpoint = process.env.TX_LOG_ENDPOINT || 'https://tx-log.powerpool.finance'; + return axios.post(`${txLogEndpoint}/log-blacklist-job`, { blockData, signature, signatureVersion: 2 }); + } } diff --git a/app/executors/FlashbotsExecutor.ts b/app/executors/FlashbotsExecutor.ts index 31c4d6b..c65b3b9 100644 --- a/app/executors/FlashbotsExecutor.ts +++ b/app/executors/FlashbotsExecutor.ts @@ -69,7 +69,12 @@ export class FlashbotsExecutor extends AbstractExecutor implements Executor { let txSimulation; try { txSimulation = await this.genericProvider.call(prepareTx(tx)); - printSolidityCustomError(this.clog.bind(this), this.agentContract.decodeError, txSimulation, tx.data as string); + printSolidityCustomError( + this.clog.bind(this), + this.agentContract.decodeError.bind(this.agentContract), + txSimulation, + tx.data as string, + ); } catch (e) { this.clog('error', 'TX node simulation error', e); } diff --git a/app/executors/PGAExecutor.ts b/app/executors/PGAExecutor.ts index b57fb01..c3e92f1 100644 --- a/app/executors/PGAExecutor.ts +++ b/app/executors/PGAExecutor.ts @@ -3,7 +3,7 @@ import { ethers, utils } from 'ethers'; import { AbstractExecutor } from './AbstractExecutor.js'; import { printSolidityCustomError } from './ExecutorUtils.js'; import logger from '../services/Logger.js'; -import { prepareTx, weiValueToGwei } from '../Utils.js'; +import { prepareTx, weiValueToGwei, jsonStringify } from '../Utils.js'; import axios from 'axios'; import { Network } from '../Network'; @@ -62,7 +62,12 @@ export class PGAExecutor extends AbstractExecutor implements Executor { envelope.executorCallbacks.txEstimationFailed(e, tx.data as string); return callback(this.err(`gasLimitEstimation failed with error: ${e.message}`)); } - printSolidityCustomError(this.clog.bind(this), this.agentContract.decodeError, txSimulation, tx.data as string); + printSolidityCustomError( + this.clog.bind(this), + this.agentContract.decodeError.bind(this.agentContract), + txSimulation, + tx.data as string, + ); envelope.executorCallbacks.txEstimationFailed(e, tx.data as string); @@ -190,25 +195,26 @@ export class PGAExecutor extends AbstractExecutor implements Executor { confirmedAtBlockDateTime: new Date(parseInt(this.network.getLatestBlockTimestamp().toString()) * 1000), }; } + const chainId = networkStatusObj['chainId']; const txData = { - transactionJson: JSON.stringify(prepareTx(transaction)), - metadataJson: JSON.stringify({ + transactionJson: jsonStringify(prepareTx(transaction)), + metadataJson: jsonStringify({ appEnv: process.env.APP_ENV, appVersion: this.network.getAppVersion(), baseFeeGwei: weiValueToGwei(networkStatusObj['baseFee']), maxPriorityFeeGwei: weiValueToGwei(BigInt(await this.network.getMaxPriorityFeePerGas().catch(() => 0))), - chainId: networkStatusObj['chainId'], keeperId: agent ? agent.keeperId : null, rpc: networkStatusObj['rpc'], rpcClient: await this.network.getClientVersion(), resendCount, + chainId, txHash, prevTxHash, ...timeData, }), }; const signature = await this.workerSigner._signTypedData({}, types, txData); - const txLogEndpoint = process.env.TX_LOG_ENDPOINT || 'https://tx-log.powerpool.finance'; + const txLogEndpoint = process.env.TX_LOG_ENDPOINT || 'https://tx-log.powerpool.finance'; // TODO: add ${chainId}. return axios.post(`${txLogEndpoint}/log-transaction`, { txData, signature, signatureVersion: 1 }); } } diff --git a/app/jobs/RandaoJob.ts b/app/jobs/RandaoJob.ts index f1e5894..e317829 100644 --- a/app/jobs/RandaoJob.ts +++ b/app/jobs/RandaoJob.ts @@ -104,6 +104,12 @@ export class RandaoJob extends AbstractJob { this.reservedSlasherId = 0; } + public applyWasExecuted() { + this.slashingPossibleAfter = 0; + this.reservedSlasherId = 0; + super.applyWasExecuted(); + } + public applyClearResolverTimeouts(): void { this.t1 = 0; this.b1 = 0n; @@ -176,7 +182,7 @@ export class RandaoJob extends AbstractJob { if (this.failedInitiateSlashingEstimationsInARow > this.BLACKLIST_ESTIMATIONS_LIMIT) { this.applyClearResolverTimeouts(); - this.agent.addJobToBlacklist(this.key); + this.agent.addJobToBlacklist(this.key, err.message); this.failedInitiateSlashingEstimationsInARow = 0; } else { this._unlockInitiateSlashing(); @@ -305,7 +311,7 @@ export class RandaoJob extends AbstractJob { // Assume that a failed execution behaviour is equal to a failed estimation this.failedExecuteEstimationsInARow += 1; if (this.failedExecuteEstimationsInARow > this.BLACKLIST_ESTIMATIONS_LIMIT) { - this.agent.addJobToBlacklist(this.key); + this.agent.addJobToBlacklist(this.key, err.message); this.failedExecuteEstimationsInARow = 0; } }