Handle slashing failed, log blacklisted jobs

app/Types.ts

@@ -231,6 +231,7 @@ export interface Executor {
   push(key: string, tx: TxEnvelope);
   sendBlockDelayLog(agent: IAgent, delay, blockNumber);
   sendNewBlockDelayLog(agent: IAgent, delay, blockNumber);
+  sendAddBlacklistedJob(agent: IAgent, jobKey, errMessage);
 }
 
 export interface ClientWrapper {
@@ -400,7 +401,7 @@ export interface IAgent {
 
   exitIfStrictTopic(topic): void;
 
-  addJobToBlacklist(jobKey);
+  addJobToBlacklist(jobKey, errMessage);
 
   getIsAgentUp(): boolean;
 

app/Utils.ts

@@ -118,7 +118,11 @@ export function buildSignature(abiItem: { name: string; inputs: object[] }): str
 }
 
 export function buildAbiSelector(signature: string): string {
-  return keccak256(utils.toUtf8Bytes(signature)).slice(0, 10);
+  return hashString(signature).slice(0, 10);
+}
+
+export function hashString(signature: string): string {
+  return keccak256(utils.toUtf8Bytes(signature));
 }
 
 export function parseRawJob(rawJob: string): ParsedRawJob {

app/agents/AbstractAgent.ts

@@ -305,9 +305,10 @@ export abstract class AbstractAgent implements IAgent {
     this.network.exitIfStrictTopic(topic);
   }
 
-  public addJobToBlacklist(jobKey) {
-    this.clog('info', `addJobToBlacklist: ${jobKey}`);
+  public addJobToBlacklist(jobKey, errMessage) {
+    this.clog('info', `addJobToBlacklist: ${jobKey}, errMessage ${errMessage}`);
     this.blacklistedJobs.add(jobKey);
+    this.executor.sendAddBlacklistedJob(this, jobKey, errMessage);
   }
 
   public getJobOwnerBalance(address: string): BigNumber {
@@ -585,12 +586,24 @@ export abstract class AbstractAgent implements IAgent {
 
   private parseAndSetUnrecognizedErrorMessage(err) {
     try {
+      let decodedError;
       if (err.reason && err.reason.includes('unrecognized custom error')) {
-        const decodedError = this.contract.decodeError(err.reason.split('data: ')[1].slice(0, -1));
-        err.message = `Error: VM Exception while processing transaction: reverted with ${
-          decodedError.name
-        } decoded error and ${JSON.stringify(decodedError.args)} args`;
+        decodedError = this.contract.decodeError(err.reason.split('data: ')[1].slice(0, -1));
+      } else if (err.message && err.message.includes('error={"code":3')) {
+        // 'cannot estimate gas; transaction may fail or may require manual gas limit [ See: https://links.ethers.org/v5-errors-UNPREDICTABLE_GAS_LIMIT ] (reason="execution reverted", method="estimateGas", transaction={"from":"0x779bEfe2b4C43cD1F87924defd13c8b9d3B1E1d8","maxPriorityFeePerGas":{"type":"BigNumber","hex":"0x05196259dd"},"maxFeePerGas":{"type":"BigNumber","hex":"0x05196259ed"},"to":"0x071412e301C2087A4DAA055CF4aFa2683cE1e499","data":"0x00000000ef0b5a45ff9b79d4b9162130bf0cd44dcf68b90d0000010200003066f23ebc0000000000000000000000000000000000000000000000000000000000000000","type":2,"accessList":null}, error={"code":3,"response":"{\"jsonrpc\":\"2.0\",\"id\":20442,\"error\":{\"code\":3,\"message\":\"execution reverted\",\"data\":\"0xbe32c0ad\"}}\n"}, code=UNPREDICTABLE_GAS_LIMIT, version=providers/5.7.2)'
+        // ->
+        // '{"code":3,"response":{"jsonrpc":"2.0","id":20442,"error":{"code":3,"message":"execution reverted","data":"0xbe32c0ad"}}}'
+        const responseJson = err.message
+          .split('error=')[1]
+          .split(', code=UNPREDICTABLE_GAS_LIMIT')[0]
+          .replace('\n', '')
+          .replace('}"', '}')
+          .replace('"{', '{');
+        decodedError = this.contract.decodeError(JSON.parse(responseJson).response.error.data);
       }
+      err.message =
+        `Error: VM Exception while processing transaction: reverted with ${decodedError.name} ` +
+        `decoded error and ${JSON.stringify(decodedError.args)} args`;
     } catch (_) {}
   }
 

app/artifacts/PPAgentV2.3.0.randao.json

@@ -10,6 +10,11 @@
     "stateMutability": "nonpayable",
     "type": "constructor"
   },
+  {
+    "inputs": [],
+    "name": "ActivationNotInitiated",
+    "type": "error"
+  },
   {
     "inputs": [
       {

app/executors/AbstractExecutor.ts

@@ -1,6 +1,6 @@
 import { ethers } from 'ethers';
 import { ContractWrapper, ExecutorConfig, IAgent, TxEnvelope } from '../Types.js';
-import { getTxString } from '../Utils.js';
+import { getTxString, hashString, jsonStringify } from '../Utils.js';
 import { Network } from '../Network';
 import axios from 'axios';
 
@@ -142,22 +142,22 @@ export abstract class AbstractExecutor {
     }
     this.lastDelaySentAtMs = this.network.nowMs();
 
-    return this.logBlockDelay(agent, delay, blockNumber);
+    return this.logBlockDelay(agent, delay, blockNumber, false);
   }
 
   async sendNewBlockDelayLog(agent: IAgent, delay, blockNumber) {
-    return this.logBlockDelay(agent, delay, blockNumber);
+    return this.logBlockDelay(agent, delay, blockNumber, true);
   }
 
-  async logBlockDelay(agent: IAgent, delay, blockNumber) {
+  async logBlockDelay(agent: IAgent, delay, blockNumber, isNotEmitted) {
     const types = {
       Mail: [{ name: 'metadataJson', type: 'string' }],
     };
     const networkStatusObj = this.network.getStatusObjectForApi();
     const blockData = {
-      metadataJson: JSON.stringify({
+      metadataJson: jsonStringify({
         delay,
-        isNotEmitted: true,
+        isNotEmitted,
         keeperId: agent.keeperId,
         rpc: networkStatusObj['rpc'],
         rpcClient: await this.network.getClientVersion(),
@@ -172,4 +172,30 @@ export abstract class AbstractExecutor {
     const txLogEndpoint = process.env.TX_LOG_ENDPOINT || 'https://tx-log.powerpool.finance';
     return axios.post(`${txLogEndpoint}/log-block-delay`, { blockData, signature, signatureVersion: 1 });
   }
+
+  async sendAddBlacklistedJob(agent: IAgent, jobKey, errorMessage) {
+    const types = {
+      Mail: [{ name: 'metadataHash', type: 'string' }],
+    };
+    const networkStatusObj = this.network.getStatusObjectForApi();
+    const blockData = {
+      metadataJson: jsonStringify({
+        jobKey,
+        errorMessage,
+        keeperId: agent.keeperId,
+        rpc: networkStatusObj['rpc'],
+        rpcClient: await this.network.getClientVersion(),
+        blockNumber: networkStatusObj['latestBlockNumber'],
+        agent: agent.address.toLowerCase(),
+        chainId: networkStatusObj['chainId'],
+        appVersion: this.network.getAppVersion(),
+        appEnv: process.env.APP_ENV,
+      }),
+    };
+    const signature = await this.workerSigner._signTypedData({}, types, {
+      metadataHash: hashString(blockData.metadataJson),
+    });
+    const txLogEndpoint = process.env.TX_LOG_ENDPOINT || 'https://tx-log.powerpool.finance';
+    return axios.post(`${txLogEndpoint}/log-blacklist-job`, { blockData, signature, signatureVersion: 2 });
+  }
 }

app/executors/FlashbotsExecutor.ts

@@ -69,7 +69,12 @@ export class FlashbotsExecutor extends AbstractExecutor implements Executor {
       let txSimulation;
       try {
         txSimulation = await this.genericProvider.call(prepareTx(tx));
-        printSolidityCustomError(this.clog.bind(this), this.agentContract.decodeError, txSimulation, tx.data as string);
+        printSolidityCustomError(
+          this.clog.bind(this),
+          this.agentContract.decodeError.bind(this.agentContract),
+          txSimulation,
+          tx.data as string,
+        );
       } catch (e) {
         this.clog('error', 'TX node simulation error', e);
       }

app/executors/PGAExecutor.ts

@@ -3,7 +3,7 @@ import { ethers, utils } from 'ethers';
 import { AbstractExecutor } from './AbstractExecutor.js';
 import { printSolidityCustomError } from './ExecutorUtils.js';
 import logger from '../services/Logger.js';
-import { prepareTx, weiValueToGwei } from '../Utils.js';
+import { prepareTx, weiValueToGwei, jsonStringify } from '../Utils.js';
 import axios from 'axios';
 import { Network } from '../Network';
 
@@ -62,7 +62,12 @@ export class PGAExecutor extends AbstractExecutor implements Executor {
         envelope.executorCallbacks.txEstimationFailed(e, tx.data as string);
         return callback(this.err(`gasLimitEstimation failed with error: ${e.message}`));
       }
-      printSolidityCustomError(this.clog.bind(this), this.agentContract.decodeError, txSimulation, tx.data as string);
+      printSolidityCustomError(
+        this.clog.bind(this),
+        this.agentContract.decodeError.bind(this.agentContract),
+        txSimulation,
+        tx.data as string,
+      );
 
       envelope.executorCallbacks.txEstimationFailed(e, tx.data as string);
 
@@ -190,25 +195,26 @@ export class PGAExecutor extends AbstractExecutor implements Executor {
         confirmedAtBlockDateTime: new Date(parseInt(this.network.getLatestBlockTimestamp().toString()) * 1000),
       };
     }
+    const chainId = networkStatusObj['chainId'];
     const txData = {
-      transactionJson: JSON.stringify(prepareTx(transaction)),
-      metadataJson: JSON.stringify({
+      transactionJson: jsonStringify(prepareTx(transaction)),
+      metadataJson: jsonStringify({
         appEnv: process.env.APP_ENV,
         appVersion: this.network.getAppVersion(),
         baseFeeGwei: weiValueToGwei(networkStatusObj['baseFee']),
         maxPriorityFeeGwei: weiValueToGwei(BigInt(await this.network.getMaxPriorityFeePerGas().catch(() => 0))),
-        chainId: networkStatusObj['chainId'],
         keeperId: agent ? agent.keeperId : null,
         rpc: networkStatusObj['rpc'],
         rpcClient: await this.network.getClientVersion(),
         resendCount,
+        chainId,
         txHash,
         prevTxHash,
         ...timeData,
       }),
     };
     const signature = await this.workerSigner._signTypedData({}, types, txData);
-    const txLogEndpoint = process.env.TX_LOG_ENDPOINT || 'https://tx-log.powerpool.finance';
+    const txLogEndpoint = process.env.TX_LOG_ENDPOINT || 'https://tx-log.powerpool.finance'; // TODO: add ${chainId}.
     return axios.post(`${txLogEndpoint}/log-transaction`, { txData, signature, signatureVersion: 1 });
   }
 }

app/jobs/RandaoJob.ts

@@ -104,6 +104,12 @@ export class RandaoJob extends AbstractJob {
     this.reservedSlasherId = 0;
   }
 
+  public applyWasExecuted() {
+    this.slashingPossibleAfter = 0;
+    this.reservedSlasherId = 0;
+    super.applyWasExecuted();
+  }
+
   public applyClearResolverTimeouts(): void {
     this.t1 = 0;
     this.b1 = 0n;
@@ -176,7 +182,7 @@ export class RandaoJob extends AbstractJob {
 
       if (this.failedInitiateSlashingEstimationsInARow > this.BLACKLIST_ESTIMATIONS_LIMIT) {
         this.applyClearResolverTimeouts();
-        this.agent.addJobToBlacklist(this.key);
+        this.agent.addJobToBlacklist(this.key, err.message);
         this.failedInitiateSlashingEstimationsInARow = 0;
       } else {
         this._unlockInitiateSlashing();
@@ -305,7 +311,7 @@ export class RandaoJob extends AbstractJob {
       // Assume that a failed execution behaviour is equal to a failed estimation
       this.failedExecuteEstimationsInARow += 1;
       if (this.failedExecuteEstimationsInARow > this.BLACKLIST_ESTIMATIONS_LIMIT) {
-        this.agent.addJobToBlacklist(this.key);
+        this.agent.addJobToBlacklist(this.key, err.message);
         this.failedExecuteEstimationsInARow = 0;
       }
     }