diff --git a/.changeset/giant-eels-jump.md b/.changeset/giant-eels-jump.md new file mode 100644 index 00000000000..5ab8ca875ca --- /dev/null +++ b/.changeset/giant-eels-jump.md @@ -0,0 +1,5 @@ +--- +"chainlink": patch +--- + +Add error handling for Arbitrum RPC server timeouts. #added diff --git a/core/chains/evm/client/errors.go b/core/chains/evm/client/errors.go index 1075dc40606..bde97185580 100644 --- a/core/chains/evm/client/errors.go +++ b/core/chains/evm/client/errors.go @@ -64,6 +64,7 @@ const ( ServiceUnavailable TerminallyStuck TooManyResults + ServiceTimeout ) type ClientErrors map[int]*regexp.Regexp @@ -160,7 +161,8 @@ var arbitrum = ClientErrors{ Fatal: arbitrumFatal, L2FeeTooLow: regexp.MustCompile(`(: |^)max fee per gas less than block base fee(:|$)`), L2Full: regexp.MustCompile(`(: |^)(queue full|sequencer pending tx pool full, please try again)(:|$)`), - ServiceUnavailable: regexp.MustCompile(`(: |^)502 Bad Gateway: [\s\S]*$|network is unreachable|i/o timeout`), + ServiceUnavailable: regexp.MustCompile(`(: |^)502 Bad Gateway: [\s\S]*$|network is unreachable|i/o timeout|(: |^)503 Service Temporarily Unavailable(:|$)`), + ServiceTimeout: regexp.MustCompile(`(: |^)408 Request Timeout(:|$)`), } // Treasure @@ -398,6 +400,11 @@ func (s *SendError) IsServiceUnavailable(configErrors *ClientErrors) bool { return s.is(ServiceUnavailable, configErrors) || pkgerrors.Is(s.err, commonclient.ErroringNodeError) } +// IsServiceTimeout indicates if the error was caused by a service timeout +func (s *SendError) IsServiceTimeout(configErrors *ClientErrors) bool { + return s.is(ServiceTimeout, configErrors) +} + // IsTerminallyStuck indicates if a transaction was stuck without any chance of inclusion func (s *SendError) IsTerminallyStuckConfigError(configErrors *ClientErrors) bool { return s.is(TerminallyStuck, configErrors) @@ -619,6 +626,10 @@ func ClassifySendError(err error, clientErrors config.ClientErrors, lggr logger. lggr.Errorw(fmt.Sprintf("service unavailable while sending transaction %x", tx.Hash()), "err", sendError, "etx", tx) return commonclient.Retryable } + if sendError.IsServiceTimeout(configErrors) { + lggr.Errorw(fmt.Sprintf("service timed out while sending transaction %x", tx.Hash()), "err", sendError, "etx", tx) + return commonclient.Retryable + } if sendError.IsTimeout() { lggr.Errorw(fmt.Sprintf("timeout while sending transaction %x", tx.Hash()), "err", sendError, "etx", tx) return commonclient.Retryable @@ -666,7 +677,7 @@ var drpc = ClientErrors{ // Linkpool, Blockdaemon, and Chainstack all return "request timed out" if the log results are too large for them to process var defaultClient = ClientErrors{ - TooManyResults: regexp.MustCompile(`request timed out`), + TooManyResults: regexp.MustCompile(`request timed out|408 Request Timed Out`), } // JSON-RPC error codes which can indicate a refusal of the server to process an eth_getLogs request because the result set is too large diff --git a/core/chains/evm/client/errors_test.go b/core/chains/evm/client/errors_test.go index 75ac21597d8..1f9aaa53365 100644 --- a/core/chains/evm/client/errors_test.go +++ b/core/chains/evm/client/errors_test.go @@ -245,6 +245,7 @@ func Test_Eth_Errors(t *testing.T) { {"network is unreachable", true, "Arbitrum"}, {"client error service unavailable", true, "tomlConfig"}, {"[Request ID: 825608a8-fd8a-4b5b-aea7-92999509306d] Error invoking RPC: [Request ID: 825608a8-fd8a-4b5b-aea7-92999509306d] Transaction execution returns a null value for transaction", true, "hedera"}, + {"call failed: 503 Service Temporarily Unavailable: \r\n503 Service Temporarily Unavailable\r\n\r\n

503 Service Temporarily Unavailable

\r\n\r\n\r\n", true, "Arbitrum"}, } for _, test := range tests { err = evmclient.NewSendErrorS(test.message) @@ -260,6 +261,20 @@ func Test_Eth_Errors(t *testing.T) { } }) + t.Run("IsServiceTimeout", func(t *testing.T) { + tests := []errorCase{ + {"call failed: 408 Request Timeout: {", true, "Arbitrum"}, + {"408 Request Timeout: {\"id\":303,\"jsonrpc\":\"2.0\",\"error\":{\"code\\\":-32009,\\\"message\\\":\\\"request timeout\\\"}}\",\"errVerbose\":\"408 Request Timeout:\n", true, "Arbitrum"}, + {"request timeout", false, "tomlConfig"}, + } + for _, test := range tests { + err = evmclient.NewSendErrorS(test.message) + assert.Equal(t, err.IsServiceTimeout(clientErrors), test.expect) + err = newSendErrorWrapped(test.message) + assert.Equal(t, err.IsServiceTimeout(clientErrors), test.expect) + } + }) + t.Run("IsTxFeeExceedsCap", func(t *testing.T) { tests := []errorCase{ {"tx fee (1.10 ether) exceeds the configured cap (1.00 ether)", true, "geth"},