Log failed transactions into beholder (smartcontractkit#15187)

* Logs failed transactions into beholder

* Adds log when failing to `GetTransactionStatus`

* Fixes tests

* Sets emit labels

* Logs unexpected transaction statuses

core/capabilities/targets/write_target.go

@@ -7,21 +7,27 @@ import (
 	"encoding/hex"
 	"fmt"
 	"math/big"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/google/uuid"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/capabilities"
 	"github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3/types"
+	"github.com/smartcontractkit/chainlink-common/pkg/custmsg"
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
 	commontypes "github.com/smartcontractkit/chainlink-common/pkg/types"
 	"github.com/smartcontractkit/chainlink-common/pkg/types/query/primitives"
+
+	"github.com/smartcontractkit/chainlink/v2/core/platform"
 )
 
 var (
 	_ capabilities.TargetCapability = &WriteTarget{}
 )
 
+const transactionStatusCheckInterval = 2 * time.Second
+
 type WriteTarget struct {
 	cr               ContractValueGetter
 	cw               commontypes.ChainWriter
@@ -31,7 +37,8 @@ type WriteTarget struct {
 	receiverGasMinimum uint64
 	capabilities.CapabilityInfo
 
-	lggr logger.Logger
+	emitter custmsg.MessageEmitter
+	lggr    logger.Logger
 
 	bound bool
 }
@@ -79,6 +86,7 @@ func NewWriteTarget(
 		forwarderAddress,
 		txGasLimit - ForwarderContractLogicGasCost,
 		info,
+		custmsg.NewLabeler(),
 		logger.Named(lggr, "WriteTarget"),
 		false,
 	}
@@ -309,7 +317,41 @@ func (cap *WriteTarget) Execute(ctx context.Context, rawRequest capabilities.Cap
 	}
 
 	cap.lggr.Debugw("Transaction submitted", "request", request, "transaction", txID)
-	return capabilities.CapabilityResponse{}, nil
+
+	tick := time.NewTicker(transactionStatusCheckInterval)
+	defer tick.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return capabilities.CapabilityResponse{}, nil
+		case <-tick.C:
+			txStatus, err := cap.cw.GetTransactionStatus(ctx, txID.String())
+			if err != nil {
+				cap.lggr.Errorw("Failed to get transaction status", "request", request, "transaction", txID, "err", err)
+				continue
+			}
+			switch txStatus {
+			case commontypes.Finalized:
+				cap.lggr.Debugw("Transaction finalized", "request", request, "transaction", txID)
+				return capabilities.CapabilityResponse{}, nil
+			case commontypes.Failed, commontypes.Fatal:
+				cap.lggr.Error("Transaction failed", "request", request, "transaction", txID)
+				msg := "failed to submit transaction with ID: " + txID.String()
+				err = cap.emitter.With(
+					platform.KeyWorkflowID, request.Metadata.WorkflowID,
+					platform.KeyWorkflowName, request.Metadata.WorkflowName,
+					platform.KeyWorkflowOwner, request.Metadata.WorkflowOwner,
+					platform.KeyWorkflowExecutionID, request.Metadata.WorkflowExecutionID,
+				).Emit(ctx, msg)
+				if err != nil {
+					cap.lggr.Errorf("failed to send custom message with msg: %s, err: %v", msg, err)
+				}
+				return capabilities.CapabilityResponse{}, fmt.Errorf("submitted transaction failed: %w", err)
+			default:
+				cap.lggr.Debugw("Unexpected transaction status", "request", request, "transaction", txID, "status", txStatus)
+			}
+		}
+	}
 }
 
 func (cap *WriteTarget) RegisterToWorkflow(ctx context.Context, request capabilities.RegisterToWorkflowRequest) error {

core/capabilities/targets/write_target_test.go

@@ -101,6 +101,7 @@ func TestWriteTarget(t *testing.T) {
 			Config:   config,
 			Inputs:   validInputs,
 		}
+		cw.On("GetTransactionStatus", mock.Anything, mock.Anything).Return(types.Finalized, nil).Once()
 
 		response, err2 := writeTarget.Execute(ctx, req)
 		require.NoError(t, err2)

core/services/relay/evm/write_target_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/capabilities"
+	commonTypes "github.com/smartcontractkit/chainlink-common/pkg/types"
 	"github.com/smartcontractkit/chainlink-common/pkg/values"
 
 	"github.com/smartcontractkit/chainlink/v2/common/headtracker/mocks"
@@ -110,6 +111,8 @@ func TestEvmWrite(t *testing.T) {
 	evmClient.On("CallContract", mock.Anything, mock.Anything, mock.Anything).Return(mockCall, nil).Maybe()
 	evmClient.On("CodeAt", mock.Anything, mock.Anything, mock.Anything).Return([]byte("test"), nil)
 
+	txManager.On("GetTransactionStatus", mock.Anything, mock.Anything).Return(commonTypes.Finalized, nil)
+
 	chain.On("ID").Return(big.NewInt(11155111))
 	chain.On("TxManager").Return(txManager)
 	chain.On("LogPoller").Return(poller)