Merge develop and resolve conflicts

.changeset/light-trains-chew.md

@@ -0,0 +1,5 @@
+---
+"chainlink": minor
+---
+
+Refactor chain ID logic in plugin to be chain agnostic #added

.changeset/mean-knives-knock.md

@@ -0,0 +1,5 @@
+---
+"chainlink": minor
+---
+
+#updated chainconfig: show chain type next to key bundle id in UI

.changeset/silver-avocados-buy.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+Update MultiNode with latest changes and bug fixes. Fixes an issue that caused nodes to go OutOfSync incorrectly, and also fixed context handling for sending transactions. #internal #bugfix

.github/actions/goreleaser-build-sign-publish/action.yml

@@ -45,6 +45,10 @@ runs:
       uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.0
       with:
         buildkitd-flags: ${{ inputs.enable-debug == 'true' && '--debug' || '' }}
+        # v0.16.0 until grpc fix is released
+        # see: https://github.com/docker/buildx/issues/2789#issuecomment-2487981922
+        driver-opts: |
+          image=moby/buildkit:v0.16.0
 
     - name: Set up Go
       uses: ./.github/actions/setup-go

.github/e2e-tests.yml

@@ -941,7 +941,6 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     triggers:
       - PR E2E Core Tests
-      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke/ccip && go test ccip_test.go -timeout 12m -test.parallel=2 -count=1 -json
     pyroscope_env: ci-smoke-ccipv1_6-evm-simulated
@@ -955,7 +954,6 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     triggers:
       - PR E2E Core Tests
-      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke/ccip && go test ccip_messaging_test.go -timeout 15m -test.parallel=1 -count=1 -json
     pyroscope_env: ci-smoke-ccipv1_6-evm-simulated
@@ -969,7 +967,6 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     triggers:
       - PR E2E Core Tests
-      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke/ccip && go test ccip_usdc_test.go -timeout 18m -test.parallel=1 -count=1 -json
     pyroscope_env: ci-smoke-ccipv1_6-evm-simulated
@@ -983,7 +980,6 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     triggers:
       - PR E2E Core Tests
-      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke/ccip && go test fee_boosting_test.go -timeout 15m -test.parallel=1 -count=1 -json
     pyroscope_env: ci-smoke-ccipv1_6-evm-simulated
@@ -997,7 +993,6 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     triggers:
       - PR E2E Core Tests
-      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke/ccip && go test -test.run ^TestRMN_TwoMessagesOnTwoLanesIncludingBatching$ -timeout 12m -test.parallel=1 -count=1 -json
     pyroscope_env: ci-smoke-ccipv1_6-evm-simulated
@@ -1013,7 +1008,6 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     triggers:
       - PR E2E Core Tests
-      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke/ccip && go test -test.run ^TestRMN_MultipleMessagesOnOneLaneNoWaitForExec$ -timeout 12m -test.parallel=1 -count=1 -json
     pyroscope_env: ci-smoke-ccipv1_6-evm-simulated
@@ -1030,7 +1024,6 @@ runner-test-matrix:
 #    runs_on: ubuntu-latest
 #    triggers:
 #      - PR E2E Core Tests
-#      - Merge Queue E2E Core Tests
 #      - Nightly E2E Tests
 #    test_cmd: cd integration-tests/smoke/ccip && go test -test.run ^TestRMN_NotEnoughObservers$ -timeout 12m -test.parallel=1 -count=1 -json
 #    pyroscope_env: ci-smoke-ccipv1_6-evm-simulated
@@ -1046,7 +1039,6 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     triggers:
       - PR E2E Core Tests
-      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke/ccip && go test -test.run ^TestRMN_DifferentSigners$ -timeout 12m -test.parallel=1 -count=1 -json
     pyroscope_env: ci-smoke-ccipv1_6-evm-simulated
@@ -1063,7 +1055,6 @@ runner-test-matrix:
 #    runs_on: ubuntu-latest
 #    triggers:
 #      - PR E2E Core Tests
-#      - Merge Queue E2E Core Tests
 #      - Nightly E2E Tests
 #    test_cmd: cd integration-tests/smoke/ccip && go test -test.run ^TestRMN_NotEnoughSigners$ -timeout 12m -test.parallel=1 -count=1 -json
 #    pyroscope_env: ci-smoke-ccipv1_6-evm-simulated
@@ -1080,7 +1071,6 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     triggers:
       - PR E2E Core Tests
-      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke/ccip && go test -test.run ^TestRMN_DifferentRmnNodesForDifferentChains$ -timeout 12m -test.parallel=1 -count=1 -json
     pyroscope_env: ci-smoke-ccipv1_6-evm-simulated

.github/workflows/solidity.yml

@@ -76,27 +76,41 @@ jobs:
     steps:
       - name: Checkout the repo
         uses: actions/[email protected]
+        with:
+          path: chainlink
+
       - name: Checkout diff-so-fancy
         uses: actions/[email protected]
         with:
           repository: so-fancy/diff-so-fancy
           ref: a673cb4d2707f64d92b86498a2f5f71c8e2643d5 # v1.4.3
           path: diff-so-fancy
+
       - name: Install diff-so-fancy
         run: echo "$GITHUB_WORKSPACE/diff-so-fancy" >> $GITHUB_PATH
+
       - name: Setup NodeJS
-        uses: ./.github/actions/setup-nodejs
+        uses: ./chainlink/.github/actions/setup-nodejs
         with:
+          base-path: "chainlink"
           prod: "true"
+
       - name: Setup Go
-        uses: ./.github/actions/setup-go
+        uses: ./chainlink/.github/actions/setup-go
+        with:
+          go-version-file: "chainlink/go.mod"
+
       - name: Run native compile and generate wrappers
+        working-directory: ./chainlink/contracts
         run: make wrappers-all
-        working-directory: ./contracts
+
       - name: Verify local solc binaries
+        working-directory: chainlink
         run: ./tools/ci/check_solc_hashes
+
       - name: Check if Go solidity wrappers are updated
         if: ${{ needs.changes.outputs.changes == 'true' }}
+        working-directory: chainlink
         run: |
           git add --all
           git diff --minimal --color --cached --exit-code | diff-so-fancy

.mockery.yaml

@@ -579,6 +579,21 @@ packages:
   github.com/smartcontractkit/chainlink/v2/core/services/registrysyncer:
     interfaces:
       ORM:
+  github.com/smartcontractkit/chainlink/v2/core/services/workflows/syncer:
+    interfaces:
+      ORM:
+      ContractReader:
+        config:
+          mockname: "Mock{{ .InterfaceName }}"
+          filename: contract_reader_mock.go
+          inpackage: true
+          dir: "{{ .InterfaceDir }}"
+      Handler:
+        config:
+          mockname: "Mock{{ .InterfaceName }}"
+          filename: handler_mock.go
+          inpackage: true
+          dir: "{{ .InterfaceDir }}"
   github.com/smartcontractkit/chainlink/v2/core/capabilities/targets:
     interfaces:
       ContractValueGetter:

common/client/node_lifecycle.go

@@ -7,8 +7,6 @@ import (
 	"math/big"
 	"time"
 
-	"github.com/smartcontractkit/chainlink/v2/common/types"
-
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 
@@ -17,6 +15,7 @@ import (
 	bigmath "github.com/smartcontractkit/chainlink-common/pkg/utils/big_math"
 
 	iutils "github.com/smartcontractkit/chainlink/v2/common/internal/utils"
+	"github.com/smartcontractkit/chainlink/v2/common/types"
 )
 
 var (
@@ -132,6 +131,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() {
 		}
 	}
 
+	// Get the latest chain info to use as local highest
 	localHighestChainInfo, _ := n.rpc.GetInterceptedChainInfo()
 	var pollFailures uint32
 
@@ -168,10 +168,8 @@ func (n *node[CHAIN_ID, HEAD, RPC]) aliveLoop() {
 				n.declareUnreachable()
 				return
 			}
-			_, latestChainInfo := n.StateAndLatest()
-			if outOfSync, liveNodes := n.isOutOfSyncWithPool(latestChainInfo); outOfSync {
+			if outOfSync, liveNodes := n.isOutOfSyncWithPool(); outOfSync {
 				// note: there must be another live node for us to be out of sync
-				lggr.Errorw("RPC endpoint has fallen behind", "blockNumber", latestChainInfo.BlockNumber, "totalDifficulty", latestChainInfo.TotalDifficulty, "nodeState", n.getCachedState())
 				if liveNodes < 2 {
 					lggr.Criticalf("RPC endpoint has fallen behind; %s %s", msgCannotDisable, msgDegradedState)
 					continue
@@ -310,9 +308,9 @@ func (n *node[CHAIN_ID, HEAD, RPC]) onNewFinalizedHead(lggr logger.SugaredLogger
 	}
 
 	latestFinalizedBN := latestFinalized.BlockNumber()
-	lggr.Tracew("Got latest finalized head", "latestFinalized", latestFinalized)
+	lggr.Debugw("Got latest finalized head", "latestFinalized", latestFinalized)
 	if latestFinalizedBN <= chainInfo.FinalizedBlockNumber {
-		lggr.Tracew("Ignoring previously seen finalized block number")
+		lggr.Debugw("Ignoring previously seen finalized block number")
 		return false
 	}
 
@@ -328,10 +326,10 @@ func (n *node[CHAIN_ID, HEAD, RPC]) onNewHead(lggr logger.SugaredLogger, chainIn
 	}
 
 	promPoolRPCNodeNumSeenBlocks.WithLabelValues(n.chainID.String(), n.name).Inc()
-	lggr.Tracew("Got head", "head", head)
+	lggr.Debugw("Got head", "head", head)
 	lggr = lggr.With("latestReceivedBlockNumber", chainInfo.BlockNumber, "blockNumber", head.BlockNumber(), "nodeState", n.getCachedState())
 	if head.BlockNumber() <= chainInfo.BlockNumber {
-		lggr.Tracew("Ignoring previously seen block number")
+		lggr.Debugw("Ignoring previously seen block number")
 		return false
 	}
 
@@ -358,7 +356,7 @@ const (
 // isOutOfSyncWithPool returns outOfSync true if num or td is more than SyncThresold behind the best node.
 // Always returns outOfSync false for SyncThreshold 0.
 // liveNodes is only included when outOfSync is true.
-func (n *node[CHAIN_ID, HEAD, RPC]) isOutOfSyncWithPool(localState ChainInfo) (outOfSync bool, liveNodes int) {
+func (n *node[CHAIN_ID, HEAD, RPC]) isOutOfSyncWithPool() (outOfSync bool, liveNodes int) {
 	if n.poolInfoProvider == nil {
 		n.lfcLog.Warn("skipping sync state against the pool - should only occur in tests")
 		return // skip for tests
@@ -369,16 +367,22 @@ func (n *node[CHAIN_ID, HEAD, RPC]) isOutOfSyncWithPool(localState ChainInfo) (o
 	}
 	// Check against best node
 	ln, ci := n.poolInfoProvider.LatestChainInfo()
+	localChainInfo, _ := n.rpc.GetInterceptedChainInfo()
 	mode := n.nodePoolCfg.SelectionMode()
 	switch mode {
 	case NodeSelectionModeHighestHead, NodeSelectionModeRoundRobin, NodeSelectionModePriorityLevel:
-		return localState.BlockNumber < ci.BlockNumber-int64(threshold), ln
+		outOfSync = localChainInfo.BlockNumber < ci.BlockNumber-int64(threshold)
 	case NodeSelectionModeTotalDifficulty:
 		bigThreshold := big.NewInt(int64(threshold))
-		return localState.TotalDifficulty.Cmp(bigmath.Sub(ci.TotalDifficulty, bigThreshold)) < 0, ln
+		outOfSync = localChainInfo.TotalDifficulty.Cmp(bigmath.Sub(ci.TotalDifficulty, bigThreshold)) < 0
 	default:
 		panic("unrecognized NodeSelectionMode: " + mode)
 	}
+
+	if outOfSync && n.getCachedState() == nodeStateAlive {
+		n.lfcLog.Errorw("RPC endpoint has fallen behind", "blockNumber", localChainInfo.BlockNumber, "bestLatestBlockNumber", ci.BlockNumber, "totalDifficulty", localChainInfo.TotalDifficulty)
+	}
+	return outOfSync, ln
 }
 
 // outOfSyncLoop takes an OutOfSync node and waits until isOutOfSync returns false to go back to live status
@@ -464,7 +468,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) outOfSyncLoop(syncIssues syncStatus) {
 
 			// received a new head - clear NoNewHead flag
 			syncIssues &= ^syncStatusNoNewHead
-			if outOfSync, _ := n.isOutOfSyncWithPool(localHighestChainInfo); !outOfSync {
+			if outOfSync, _ := n.isOutOfSyncWithPool(); !outOfSync {
 				// we caught up with the pool - clear NotInSyncWithPool flag
 				syncIssues &= ^syncStatusNotInSyncWithPool
 			} else {
@@ -515,7 +519,12 @@ func (n *node[CHAIN_ID, HEAD, RPC]) outOfSyncLoop(syncIssues syncStatus) {
 				finalizedHeadsSub.ResetTimer(noNewFinalizedBlocksTimeoutThreshold)
 			}
 
-			lggr.Debugw(msgReceivedFinalizedBlock, "blockNumber", latestFinalized.BlockNumber(), "syncIssues", syncIssues)
+			var highestSeen ChainInfo
+			if n.poolInfoProvider != nil {
+				highestSeen = n.poolInfoProvider.HighestUserObservations()
+			}
+
+			lggr.Debugw(msgReceivedFinalizedBlock, "blockNumber", latestFinalized.BlockNumber(), "poolHighestBlockNumber", highestSeen.FinalizedBlockNumber, "syncIssues", syncIssues)
 		case err := <-finalizedHeadsSub.Errors:
 			lggr.Errorw("Finalized head subscription was terminated", "err", err)
 			n.declareUnreachable()
@@ -648,7 +657,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) syncingLoop() {
 		case nodeStateClosed:
 			return
 		default:
-			panic(fmt.Sprintf("syncingLoop can only run for node in nodeStateSyncing state, got: %s", state))
+			panic(fmt.Sprintf("syncingLoop can only run for node in NodeStateSyncing state, got: %s", state))
 		}
 	}