From 3a3974211dbbf0404a6919a50012141f1a5f7d38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bla=C5=BE=20Hrastnik?= Date: Tue, 29 Oct 2024 01:38:38 +0900 Subject: [PATCH] CLO compat --- integration-tests/deployment/clo/env.go | 136 ----------- .../deployment/clo/offchain_client_impl.go | 21 +- .../deployment/keystone/deploy.go | 23 +- .../deployment/keystone/deploy_test.go | 222 ++++++++++++++++++ .../deployment/keystone/types.go | 4 +- 5 files changed, 262 insertions(+), 144 deletions(-) delete mode 100644 integration-tests/deployment/clo/env.go diff --git a/integration-tests/deployment/clo/env.go b/integration-tests/deployment/clo/env.go deleted file mode 100644 index 302f9794eaf..00000000000 --- a/integration-tests/deployment/clo/env.go +++ /dev/null @@ -1,136 +0,0 @@ -package clo - -import ( - "strconv" - "testing" - - "github.com/test-go/testify/require" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink/integration-tests/deployment" - "github.com/smartcontractkit/chainlink/integration-tests/deployment/clo/models" - "github.com/smartcontractkit/chainlink/integration-tests/deployment/memory" -) - -type DonEnvConfig struct { - DonName string - Chains map[uint64]deployment.Chain - Logger logger.Logger - Nops []*models.NodeOperator -} - -func NewDonEnv(t *testing.T, cfg DonEnvConfig) *deployment.Environment { - // no bootstraps in the don as far as capabilities registry is concerned - for _, nop := range cfg.Nops { - for _, node := range nop.Nodes { - for _, chain := range node.ChainConfigs { - if chain.Ocr2Config.IsBootstrap { - t.Fatalf("Don nodes should not be bootstraps nop %s node %s chain %s", nop.ID, node.ID, chain.Network.ChainID) - } - } - } - } - out := deployment.Environment{ - Name: cfg.DonName, - Offchain: NewJobClient(cfg.Logger, cfg.Nops), - NodeIDs: make([]string, 0), - Chains: cfg.Chains, - Logger: cfg.Logger, - } - // assume that all the nodes in the provided input nops are part of the don - for _, nop := range cfg.Nops { - for _, node := range nop.Nodes { - out.NodeIDs = append(out.NodeIDs, node.ID) - } - } - - return &out -} - -func NewDonEnvWithMemoryChains(t *testing.T, cfg DonEnvConfig, ignore func(*models.NodeChainConfig) bool) *deployment.Environment { - e := NewDonEnv(t, cfg) - // overwrite the chains with memory chains - chains := make(map[uint64]struct{}) - for _, nop := range cfg.Nops { - for _, node := range nop.Nodes { - for _, chain := range node.ChainConfigs { - if ignore(chain) { - continue - } - id, err := strconv.ParseUint(chain.Network.ChainID, 10, 64) - require.NoError(t, err, "failed to parse chain id to uint64") - chains[id] = struct{}{} - } - } - } - var cs []uint64 - for c := range chains { - cs = append(cs, c) - } - memoryChains := memory.NewMemoryChainsWithChainIDs(t, cs) - e.Chains = memoryChains - return e -} - -// MultiDonEnvironment is a single logical deployment environment (like dev, testnet, prod,...). -// It represents the idea that different nodesets host different capabilities. -// Each element in the DonEnv is a logical set of nodes that host the same capabilities. -// This model allows us to reuse the existing Environment abstraction while supporting multiple nodesets at -// expense of slightly abusing the original abstraction. Specifically, the abuse is that -// each Environment in the DonToEnv map is a subset of the target deployment environment. -// One element cannot represent dev and other testnet for example. -type MultiDonEnvironment struct { - donToEnv map[string]*deployment.Environment - Logger logger.Logger - // hacky but temporary to transition to Environment abstraction. set by New - Chains map[uint64]deployment.Chain -} - -func (mde MultiDonEnvironment) Flatten(name string) *deployment.Environment { - return &deployment.Environment{ - Name: name, - Chains: mde.Chains, - Logger: mde.Logger, - - // TODO: KS-460 integrate with the clo offchain client impl - // may need to extend the Environment abstraction use maps rather than slices for Nodes - // somehow we need to capture the fact that each nodes belong to nodesets which have different capabilities - // purposely nil to catch misuse until we do that work - Offchain: nil, - NodeIDs: nil, - } -} - -func newMultiDonEnvironment(logger logger.Logger, donToEnv map[string]*deployment.Environment) *MultiDonEnvironment { - chains := make(map[uint64]deployment.Chain) - for _, env := range donToEnv { - for sel, chain := range env.Chains { - if _, exists := chains[sel]; !exists { - chains[sel] = chain - } - } - } - return &MultiDonEnvironment{ - donToEnv: donToEnv, - Logger: logger, - Chains: chains, - } -} - -func NewTestEnv(t *testing.T, lggr logger.Logger, dons map[string]*deployment.Environment) *MultiDonEnvironment { - for _, don := range dons { - //don := don - seen := make(map[uint64]deployment.Chain) - // ensure that generated chains are the same for all environments. this ensures that he in memory representation - // points to a common object for all dons given the same selector. - for sel, chain := range don.Chains { - c, exists := seen[sel] - if exists { - don.Chains[sel] = c - } else { - seen[sel] = chain - } - } - } - return newMultiDonEnvironment(lggr, dons) -} diff --git a/integration-tests/deployment/clo/offchain_client_impl.go b/integration-tests/deployment/clo/offchain_client_impl.go index f59a918a41a..cebd23150f8 100644 --- a/integration-tests/deployment/clo/offchain_client_impl.go +++ b/integration-tests/deployment/clo/offchain_client_impl.go @@ -2,6 +2,7 @@ package clo import ( "context" + "fmt" "go.uber.org/zap" "google.golang.org/grpc" @@ -61,7 +62,7 @@ func (j JobClient) GetNode(ctx context.Context, in *nodev1.GetNodeRequest, opts func (j JobClient) ListNodes(ctx context.Context, in *nodev1.ListNodesRequest, opts ...grpc.CallOption) (*nodev1.ListNodesResponse, error) { //TODO CCIP-3108 - var fiterIds map[string]struct{} + fiterIds := make(map[string]any) include := func(id string) bool { if in.Filter == nil || len(in.Filter.Ids) == 0 { return true @@ -82,7 +83,7 @@ func (j JobClient) ListNodes(ctx context.Context, in *nodev1.ListNodesRequest, o nodes = append(nodes, &nodev1.Node{ Id: n.ID, Name: n.Name, - PublicKey: *n.PublicKey, // is this the correct val? + PublicKey: *n.PublicKey, IsEnabled: n.Enabled, IsConnected: n.Connected, }) @@ -184,10 +185,24 @@ func cloNodeToChainConfigs(n *models.Node) []*nodev1.ChainConfig { } func cloChainCfgToJDChainCfg(ccfg *models.NodeChainConfig) *nodev1.ChainConfig { + var ctype nodev1.ChainType + switch ccfg.Network.ChainType { + case models.ChainTypeEvm: + ctype = nodev1.ChainType_CHAIN_TYPE_EVM + case models.ChainTypeSolana: + ctype = nodev1.ChainType_CHAIN_TYPE_SOLANA + case models.ChainTypeStarknet: + ctype = nodev1.ChainType_CHAIN_TYPE_STARKNET + case models.ChainTypeAptos: + ctype = nodev1.ChainType_CHAIN_TYPE_APTOS + default: + panic(fmt.Sprintf("Unsupported chain family %v", ccfg.Network.ChainType)) + } + return &nodev1.ChainConfig{ Chain: &nodev1.Chain{ Id: ccfg.Network.ChainID, - Type: nodev1.ChainType_CHAIN_TYPE_EVM, // TODO: write conversion func from clo to jd tyes + Type: ctype, }, AccountAddress: ccfg.AccountAddress, AdminAddress: ccfg.AdminAddress, diff --git a/integration-tests/deployment/keystone/deploy.go b/integration-tests/deployment/keystone/deploy.go index 37240c1a48d..dde4910565e 100644 --- a/integration-tests/deployment/keystone/deploy.go +++ b/integration-tests/deployment/keystone/deploy.go @@ -156,6 +156,7 @@ type DonInfo struct { type Node struct { ID string + P2PID string Name string PublicKey *string ChainConfigs []*nodev1.ChainConfig @@ -166,7 +167,14 @@ func NodesFromJD(name string, nodeIDs []string, jd deployment.OffchainClient) ([ nodesFromJD, err := jd.ListNodes(context.Background(), &nodev1.ListNodesRequest{ Filter: &nodev1.ListNodesRequest_Filter{ Enabled: 1, - Ids: nodeIDs, + Ids: nodeIDs, // TODO: use p2p_id selectors instead of IDs + // Selectors: []*ptypes.Selector{ + // { + // Key: "p2p_id", + // Op: ptypes.SelectorOp_IN, + // Value: pointer.ToString(""), // TODO: + // }, + // }, }, }) if err != nil { @@ -184,11 +192,20 @@ func NodesFromJD(name string, nodeIDs []string, jd deployment.OffchainClient) ([ if idx < 0 { return nil, fmt.Errorf("node id not found") } + jdNode := nodesFromJD.Nodes[idx] + + // labelIdx := slices.IndexFunc(jdNode.GetLabels(), func(label *ptypes.Label) bool { return label.Key == "p2p_id" }) + // if labelIdx < 0 { + // return nil, fmt.Errorf("p2p_id label not found") + // } + // p2pID := *jdNode.Labels[labelIdx].Value nodes = append(nodes, Node{ - ID: nodeID, + ID: nodeID, + // P2PID: p2pID, // TODO: + P2PID: nodeID, Name: name, - PublicKey: &nodesFromJD.Nodes[idx].PublicKey, + PublicKey: &jdNode.PublicKey, // PublicKey TODO fetch via ListNodes ChainConfigs: nodeChainConfigs.GetChainConfigs(), }) diff --git a/integration-tests/deployment/keystone/deploy_test.go b/integration-tests/deployment/keystone/deploy_test.go index 7404d590052..7eccb9f0ec9 100644 --- a/integration-tests/deployment/keystone/deploy_test.go +++ b/integration-tests/deployment/keystone/deploy_test.go @@ -1,7 +1,10 @@ package keystone_test import ( + "encoding/json" "fmt" + "os" + "strconv" "testing" "github.com/ethereum/go-ethereum/accounts/abi/bind" @@ -14,6 +17,9 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + "github.com/smartcontractkit/chainlink/integration-tests/deployment/clo" + "github.com/smartcontractkit/chainlink/integration-tests/deployment/clo/models" + "github.com/smartcontractkit/chainlink/integration-tests/deployment" "github.com/smartcontractkit/chainlink/integration-tests/deployment/keystone" "github.com/smartcontractkit/chainlink/integration-tests/deployment/memory" @@ -182,3 +188,219 @@ func TestDeploy(t *testing.T) { require.NoError(t, err) } } + +// TODO: Deprecated, remove everything below that leverages CLO + +func nodeOperatorsToIDs(nops []*models.NodeOperator) (nodeIDs []string) { + for _, nop := range nops { + for _, node := range nop.Nodes { + nodeIDs = append(nodeIDs, node.ID) + } + } + return nodeIDs +} + +func TestDeployCLO(t *testing.T) { + lggr := logger.TestLogger(t) + + wfNops := loadTestNops(t, "testdata/workflow_nodes.json") + cwNops := loadTestNops(t, "testdata/chain_writer_nodes.json") + assetNops := loadTestNops(t, "testdata/asset_nodes.json") + require.Len(t, wfNops, 10) + requireChains(t, wfNops, []models.ChainType{models.ChainTypeEvm, models.ChainTypeAptos}) + require.Len(t, cwNops, 10) + requireChains(t, cwNops, []models.ChainType{models.ChainTypeEvm, models.ChainTypeEvm}) + require.Len(t, assetNops, 16) + requireChains(t, assetNops, []models.ChainType{models.ChainTypeEvm}) + + wfNodes := nodeOperatorsToIDs(wfNops) + cwNodes := nodeOperatorsToIDs(cwNops) + assetNodes := nodeOperatorsToIDs(assetNops) + + wfDon := keystone.DonCapabilities{ + Name: keystone.WFDonName, + Nodes: wfNodes, + Capabilities: []kcr.CapabilitiesRegistryCapability{keystone.OCR3Cap}, + } + cwDon := keystone.DonCapabilities{ + Name: keystone.TargetDonName, + Nodes: cwNodes, + Capabilities: []kcr.CapabilitiesRegistryCapability{keystone.WriteChainCap}, + } + assetDon := keystone.DonCapabilities{ + Name: keystone.StreamDonName, + Nodes: assetNodes, + Capabilities: []kcr.CapabilitiesRegistryCapability{keystone.StreamTriggerCap}, + } + + var allNops []*models.NodeOperator + allNops = append(allNops, wfNops...) + allNops = append(allNops, cwNops...) + allNops = append(allNops, assetNops...) + + // TODO: + // - replace the multi don code with something similar to above, a single evn with nodes being tagged + // - tag all the nodes in this new fake JD + + chains := make(map[uint64]struct{}) + for _, nop := range allNops { + for _, node := range nop.Nodes { + for _, chain := range node.ChainConfigs { + // chain selector lib doesn't support chain id 2 and we don't use it in tests + // because it's not an evm chain + if chain.Network.ChainID == "2" { // aptos chain + continue + } + id, err := strconv.ParseUint(chain.Network.ChainID, 10, 64) + require.NoError(t, err, "failed to parse chain id to uint64") + chains[id] = struct{}{} + } + } + } + var chainIDs []uint64 + for c := range chains { + chainIDs = append(chainIDs, c) + } + allChains := memory.NewMemoryChainsWithChainIDs(t, chainIDs) + + env := &deployment.Environment{ + Name: "CLO", + Offchain: clo.NewJobClient(lggr, allNops), + Chains: allChains, + Logger: lggr, + } + // assume that all the nodes in the provided input nops are part of the don + for _, nop := range allNops { + for _, node := range nop.Nodes { + env.NodeIDs = append(env.NodeIDs, node.ID) + } + } + + // sepolia; all nodes are on the this chain + registryChainSel, err := chainsel.SelectorFromChainId(11155111) + require.NoError(t, err) + + var ocr3Config = keystone.OracleConfigSource{ + MaxFaultyOracles: len(wfNops) / 3, + } + + ctx := tests.Context(t) + // explicitly deploy the contracts + cs, err := keystone.DeployContracts(lggr, env, registryChainSel) + require.NoError(t, err) + + deployReq := keystone.ConfigureContractsRequest{ + RegistryChainSel: registryChainSel, + Env: env, + OCR3Config: &ocr3Config, + Dons: []keystone.DonCapabilities{wfDon, cwDon, assetDon}, + AddressBook: cs.AddressBook, + DoContractDeploy: false, + } + deployResp, err := keystone.ConfigureContracts(ctx, lggr, deployReq) + require.NoError(t, err) + ad := deployResp.Changeset.AddressBook + addrs, err := ad.Addresses() + require.NoError(t, err) + lggr.Infow("Deployed Keystone contracts", "address book", addrs) + + // all contracts on home chain + homeChainAddrs, err := ad.AddressesForChain(registryChainSel) + require.NoError(t, err) + require.Len(t, homeChainAddrs, 3) + // only forwarder on non-home chain + for sel := range env.Chains { + chainAddrs, err := ad.AddressesForChain(sel) + require.NoError(t, err) + if sel != registryChainSel { + require.Len(t, chainAddrs, 1) + } else { + require.Len(t, chainAddrs, 3) + } + containsForwarder := false + for _, tv := range chainAddrs { + if tv.Type == keystone.KeystoneForwarder { + containsForwarder = true + break + } + } + require.True(t, containsForwarder, "no forwarder found in %v on chain %d for target don", chainAddrs, sel) + } + req := &keystone.GetContractSetsRequest{ + Chains: env.Chains, + AddressBook: ad, + } + + contractSetsResp, err := keystone.GetContractSets(req) + require.NoError(t, err) + require.Len(t, contractSetsResp.ContractSets, len(env.Chains)) + // check the registry + regChainContracts, ok := contractSetsResp.ContractSets[registryChainSel] + require.True(t, ok) + gotRegistry := regChainContracts.CapabilitiesRegistry + require.NotNil(t, gotRegistry) + // contract reads + gotDons, err := gotRegistry.GetDONs(&bind.CallOpts{}) + if err != nil { + err = keystone.DecodeErr(kcr.CapabilitiesRegistryABI, err) + require.Fail(t, fmt.Sprintf("failed to get Dons from registry at %s: %s", gotRegistry.Address().String(), err)) + } + require.NoError(t, err) + assert.Len(t, gotDons, len(deployReq.Dons)) + + for n, info := range deployResp.DonInfos { + found := false + for _, gdon := range gotDons { + if gdon.Id == info.Id { + found = true + assert.EqualValues(t, info, gdon) + break + } + } + require.True(t, found, "don %s not found in registry", n) + } + // check the forwarder + for _, cs := range contractSetsResp.ContractSets { + forwarder := cs.Forwarder + require.NotNil(t, forwarder) + // any read to ensure that the contract is deployed correctly + _, err := forwarder.Owner(&bind.CallOpts{}) + require.NoError(t, err) + // TODO expand this test; there is no get method on the forwarder so unclear how to test it + } + // check the ocr3 contract + for chainSel, cs := range contractSetsResp.ContractSets { + if chainSel != registryChainSel { + require.Nil(t, cs.OCR3) + continue + } + require.NotNil(t, cs.OCR3) + // any read to ensure that the contract is deployed correctly + _, err := cs.OCR3.LatestConfigDetails(&bind.CallOpts{}) + require.NoError(t, err) + } +} + +func requireChains(t *testing.T, donNops []*models.NodeOperator, cs []models.ChainType) { + got := make(map[models.ChainType]struct{}) + want := make(map[models.ChainType]struct{}) + for _, c := range cs { + want[c] = struct{}{} + } + for _, nop := range donNops { + for _, node := range nop.Nodes { + for _, cc := range node.ChainConfigs { + got[cc.Network.ChainType] = struct{}{} + } + } + require.EqualValues(t, want, got, "did not find all chains in node %s", nop.Name) + } +} + +func loadTestNops(t *testing.T, pth string) []*models.NodeOperator { + f, err := os.ReadFile(pth) + require.NoError(t, err) + var nops []*models.NodeOperator + require.NoError(t, json.Unmarshal(f, &nops)) + return nops +} diff --git a/integration-tests/deployment/keystone/types.go b/integration-tests/deployment/keystone/types.go index 9cd77eda76d..f3d107fd8a7 100644 --- a/integration-tests/deployment/keystone/types.go +++ b/integration-tests/deployment/keystone/types.go @@ -118,7 +118,7 @@ func newOcr2NodeFromClo(n *Node, registryChainSel uint64) (*ocr2Node, error) { if exists { cfgs[chaintype.Aptos] = aptosCC } - return newOcr2Node(n.ID, cfgs, *n.PublicKey) + return newOcr2Node(n.P2PID, cfgs, *n.PublicKey) } func newOcr2Node(id string, ccfgs map[chaintype.ChainType]*v1.ChainConfig, csaPubKey string) (*ocr2Node, error) { @@ -212,7 +212,7 @@ func (dc DonInfo) nodeIdToNop(cs uint64) (map[string]capabilities_registry.Capab //TODO validate chainType field if chain.Chain.Id == cidStr { found = true - out[node.ID] = capabilities_registry.CapabilitiesRegistryNodeOperator{ + out[node.P2PID] = capabilities_registry.CapabilitiesRegistryNodeOperator{ Name: node.Name, Admin: adminAddr(chain.AdminAddress), }