diff --git a/build/Dockerfile.debian.dev b/build/Dockerfile.debian.dev index 387ef70ca..6659f579b 100644 --- a/build/Dockerfile.debian.dev +++ b/build/Dockerfile.debian.dev @@ -54,7 +54,6 @@ RUN make protogen_local && \ RUN go get -d -v ./app/pocket RUN go build -o /usr/local/bin/pocket ./app/pocket RUN go build -tags=debug -o /usr/local/bin/p1 ./app/client - RUN go build -o /usr/local/bin/cluster-manager ./build/localnet/cluster-manager CMD ["/usr/local/bin/pocket"] diff --git a/build/config/config.validator1.json b/build/config/config.validator1.json index 6031f1a37..216b2b512 100644 --- a/build/config/config.validator1.json +++ b/build/config/config.validator1.json @@ -56,13 +56,14 @@ }, "servicer": { "enabled": true, + "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e", "chains": ["0001"] }, "ibc": { "enabled": true, "stores_dir": "/var/ibc", "host": { - "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e" + "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e" } } } diff --git a/build/localnet/Tiltfile b/build/localnet/Tiltfile index d4534df35..abca365da 100644 --- a/build/localnet/Tiltfile +++ b/build/localnet/Tiltfile @@ -48,7 +48,7 @@ deps = [ "build/debug.go", "consensus", "p2p", - "persistance", + "persistence", "rpc", "runtime", "shared", @@ -56,6 +56,11 @@ deps = [ "utility", "vendor", "logger", + "e2e", + "ibc", + "internal", + "state_machine", + "tools", ] deps_full_path = [root_dir + "/" + depdir for depdir in deps] diff --git a/build/localnet/cluster-manager/main.go b/build/localnet/cluster-manager/main.go index 8fb1004fd..e443537df 100644 --- a/build/localnet/cluster-manager/main.go +++ b/build/localnet/cluster-manager/main.go @@ -57,7 +57,8 @@ func init() { clusterManagerCmd.PersistentFlags().StringVar( &flags.RemoteCLIURL, "remote_cli_url", - defaults.Validator1EndpointK8SHostname, + // defaults.Validator1EndpointK8SHostname, + defaults.FullNode1EndpointK8SHostname, "takes a remote endpoint in the form of ://: (uses RPC Port)", ) diff --git a/charts/pocket/README.md b/charts/pocket/README.md index bbac55a3d..f1b61fd69 100644 --- a/charts/pocket/README.md +++ b/charts/pocket/README.md @@ -43,6 +43,7 @@ privateKeySecretKeyRef: | config.consensus.pacemaker_config.manual | bool | `true` | | | config.consensus.pacemaker_config.timeout_msec | int | `10000` | | | config.consensus.private_key | string | `""` | | +| config.consensus.server_mode_enabled | bool | `true` | | | config.fisherman.enabled | bool | `false` | | | config.ibc.enabled | bool | `true` | | | config.ibc.host.private_key | string | `""` | | diff --git a/charts/pocket/values.yaml b/charts/pocket/values.yaml index 9587814c7..2a8b459ec 100644 --- a/charts/pocket/values.yaml +++ b/charts/pocket/values.yaml @@ -80,6 +80,7 @@ config: manual: true debug_time_between_steps_msec: 1000 private_key: "" # @ignored This value is needed but ignored - use privateKeySecretKeyRef instead + server_mode_enabled: true utility: max_mempool_transaction_bytes: 1073741824 max_mempool_transactions: 9000 diff --git a/consensus/block.go b/consensus/block.go index cf0038fad..5a3750b4c 100644 --- a/consensus/block.go +++ b/consensus/block.go @@ -34,9 +34,12 @@ func (m *consensusModule) commitBlock(block *coreTypes.Block) error { return nil } +// isBlockInMessageValidBasic does basic validation of the block in the hotstuff message such as: +// - validating if the block could/should be nil +// - the state hash of the block +// - the size of the block // ADDTEST: Add unit tests specific to block validation -// IMPROVE: Rename to provide clarity of operation. ValidateBasic() is typically a stateless check not stateful -func (m *consensusModule) isValidMessageBlock(msg *typesCons.HotstuffMessage) (bool, error) { +func (m *consensusModule) isBlockInMessageValidBasic(msg *typesCons.HotstuffMessage) (bool, error) { block := msg.GetBlock() step := msg.GetStep() @@ -73,12 +76,17 @@ func (m *consensusModule) isValidMessageBlock(msg *typesCons.HotstuffMessage) (b return true, nil } -// Creates a new Utility Unit Of Work and clears/nullifies any previous UOW if they exist +// refreshUtilityUnitOfWork is a helper that creates a new Utility Unit Of Work and clears/nullifies a previous one if it exists func (m *consensusModule) refreshUtilityUnitOfWork() error { + // m.m.Lock() + // defer m.m.Unlock() + // Catch-all structure to release the previous utility UOW if it wasn't properly cleaned up. utilityUnitOfWork := m.utilityUnitOfWork + + // TECHDEBT: This should, theoretically, never happen. Need to identify all + // code paths where it does and fix it. if utilityUnitOfWork != nil { - // TODO: This should, ideally, never be called m.logger.Warn().Bool("TODO", true).Msg(typesCons.NilUtilityUOWWarning) if err := utilityUnitOfWork.Release(); err != nil { m.logger.Warn().Err(err).Msg("failed to release utility unit of work") diff --git a/consensus/debugging.go b/consensus/debugging.go index d7ae18134..39b34ec92 100644 --- a/consensus/debugging.go +++ b/consensus/debugging.go @@ -2,7 +2,6 @@ package consensus import ( typesCons "github.com/pokt-network/pocket/consensus/types" - cryptoPocket "github.com/pokt-network/pocket/shared/crypto" "github.com/pokt-network/pocket/shared/messaging" "google.golang.org/protobuf/types/known/anypb" ) @@ -83,74 +82,43 @@ func (m *consensusModule) togglePacemakerManualMode(_ *messaging.DebugMessage) { m.paceMaker.SetManualMode(newMode) } -// requests current block from all validators +// sendGetBlockStateSyncMessage sends a messages to request specific blocks from peers func (m *consensusModule) sendGetBlockStateSyncMessage(_ *messaging.DebugMessage) { - currentHeight := m.CurrentHeight() - requestHeight := currentHeight - 1 - peerAddress := m.GetNodeAddress() - stateSyncGetBlockMessage := &typesCons.StateSyncMessage{ Message: &typesCons.StateSyncMessage_GetBlockReq{ GetBlockReq: &typesCons.GetBlockRequest{ - PeerAddress: peerAddress, - Height: requestHeight, + PeerAddress: m.GetNodeAddress(), + Height: m.CurrentHeight() - 1, }, }, } - - validators, err := m.getValidatorsAtHeight(currentHeight) + anyMsg, err := anypb.New(stateSyncGetBlockMessage) if err != nil { - m.logger.Debug().Msgf(typesCons.ErrPersistenceGetAllValidators.Error(), err) + m.logger.Error().Err(err).Str("proto_type", "GetBlockRequest").Msg("failed to create StateSyncGetBlockMessage") + return } - - for _, val := range validators { - if m.GetNodeAddress() == val.GetAddress() { - continue - } - valAddress := cryptoPocket.AddressFromString(val.GetAddress()) - - anyMsg, err := anypb.New(stateSyncGetBlockMessage) - if err != nil { - m.logger.Error().Err(err).Str("proto_type", "GetBlockRequest").Msg("failed to send StateSyncMessage") - } - - if err := m.GetBus().GetP2PModule().Send(valAddress, anyMsg); err != nil { - m.logger.Error().Err(err).Msg(typesCons.ErrSendMessage.Error()) - } + if err := m.GetBus().GetP2PModule().Broadcast(anyMsg); err != nil { + m.logger.Error().Err(err).Msg(typesCons.ErrBroadcastMessage.Error()) + return } } -// requests metadata from all validators +// sendGetMetadataStateSyncMessage sends a message to request metadata from their peers func (m *consensusModule) sendGetMetadataStateSyncMessage(_ *messaging.DebugMessage) { - currentHeight := m.CurrentHeight() - peerAddress := m.GetNodeAddress() - stateSyncMetaDataReqMessage := &typesCons.StateSyncMessage{ Message: &typesCons.StateSyncMessage_MetadataReq{ MetadataReq: &typesCons.StateSyncMetadataRequest{ - PeerAddress: peerAddress, + PeerAddress: m.GetNodeAddress(), }, }, } - - validators, err := m.getValidatorsAtHeight(currentHeight) + anyMsg, err := anypb.New(stateSyncMetaDataReqMessage) if err != nil { - m.logger.Debug().Msgf(typesCons.ErrPersistenceGetAllValidators.Error(), err) + m.logger.Error().Err(err).Str("proto_type", "StateSyncMessage").Msg("failed to create StateSyncMetadataRequest") + return } - - for _, val := range validators { - if m.GetNodeAddress() == val.GetAddress() { - continue - } - valAddress := cryptoPocket.AddressFromString(val.GetAddress()) - - anyMsg, err := anypb.New(stateSyncMetaDataReqMessage) - if err != nil { - m.logger.Error().Err(err).Str("proto_type", "GetMetadataRequest").Msg("failed to send StateSyncMessage") - } - - if err := m.GetBus().GetP2PModule().Send(valAddress, anyMsg); err != nil { - m.logger.Error().Err(err).Msg(typesCons.ErrSendMessage.Error()) - } + if m.GetBus().GetP2PModule().Broadcast(anyMsg) != nil { + m.logger.Error().Err(err).Msg(typesCons.ErrBroadcastMessage.Error()) + return } } diff --git a/consensus/doc/PROTOCOL_STATE_SYNC.md b/consensus/doc/PROTOCOL_STATE_SYNC.md index 955600cf8..af8a76e01 100644 --- a/consensus/doc/PROTOCOL_STATE_SYNC.md +++ b/consensus/doc/PROTOCOL_STATE_SYNC.md @@ -1,6 +1,13 @@ # State Sync Protocol Design -_NOTE: This document makes some assumption of P2P implementation details, so please see [p2p](../../p2p/README.md) for the latest source of truth._ +⚠️ IMPORTANT NOTES TO THE (last updated on 08/03/2023): + +- TECHDEBT(#821): Once the FSM is remove, state sync will look completely different +- State Sync implementation is a WIP and has taken several different shapes. +- This document is out of date and needs to be updated to reflect the latest implementation. This will be done once a functional implementation is in place. +- This document makes some assumption of P2P implementation details, so please see [p2p](../../p2p/README.md) for the latest source of truth. + +## Table of Contents - [Background](#background) - [State Sync - Peer Metadata](#state-sync---peer-metadata) @@ -55,9 +62,9 @@ Node gathers peer metadata from its peers in `StateSyncMetadataResponse` type, d ```golang type StateSyncMetadataResponse struct { - PeerAddress string - MinHeight uint64 - MaxHeight uint64 + PeerAddress string + MinHeight uint64 + MaxHeight uint64 } ``` @@ -87,7 +94,7 @@ type StateSyncModule interface { // ... GetAggregatedStateSyncMetadata() *StateSyncMetadataResponse // Aggregated metadata received from peers. IsSynced() (bool, error) - StartSyncing() error + Start() error // ... } ``` @@ -105,7 +112,7 @@ For every new block and block proposal `Validator`s receive: According to the result of the `IsSynced()` function: -- If the node is out of sync, it runs `StartSyncing()` function. Node requests blocks one by one using the minimum and maximum height in aggregated state sync metadata. +- If the node is out of sync, it runs `Start()` function. Node requests blocks one by one using the minimum and maximum height in aggregated state sync metadata. - If the node is in sync with its peers it rejects the block and/or block proposal. ```mermaid @@ -114,13 +121,13 @@ flowchart TD A[Node] --> B[Periodic
Sync] A[Node] --> |New Block| C{IsSynced} - %% periodic snyc + %% periodic sync B --> |Request
metadata| D[Peers] D[Peers] --> |Collect metadata| B[Periodic
Sync] - %% is node sycnhed - C --> |No| E[StartSyncing] + %% is node synched + C --> |No| E[Start] C --> |Yes| F[Apply Block] %% syncing @@ -154,7 +161,7 @@ In `Unsynced` Mode, node transitions to `Sync Mode` by sending `Consensus_IsSync ### Sync Mode -In `Sync` Mode, the Node is catching up to the latest block by making `GetBlock` requests, via `StartSyncing()` function to eligible peers in its address book. A peer can handle a `GetBlock` request if `PeerSyncMetadata.MinHeight` <= `localSyncState.MaxHeight` <= `PeerSyncMetadata.MaxHeight`. +In `Sync` Mode, the Node is catching up to the latest block by making `GetBlock` requests, via `Start()` function to eligible peers in its address book. A peer can handle a `GetBlock` request if `PeerSyncMetadata.MinHeight` <= `localSyncState.MaxHeight` <= `PeerSyncMetadata.MaxHeight`. Though it is unspecified whether or not a Node may make `GetBlock` requests in order or in parallel, the cryptographic restraints of block processing require the Node to call `CommitBlock` sequentially until it is `Synced`. diff --git a/consensus/e2e_tests/hotstuff_test.go b/consensus/e2e_tests/hotstuff_test.go index f2f314853..10e35939b 100644 --- a/consensus/e2e_tests/hotstuff_test.go +++ b/consensus/e2e_tests/hotstuff_test.go @@ -1,6 +1,7 @@ package e2e_tests import ( + "reflect" "testing" "time" @@ -10,36 +11,29 @@ import ( "github.com/pokt-network/pocket/shared/codec" "github.com/pokt-network/pocket/shared/modules" "github.com/stretchr/testify/require" - "google.golang.org/protobuf/types/known/anypb" ) -func TestHotstuff4Nodes1BlockHappyPath(t *testing.T) { +func TestHotstuff_4Nodes1BlockHappyPath(t *testing.T) { // Test preparation clockMock := clock.NewMock() timeReminder(t, clockMock, time.Second) // Test configs - runtimeMgrs := GenerateNodeRuntimeMgrs(t, numValidators, clockMock) - buses := GenerateBuses(t, runtimeMgrs) + runtimeMgrs := generateNodeRuntimeMgrs(t, numValidators, clockMock) + buses := generateBuses(t, runtimeMgrs) // Create & start test pocket nodes - eventsChannel := make(modules.EventsChannel, 100) - pocketNodes := CreateTestConsensusPocketNodes(t, buses, eventsChannel) - err := StartAllTestPocketNodes(t, pocketNodes) + sharedNetworkChannel := make(modules.EventsChannel, 100) + pocketNodes := createTestConsensusPocketNodes(t, buses, sharedNetworkChannel) + err := startAllTestPocketNodes(t, pocketNodes) require.NoError(t, err) - // Debug message to start consensus by triggering first view change - for _, pocketNode := range pocketNodes { - TriggerNextView(t, pocketNode) - } - advanceTime(t, clockMock, 10*time.Millisecond) - // Wait for nodes to reach height=1 by generating a block - block := WaitForNextBlock(t, clockMock, eventsChannel, pocketNodes, 1, 0, 500, true) + block := WaitForNextBlock(t, clockMock, sharedNetworkChannel, pocketNodes, 1, 0, 500, true) require.Equal(t, uint64(1), block.BlockHeader.Height) // Expecting NewRound messages for height=2 to be sent after a block is committed - _, err = waitForProposalMsgs(t, clockMock, eventsChannel, pocketNodes, 2, uint8(consensus.NewRound), 0, 0, numValidators*numValidators, 500, true) + _, err = waitForProposalMsgs(t, clockMock, sharedNetworkChannel, pocketNodes, 2, uint8(consensus.NewRound), 0, 0, numValidators*numValidators, 500, true) require.NoError(t, err) // TODO(#615): Add QC verification here after valid block mocking is implemented with issue #352. @@ -51,38 +45,23 @@ func TestHotstuff4Nodes1BlockHappyPath(t *testing.T) { requesterNode := pocketNodes[2] requesterNodePeerAddress := requesterNode.GetBus().GetConsensusModule().GetNodeAddress() - stateSyncGetBlockReq := typesCons.GetBlockRequest{ - PeerAddress: requesterNodePeerAddress, - Height: 1, - } - - stateSyncGetBlockMessage := &typesCons.StateSyncMessage{ - Message: &typesCons.StateSyncMessage_GetBlockReq{ - GetBlockReq: &stateSyncGetBlockReq, - }, - } - - anyProto, err := anypb.New(stateSyncGetBlockMessage) - require.NoError(t, err) - // Send get block request to the server node - P2PSend(t, serverNode, anyProto) + stateSyncGetBlockMsg := prepareStateSyncGetBlockMessage(t, requesterNodePeerAddress, 1) + send(t, serverNode, stateSyncGetBlockMsg) - // Server node is waiting for the get block request message - numExpectedMsgs := 1 - errMsg := "StateSync Get Block Request Message" - receivedMsg, err := WaitForNetworkStateSyncEvents(t, clockMock, eventsChannel, errMsg, numExpectedMsgs, 500, false) + // Server node is waiting for the get block response message + receivedMsg, err := waitForNetworkStateSyncEvents(t, clockMock, sharedNetworkChannel, "error waiting for StateSync.GetBlockRequest message", 1, 500, false, reflect.TypeOf(&typesCons.StateSyncMessage_GetBlockRes{})) require.NoError(t, err) + // Verify that it was a get block request of the right height msg, err := codec.GetCodec().FromAny(receivedMsg[0]) require.NoError(t, err) - stateSyncGetBlockResMessage, ok := msg.(*typesCons.StateSyncMessage) require.True(t, ok) - getBlockRes := stateSyncGetBlockResMessage.GetGetBlockRes() require.NotEmpty(t, getBlockRes) + // Validate the data in the block received require.Equal(t, uint64(1), getBlockRes.Block.GetBlockHeader().Height) } @@ -135,53 +114,53 @@ func TestQuorumCertificate_ResistenceToSignatureMalleability(t *testing.T) { t.Skip() } -func TestHotstuff4Nodes1Byzantine1Block(t *testing.T) { +func TestHotstuff_4Nodes1Byzantine1Block(t *testing.T) { t.Skip() } -func TestHotstuff4Nodes2Byzantine1Block(t *testing.T) { +func TestHotstuff_4Nodes2Byzantine1Block(t *testing.T) { t.Skip() } -func TestHotstuff4Nodes1BlockNetworkPartition(t *testing.T) { +func TestHotstuff_4Nodes1BlockNetworkPartition(t *testing.T) { t.Skip() } -func TestHotstuff4Nodes1Block4Rounds(t *testing.T) { +func TestHotstuff_4Nodes1Block4Rounds(t *testing.T) { t.Skip() } -func TestHotstuff4Nodes2Blocks(t *testing.T) { +func TestHotstuff_4Nodes2Blocks(t *testing.T) { t.Skip() } -func TestHotstuff4Nodes2NewNodes1Block(t *testing.T) { +func TestHotstuff_4Nodes2NewNodes1Block(t *testing.T) { t.Skip() } -func TestHotstuff4Nodes2DroppedNodes1Block(t *testing.T) { +func TestHotstuff_4Nodes2DroppedNodes1Block(t *testing.T) { t.Skip() } -func TestHotstuff4NodesFailOnPrepare(t *testing.T) { +func TestHotstuff_4NodesFailOnPrepare(t *testing.T) { t.Skip() } -func TestHotstuff4NodesFailOnPrecommit(t *testing.T) { +func TestHotstuff_4NodesFailOnPrecommit(t *testing.T) { t.Skip() } -func TestHotstuff4NodesFailOnCommit(t *testing.T) { +func TestHotstuff_4NodesFailOnCommit(t *testing.T) { t.Skip() } -func TestHotstuff4NodesFailOnDecide(t *testing.T) { +func TestHotstuff_4NodesFailOnDecide(t *testing.T) { t.Skip() } -func TestHotstuffValidatorWithLockedQC(t *testing.T) { +func TestHotstuff_ValidatorWithLockedQC(t *testing.T) { t.Skip() } -func TestHotstuffValidatorWithLockedQCMissingNewRoundMsg(t *testing.T) { +func TestHotstuff_ValidatorWithLockedQCMissingNewRoundMsg(t *testing.T) { t.Skip() } diff --git a/consensus/e2e_tests/pacemaker_test.go b/consensus/e2e_tests/pacemaker_test.go index 6255b1fb0..595ae3b8a 100644 --- a/consensus/e2e_tests/pacemaker_test.go +++ b/consensus/e2e_tests/pacemaker_test.go @@ -23,44 +23,42 @@ func TestPacemakerTimeoutIncreasesRound(t *testing.T) { paceMakerTimeoutMsec := uint64(10000) // Set a small pacemaker timeout paceMakerTimeout := time.Duration(paceMakerTimeoutMsec) * time.Millisecond consensusMessageTimeout := time.Duration(paceMakerTimeoutMsec / 5) // Must be smaller than pacemaker timeout because we expect a deterministic number of consensus messages. - runtimeMgrs := GenerateNodeRuntimeMgrs(t, numValidators, clockMock) + runtimeMgrs := generateNodeRuntimeMgrs(t, numValidators, clockMock) for _, runtimeConfig := range runtimeMgrs { consCfg := runtimeConfig.GetConfig().Consensus.PacemakerConfig consCfg.TimeoutMsec = paceMakerTimeoutMsec } - buses := GenerateBuses(t, runtimeMgrs) + buses := generateBuses(t, runtimeMgrs) // Create & start test pocket nodes - eventsChannel := make(modules.EventsChannel, 100) - pocketNodes := CreateTestConsensusPocketNodes(t, buses, eventsChannel) - err := StartAllTestPocketNodes(t, pocketNodes) + sharedNetworkChannel := make(modules.EventsChannel, 100) + pocketNodes := createTestConsensusPocketNodes(t, buses, sharedNetworkChannel) + err := startAllTestPocketNodes(t, pocketNodes) require.NoError(t, err) // Debug message to start consensus by triggering next view - for _, pocketNode := range pocketNodes { - TriggerNextView(t, pocketNode) - } + triggerNextView(t, pocketNodes) // Advance time by an amount shorter than the pacemaker timeout advanceTime(t, clockMock, 10*time.Millisecond) - _, err = waitForProposalMsgs(t, clockMock, eventsChannel, pocketNodes, 1, uint8(consensus.NewRound), 0, 0, numValidators*numValidators, consensusMessageTimeout, true) + _, err = waitForProposalMsgs(t, clockMock, sharedNetworkChannel, pocketNodes, 1, uint8(consensus.NewRound), 0, 0, numValidators*numValidators, consensusMessageTimeout, true) require.NoError(t, err) // Force the pacemaker to time out forcePacemakerTimeout(t, clockMock, paceMakerTimeout) // Wait for the round=1 to fail - _, err = waitForProposalMsgs(t, clockMock, eventsChannel, pocketNodes, 1, uint8(consensus.NewRound), 1, 0, numValidators*numValidators, consensusMessageTimeout, true) + _, err = waitForProposalMsgs(t, clockMock, sharedNetworkChannel, pocketNodes, 1, uint8(consensus.NewRound), 1, 0, numValidators*numValidators, consensusMessageTimeout, true) require.NoError(t, err) forcePacemakerTimeout(t, clockMock, paceMakerTimeout) // Wait for the round=2 to fail - _, err = waitForProposalMsgs(t, clockMock, eventsChannel, pocketNodes, 1, uint8(consensus.NewRound), 2, 0, numValidators*numValidators, consensusMessageTimeout, true) + _, err = waitForProposalMsgs(t, clockMock, sharedNetworkChannel, pocketNodes, 1, uint8(consensus.NewRound), 2, 0, numValidators*numValidators, consensusMessageTimeout, true) require.NoError(t, err) forcePacemakerTimeout(t, clockMock, paceMakerTimeout) // Wait for the round=3 to succeed - newRoundMessages, err := waitForProposalMsgs(t, clockMock, eventsChannel, pocketNodes, 1, uint8(consensus.NewRound), 3, 0, numValidators*numValidators, consensusMessageTimeout, true) + newRoundMessages, err := waitForProposalMsgs(t, clockMock, sharedNetworkChannel, pocketNodes, 1, uint8(consensus.NewRound), 3, 0, numValidators*numValidators, consensusMessageTimeout, true) require.NoError(t, err) broadcastMessages(t, newRoundMessages, pocketNodes) advanceTime(t, clockMock, 10*time.Millisecond) @@ -68,7 +66,7 @@ func TestPacemakerTimeoutIncreasesRound(t *testing.T) { // Get the expected leader id for round=3 leaderId := typesCons.NodeId(pocketNodes[1].GetBus().GetConsensusModule().GetLeaderForView(1, 3, uint8(consensus.NewRound))) // Wait for nodes to proceed to Propose step in round=3 - _, err = waitForProposalMsgs(t, clockMock, eventsChannel, pocketNodes, 1, uint8(consensus.Prepare), 3, leaderId, numValidators, consensusMessageTimeout, true) + _, err = waitForProposalMsgs(t, clockMock, sharedNetworkChannel, pocketNodes, 1, uint8(consensus.Prepare), 3, leaderId, numValidators, consensusMessageTimeout, true) require.NoError(t, err) } @@ -77,13 +75,13 @@ func TestPacemakerCatchupSameStepDifferentRounds(t *testing.T) { clockMock := clock.NewMock() timeReminder(t, clockMock, time.Second) - runtimeConfigs := GenerateNodeRuntimeMgrs(t, numValidators, clockMock) - buses := GenerateBuses(t, runtimeConfigs) + runtimeConfigs := generateNodeRuntimeMgrs(t, numValidators, clockMock) + buses := generateBuses(t, runtimeConfigs) // Create & start test pocket nodes - eventsChannel := make(modules.EventsChannel, 100) - pocketNodes := CreateTestConsensusPocketNodes(t, buses, eventsChannel) - err := StartAllTestPocketNodes(t, pocketNodes) + sharedNetworkChannel := make(modules.EventsChannel, 100) + pocketNodes := createTestConsensusPocketNodes(t, buses, sharedNetworkChannel) + err := startAllTestPocketNodes(t, pocketNodes) require.NoError(t, err) // Starting point @@ -92,7 +90,7 @@ func TestPacemakerCatchupSameStepDifferentRounds(t *testing.T) { // UnitTestNet configs paceMakerTimeoutMsec := uint64(500) // Set a small pacemaker timeout - runtimeMgrs := GenerateNodeRuntimeMgrs(t, numValidators, clockMock) + runtimeMgrs := generateNodeRuntimeMgrs(t, numValidators, clockMock) for _, runtimeConfig := range runtimeMgrs { runtimeConfig.GetConfig().Consensus.PacemakerConfig.TimeoutMsec = paceMakerTimeoutMsec } @@ -112,22 +110,23 @@ func TestPacemakerCatchupSameStepDifferentRounds(t *testing.T) { // Prepare leader info leaderRound := uint64(6) + // Any node in pocketNodes mapping can be used to get this function + leaderFn := pocketNodes[1].GetBus().GetConsensusModule().GetLeaderForView + // Get leaderId for the given height, round and step, by using the Consensus Modules' GetLeaderForView() function. - // Any node in pocketNodes mapping can be used to call GetLeaderForView() function. - leaderId := typesCons.NodeId(pocketNodes[1].GetBus().GetConsensusModule().GetLeaderForView(testHeight, leaderRound, uint8(consensus.Prepare))) + leaderId := typesCons.NodeId(leaderFn(testHeight, leaderRound, uint8(consensus.Prepare))) leader := pocketNodes[leaderId] leaderPK, err := leader.GetBus().GetConsensusModule().GetPrivateKey() require.NoError(t, err) - block := generatePlaceholderBlock(testHeight, leaderPK.Address()) - leader.GetBus().GetConsensusModule().SetBlock(block) - // Set the leader to be in the highest round. - pocketNodes[1].GetBus().GetConsensusModule().SetRound(leaderRound - 2) - pocketNodes[2].GetBus().GetConsensusModule().SetRound(leaderRound - 3) + require.Equal(t, typesCons.NodeId(1), leaderId) pocketNodes[leaderId].GetBus().GetConsensusModule().SetRound(leaderRound) - pocketNodes[4].GetBus().GetConsensusModule().SetRound(leaderRound - 4) + pocketNodes[2].GetBus().GetConsensusModule().SetRound(leaderRound - 1) + pocketNodes[3].GetBus().GetConsensusModule().SetRound(leaderRound - 2) + pocketNodes[4].GetBus().GetConsensusModule().SetRound(leaderRound - 3) + block := generatePlaceholderBlock(testHeight, leaderPK.Address()) prepareProposal := &typesCons.HotstuffMessage{ Type: consensus.Propose, Height: testHeight, @@ -144,13 +143,13 @@ func TestPacemakerCatchupSameStepDifferentRounds(t *testing.T) { broadcastMessages(t, []*anypb.Any{anyMsg}, pocketNodes) advanceTime(t, clockMock, 10*time.Millisecond) - _, err = WaitForNetworkConsensusEvents(t, clockMock, eventsChannel, 2, consensus.Vote, numExpectedMsgs, time.Duration(msgTimeout), true) + _, err = waitForNetworkConsensusEvents(t, clockMock, sharedNetworkChannel, 2, consensus.Vote, numExpectedMsgs, time.Duration(msgTimeout), true) require.NoError(t, err) // Check that all the nodes caught up to the leader's (i.e. the latest) round for nodeId, pocketNode := range pocketNodes { - nodeState := GetConsensusNodeState(pocketNode) + nodeState := getConsensusNodeState(pocketNode) if nodeId == leaderId { require.Equal(t, consensus.Prepare.String(), typesCons.HotstuffStep(nodeState.Step).String()) } else { diff --git a/consensus/e2e_tests/state_sync_test.go b/consensus/e2e_tests/state_sync_test.go index d46eb8c25..1eff6ceb1 100644 --- a/consensus/e2e_tests/state_sync_test.go +++ b/consensus/e2e_tests/state_sync_test.go @@ -8,116 +8,72 @@ import ( "github.com/benbjohnson/clock" "github.com/pokt-network/pocket/consensus" typesCons "github.com/pokt-network/pocket/consensus/types" + "github.com/pokt-network/pocket/runtime" "github.com/pokt-network/pocket/shared/codec" + "github.com/pokt-network/pocket/shared/messaging" "github.com/pokt-network/pocket/shared/modules" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "google.golang.org/protobuf/types/known/anypb" ) -func TestStateSync_ServerGetMetaDataReq_Success(t *testing.T) { - // Test preparation - clockMock := clock.NewMock() - timeReminder(t, clockMock, time.Second) - - runtimeMgrs := GenerateNodeRuntimeMgrs(t, numValidators, clockMock) - buses := GenerateBuses(t, runtimeMgrs) - - // Create & start test pocket nodes - eventsChannel := make(modules.EventsChannel, 100) - pocketNodes := CreateTestConsensusPocketNodes(t, buses, eventsChannel) - err := StartAllTestPocketNodes(t, pocketNodes) - require.NoError(t, err) - - testHeight := uint64(4) +func TestStateSync_MetadataRequestResponse_Success(t *testing.T) { + clockMock, sharedNetworkChannel, pocketNodes := prepareStateSyncTestEnvironment(t) // Choose node 1 as the server node - // Set server node's height to test height. serverNode := pocketNodes[1] serverNodePeerId := serverNode.GetBus().GetConsensusModule().GetNodeAddress() - serverNode.GetBus().GetConsensusModule().SetHeight(testHeight) + // Set server node's height to test height. + serverNode.GetBus().GetConsensusModule().SetHeight(uint64(4)) // Choose node 2 as the requester node. requesterNode := pocketNodes[2] requesterNodePeerAddress := requesterNode.GetBus().GetConsensusModule().GetNodeAddress() - // Test MetaData Req - stateSyncMetaDataReqMessage := &typesCons.StateSyncMessage{ - Message: &typesCons.StateSyncMessage_MetadataReq{ - MetadataReq: &typesCons.StateSyncMetadataRequest{ - PeerAddress: requesterNodePeerAddress, - }, - }, - } - anyProto, err := anypb.New(stateSyncMetaDataReqMessage) - require.NoError(t, err) - // Send metadata request to the server node - P2PSend(t, serverNode, anyProto) + anyProto := prepareStateSyncGetMetadataMessage(t, requesterNodePeerAddress) + send(t, serverNode, anyProto) - // Start waiting for the metadata request on server node, - errMsg := "StateSync Metadata Request" - receivedMsg, err := WaitForNetworkStateSyncEvents(t, clockMock, eventsChannel, errMsg, 1, 500, false) + // Wait for response from the server node + receivedMsgs, err := waitForNetworkStateSyncEvents(t, clockMock, sharedNetworkChannel, "did not receive response to state sync metadata request", 1, 500, false, reflect.TypeOf(&typesCons.StateSyncMessage_MetadataRes{})) require.NoError(t, err) - msg, err := codec.GetCodec().FromAny(receivedMsg[0]) + // Extract the response + msg, err := codec.GetCodec().FromAny(receivedMsgs[0]) require.NoError(t, err) - - stateSyncMetaDataResMessage, ok := msg.(*typesCons.StateSyncMessage) + stateSyncMetaDataResMsg, ok := msg.(*typesCons.StateSyncMessage) require.True(t, ok) + stateSyncMetaDataRes := stateSyncMetaDataResMsg.GetMetadataRes() + require.NotEmpty(t, stateSyncMetaDataRes) - metaDataRes := stateSyncMetaDataResMessage.GetMetadataRes() - require.NotEmpty(t, metaDataRes) - - require.Equal(t, uint64(4), metaDataRes.MaxHeight) - require.Equal(t, uint64(1), metaDataRes.MinHeight) - require.Equal(t, serverNodePeerId, metaDataRes.PeerAddress) + // Validate the response + require.Equal(t, uint64(3), stateSyncMetaDataRes.MaxHeight) // 3 because node sends the last persisted height + require.Equal(t, uint64(1), stateSyncMetaDataRes.MinHeight) + require.Equal(t, serverNodePeerId, stateSyncMetaDataRes.PeerAddress) } -func TestStateSync_ServerGetBlock_Success(t *testing.T) { - // Test preparation - clockMock := clock.NewMock() - timeReminder(t, clockMock, time.Second) - - // Test configs - runtimeMgrs := GenerateNodeRuntimeMgrs(t, numValidators, clockMock) - buses := GenerateBuses(t, runtimeMgrs) +func TestStateSync_BlockRequestResponse_Success(t *testing.T) { + clockMock, sharedNetworkChannel, pocketNodes := prepareStateSyncTestEnvironment(t) - // Create & start test pocket nodes - eventsChannel := make(modules.EventsChannel, 100) - pocketNodes := CreateTestConsensusPocketNodes(t, buses, eventsChannel) - err := StartAllTestPocketNodes(t, pocketNodes) - require.NoError(t, err) - - testHeight := uint64(5) + // Choose node 1 as the server node serverNode := pocketNodes[1] - serverNode.GetBus().GetConsensusModule().SetHeight(testHeight) + // Set server node's height to test height. + serverNode.GetBus().GetConsensusModule().SetHeight(uint64(5)) // Choose node 2 as the requester node requesterNode := pocketNodes[2] requesterNodePeerAddress := requesterNode.GetBus().GetConsensusModule().GetNodeAddress() - // Passing Test - // Test GetBlock Req - stateSyncGetBlockMessage := &typesCons.StateSyncMessage{ - Message: &typesCons.StateSyncMessage_GetBlockReq{ - GetBlockReq: &typesCons.GetBlockRequest{ - PeerAddress: requesterNodePeerAddress, - Height: 1, - }, - }, - } - - anyProto, err := anypb.New(stateSyncGetBlockMessage) - require.NoError(t, err) + // Prepare GetBlockRequest + stateSyncGetBlockMsg := prepareStateSyncGetBlockMessage(t, requesterNodePeerAddress, 1) // Send get block request to the server node - P2PSend(t, serverNode, anyProto) + send(t, serverNode, stateSyncGetBlockMsg) - // Start waiting for the get block request on server node, expect to return error - errMsg := "StateSync Get Block Request Message" - receivedMsg, err := WaitForNetworkStateSyncEvents(t, clockMock, eventsChannel, errMsg, 1, 500, false) + // Start waiting for the get block response on server node + receivedMsg, err := waitForNetworkStateSyncEvents(t, clockMock, sharedNetworkChannel, "error waiting on response to a get block request", 1, 500, false, reflect.TypeOf(&typesCons.StateSyncMessage_GetBlockRes{})) require.NoError(t, err) + // validate the response msg, err := codec.GetCodec().FromAny(receivedMsg[0]) require.NoError(t, err) @@ -128,25 +84,16 @@ func TestStateSync_ServerGetBlock_Success(t *testing.T) { require.NotEmpty(t, getBlockRes) require.Equal(t, uint64(1), getBlockRes.Block.GetBlockHeader().Height) -} - -func TestStateSync_ServerGetBlock_FailNonExistingBlock(t *testing.T) { - // Test preparation - clockMock := clock.NewMock() - timeReminder(t, clockMock, time.Second) - // Test configs - runtimeMgrs := GenerateNodeRuntimeMgrs(t, numValidators, clockMock) - buses := GenerateBuses(t, runtimeMgrs) + // IMPROVE: What other data should we validate from the response? +} - // Create & start test pocket nodes - eventsChannel := make(modules.EventsChannel, 100) - pocketNodes := CreateTestConsensusPocketNodes(t, buses, eventsChannel) - err := StartAllTestPocketNodes(t, pocketNodes) - require.NoError(t, err) +func TestStateSync_BlockRequestResponse_FailNonExistingBlock(t *testing.T) { + clockMock, sharedNetworkChannel, pocketNodes := prepareStateSyncTestEnvironment(t) testHeight := uint64(5) + // Choose node 1 as the server node and set its height serverNode := pocketNodes[1] serverNode.GetBus().GetConsensusModule().SetHeight(testHeight) @@ -154,140 +101,158 @@ func TestStateSync_ServerGetBlock_FailNonExistingBlock(t *testing.T) { requesterNode := pocketNodes[2] requesterNodePeerAddress := requesterNode.GetBus().GetConsensusModule().GetNodeAddress() - // Failing Test - // Get Block Req is current block height + 1 - requestHeight := testHeight + 1 - stateSyncGetBlockMessage := &typesCons.StateSyncMessage{ - Message: &typesCons.StateSyncMessage_GetBlockReq{ - GetBlockReq: &typesCons.GetBlockRequest{ - PeerAddress: requesterNodePeerAddress, - Height: requestHeight, - }, - }, - } - - anyProto, err := anypb.New(stateSyncGetBlockMessage) - require.NoError(t, err) + // Prepare a get block request for a non existing block (server is only at height 5) + stateSyncGetBlockMsg := prepareStateSyncGetBlockMessage(t, requesterNodePeerAddress, testHeight+2) // Send get block request to the server node - P2PSend(t, serverNode, anyProto) + send(t, serverNode, stateSyncGetBlockMsg) // Start waiting for the get block request on server node, expect to return error - errMsg := "StateSync Get Block Request Message" - _, err = WaitForNetworkStateSyncEvents(t, clockMock, eventsChannel, errMsg, 1, 500, false) + errMsg := "expecting to time out waiting on a response from a non existent" + _, err := waitForNetworkStateSyncEvents(t, clockMock, sharedNetworkChannel, errMsg, 1, 500, false, reflect.TypeOf(&typesCons.StateSyncMessage_GetBlockRes{})) require.Error(t, err) } func TestStateSync_UnsyncedPeerSyncs_Success(t *testing.T) { - // Test preparation - clockMock := clock.NewMock() - timeReminder(t, clockMock, time.Second) - runtimeMgrs := GenerateNodeRuntimeMgrs(t, numValidators, clockMock) - buses := GenerateBuses(t, runtimeMgrs) + clockMock, sharedNetworkChannel, pocketNodes := prepareStateSyncTestEnvironment(t) - // Create & start test pocket nodes - eventsChannel := make(modules.EventsChannel, 100) - pocketNodes := CreateTestConsensusPocketNodes(t, buses, eventsChannel) - - err := StartAllTestPocketNodes(t, pocketNodes) - require.NoError(t, err) - - // Prepare leader info - testHeight := uint64(3) - testRound := uint64(0) - testStep := uint8(consensus.NewRound) - - // Prepare unsynced node info - unsyncedNode := pocketNodes[2] - unsyncedNodeId := typesCons.NodeId(2) + // Select node 2 as the unsynched node that will catch up + unsyncedNodeId := typesCons.NodeId(pocketNodes[2].GetBus().GetConsensusModule().GetNodeId()) + unsyncedNode := pocketNodes[unsyncedNodeId] unsyncedNodeHeight := uint64(2) + targetHeight := uint64(6) - // Set the unsynced node to height (2) and rest of the nodes to height (3) + // Set the unsynced node to height (2) and rest of the nodes to height (4) + unsyncedNode.GetBus().GetConsensusModule().SetHeight(unsyncedNodeHeight) for id, pocketNode := range pocketNodes { - if id == unsyncedNodeId { - pocketNode.GetBus().GetConsensusModule().SetHeight(unsyncedNodeHeight) - } else { - pocketNode.GetBus().GetConsensusModule().SetHeight(testHeight) + consensusMod := pocketNode.GetBus().GetConsensusModule() + if id != unsyncedNodeId { + consensusMod.SetHeight(targetHeight) } - pocketNode.GetBus().GetConsensusModule().SetStep(testStep) - pocketNode.GetBus().GetConsensusModule().SetRound(testRound) - - utilityUnitOfWork, err := pocketNode.GetBus().GetUtilityModule().NewUnitOfWork(int64(testHeight)) - require.NoError(t, err) - pocketNode.GetBus().GetConsensusModule().SetUtilityUnitOfWork(utilityUnitOfWork) + consensusMod.SetStep(uint8(consensus.NewRound)) + consensusMod.SetRound(uint64(0)) } - // Debug message to start consensus by triggering first view change - for _, pocketNode := range pocketNodes { - TriggerNextView(t, pocketNode) + // Sanity check unsynched node is at height 2 + assertHeight(t, unsyncedNodeId, uint64(2), getConsensusNodeState(unsyncedNode).Height) + + // Broadcast metadata to all the others nodes so the node that's behind has a view of the network + anyProto := prepareStateSyncGetMetadataMessage(t, unsyncedNode.GetBus().GetConsensusModule().GetNodeAddress()) + broadcast(t, pocketNodes, anyProto) + + // Make sure the unsynched node has a view of the network + receivedMsgs, err := waitForNetworkStateSyncEvents(t, clockMock, sharedNetworkChannel, "did not receive response to state sync metadata request", len(pocketNodes), 500, false, nil) + require.NoError(t, err) + for _, msg := range receivedMsgs { + send(t, unsyncedNode, msg) } - currentRound := testRound + 1 + // IMPROVE: Look into ways to assert on unsynched.MinHeightViewOfNetwork and unsynched.MaxHeightViewOfNetwork + + // Trigger the next round of consensus so the unsynched nodes is prompted to start synching + triggerNextView(t, pocketNodes) + advanceTime(t, clockMock, 10*time.Millisecond) - // Get leaderId for the given height, round and step, by using the Consensus Modules' GetLeaderForView() function. - // Any node in pocketNodes mapping can be used to call GetLeaderForView() function. - leaderId := typesCons.NodeId(pocketNodes[1].GetBus().GetConsensusModule().GetLeaderForView(testHeight, currentRound, testStep)) - leader := pocketNodes[leaderId] - leaderPK, err := leader.GetBus().GetConsensusModule().GetPrivateKey() + // Wait for proposal messages + proposalMsgs, err := waitForNetworkConsensusEvents(t, clockMock, sharedNetworkChannel, typesCons.HotstuffStep(consensus.NewRound), consensus.Propose, numValidators*numValidators, 500, false) require.NoError(t, err) - block := generatePlaceholderBlock(testHeight, leaderPK.Address()) - leader.GetBus().GetConsensusModule().SetBlock(block) + // Broadcast the proposal messages to all nodes + broadcastMessages(t, proposalMsgs, pocketNodes) + advanceTime(t, clockMock, 10*time.Millisecond) - // Assert that unsynced node has a different view of the network than the rest of the nodes - newRoundMessages, err := WaitForNetworkConsensusEvents(t, clockMock, eventsChannel, consensus.NewRound, consensus.Propose, numValidators*numValidators, 500, true) + // TODO: Figure out why we have one extra (non harmful) request at the very beginning + _, err = waitForNetworkStateSyncEvents(t, clockMock, sharedNetworkChannel, "error waiting on response to a get block request", 1, 5000, false, reflect.TypeOf(&typesCons.StateSyncMessage_GetBlockReq{})) require.NoError(t, err) - for nodeId, pocketNode := range pocketNodes { - nodeState := GetConsensusNodeState(pocketNode) - if nodeId == unsyncedNodeId { - assertNodeConsensusView(t, nodeId, - typesCons.ConsensusNodeState{ - Height: unsyncedNodeHeight, - Step: testStep, - Round: uint8(currentRound), - }, - nodeState) - } else { - assertNodeConsensusView(t, nodeId, - typesCons.ConsensusNodeState{ - Height: testHeight, - Step: testStep, - Round: uint8(currentRound), - }, - nodeState) + for unsyncedNodeHeight < targetHeight { + // Wait for the unsynched node to request the block at the current height + blockRequests, err := waitForNetworkStateSyncEvents(t, clockMock, sharedNetworkChannel, "error waiting on response to a get block request", 1, 5000, false, reflect.TypeOf(&typesCons.StateSyncMessage_GetBlockReq{})) + require.NoError(t, err) + + // Validate the height being requested is correct + msg, err := codec.GetCodec().FromAny(blockRequests[0]) + require.NoError(t, err) + heightRequested := msg.(*typesCons.StateSyncMessage).GetGetBlockReq().Height + require.Equal(t, unsyncedNodeHeight, heightRequested) + + // Broadcast the block request to all nodes + broadcast(t, pocketNodes, blockRequests[0]) + advanceTime(t, clockMock, 10*time.Millisecond) + + // Wait for the unsynched node to receive the block responses + blockResponses, err := waitForNetworkStateSyncEvents(t, clockMock, sharedNetworkChannel, "error waiting on response to a get block response", numValidators-1, 5000, false, reflect.TypeOf(&typesCons.StateSyncMessage_GetBlockRes{})) + require.NoError(t, err) + + // Validate that the block is the same from all the validators who send it (non byzantine scenario) + var blockResponse *typesCons.GetBlockResponse + for _, msg := range blockResponses { + msgAny, err := codec.GetCodec().FromAny(msg) + require.NoError(t, err) + + stateSyncMessage, ok := msgAny.(*typesCons.StateSyncMessage) + require.True(t, ok) + + if blockResponse == nil { + blockResponse = stateSyncMessage.GetGetBlockRes() + continue + } + require.Equal(t, blockResponse.Block, stateSyncMessage.GetGetBlockRes().Block) } - require.Equal(t, false, nodeState.IsLeader) - require.Equal(t, typesCons.NodeId(0), nodeState.LeaderId) - } - metadataReceived := &typesCons.StateSyncMetadataResponse{ - PeerAddress: "unused_peer_addr_in_tests", - MinHeight: uint64(1), - MaxHeight: testHeight, - } + // Send one of the responses (since they are equal) to the unsynched node to apply it + send(t, unsyncedNode, blockResponses[0]) + advanceTime(t, clockMock, 10*time.Millisecond) + debugChannel := unsyncedNode.GetBus().GetDebugEventBus() + loop: + for { + select { + case e := <-debugChannel: + if e.GetContentType() == messaging.ConsensusNewHeightEventType { + msg, err := codec.GetCodec().FromAny(e.Content) + require.NoError(t, err) + blockCommittedEvent, ok := msg.(*messaging.ConsensusNewHeightEvent) + require.True(t, ok) + if unsyncedNodeHeight == blockCommittedEvent.Height { + break loop + } + } + case <-time.After(time.Second): + assert.Fail(t, "Timed out waiting for block %d to be committed...", unsyncedNodeHeight) + } + } - // Simulate state sync metadata response by pushing metadata to the unsynced node's consensus module - consensusModImpl := GetConsensusModImpl(unsyncedNode) - consensusModImpl.MethodByName("PushStateSyncMetadataResponse").Call([]reflect.Value{reflect.ValueOf(metadataReceived)}) + // ensure unsynced node height increased + nodeState := getConsensusNodeState(unsyncedNode) + assertHeight(t, unsyncedNodeId, unsyncedNodeHeight+1, nodeState.Height) - for _, message := range newRoundMessages { - P2PBroadcast(t, pocketNodes, message) + // Same as `unsyncedNodeHeight+=1` + unsyncedNodeHeight = unsyncedNode.GetBus().GetConsensusModule().CurrentHeight() } - advanceTime(t, clockMock, 10*time.Millisecond) - // 2. Propose - _, err = WaitForNetworkConsensusEvents(t, clockMock, eventsChannel, consensus.Prepare, consensus.Propose, numValidators, 500, true) - require.NoError(t, err) + assertHeight(t, unsyncedNodeId, targetHeight, getConsensusNodeState(unsyncedNode).Height) +} + +func prepareStateSyncTestEnvironment(t *testing.T) (*clock.Mock, modules.EventsChannel, idToNodeMapping) { + // Test preparation + clockMock := clock.NewMock() + timeReminder(t, clockMock, time.Second) - // TODO(#352): This function will be updated once state sync implementation is complete - err = WaitForNodeToSync(t, clockMock, eventsChannel, unsyncedNode, pocketNodes, testHeight) + // Test configs + runtimeMgrs := generateNodeRuntimeMgrs(t, numValidators, clockMock) + buses := generateBuses(t, runtimeMgrs, runtime.WithNewDebugEventsChannel()) + + // This channel captures all the messages that consensus nodes would send to each other over the network + sharedNetworkChannel := make(modules.EventsChannel, 100) + + // Create & start test pocket nodes + pocketNodes := createTestConsensusPocketNodes(t, buses, sharedNetworkChannel) + err := startAllTestPocketNodes(t, pocketNodes) require.NoError(t, err) - // TODO(#352): Add height check once state sync implmentation is complete + return clockMock, sharedNetworkChannel, pocketNodes } -// TODO(#352): Implement these tests - +// INCOMPLETE: Implement the following tests func TestStateSync_UnsyncedPeerSyncsABlock_Success(t *testing.T) { t.Skip() } diff --git a/consensus/e2e_tests/utils_test.go b/consensus/e2e_tests/utils_test.go index fef2fbc00..a812a5fbe 100644 --- a/consensus/e2e_tests/utils_test.go +++ b/consensus/e2e_tests/utils_test.go @@ -29,6 +29,7 @@ import ( "github.com/pokt-network/pocket/shared/messaging" "github.com/pokt-network/pocket/shared/modules" mockModules "github.com/pokt-network/pocket/shared/modules/mocks" + "github.com/pokt-network/pocket/shared/utils" "github.com/pokt-network/pocket/state_machine" "github.com/stretchr/testify/require" "google.golang.org/protobuf/types/known/anypb" @@ -39,22 +40,25 @@ func TestMain(m *testing.M) { os.Exit(exitCode) } -// TODO(integration): These are temporary variables used in the prototype integration phase that -// will need to be parameterized later once the test framework design matures. +// TECHDEBT: Constants in the `e2e_tests` test suite that should be parameterized const ( - numValidators = 4 - stateHash = "42" + numValidators = 4 // The number of validators in the testing network created + dummyStateHash = "42" // The state hash returned for all committed blocks + numMockedBlocks = 200 // The number of mocked blocks in in memory for testing purposes ) var maxTxBytes = defaults.DefaultConsensusMaxMempoolBytes -type IdToNodeMapping map[typesCons.NodeId]*shared.Node +type idToNodeMapping map[typesCons.NodeId]*shared.Node +type idToPrivKeyMapping map[typesCons.NodeId]cryptoPocket.PrivateKey /*** Node Generation Helpers ***/ -func GenerateNodeRuntimeMgrs(_ *testing.T, validatorCount int, clockMgr clock.Clock) []*runtime.Manager { +//nolint:unparam // validatorCount will be varied in the future +func generateNodeRuntimeMgrs(t *testing.T, validatorCount int, clockMgr clock.Clock) []*runtime.Manager { + t.Helper() + runtimeMgrs := make([]*runtime.Manager, validatorCount) - var validatorKeys []string genesisState, validatorKeys := test_artifacts.NewGenesisState(validatorCount, 1, 1, 1) cfgs := test_artifacts.NewDefaultConfigs(validatorKeys) for i, config := range cfgs { @@ -73,13 +77,14 @@ func GenerateNodeRuntimeMgrs(_ *testing.T, validatorCount int, clockMgr clock.Cl return runtimeMgrs } -func CreateTestConsensusPocketNodes( +// TECHDEBT: Try to avoid exposing `modules.EventsChannel` outside the `shared` package and adding the appropriate mocks to the bus. +func createTestConsensusPocketNodes( t *testing.T, buses []modules.Bus, - eventsChannel modules.EventsChannel, -) (pocketNodes IdToNodeMapping) { - pocketNodes = make(IdToNodeMapping, len(buses)) - // TODO(design): The order here is important in order for NodeId to be set correctly below. + sharedNetworkChannel modules.EventsChannel, +) (pocketNodes idToNodeMapping) { + pocketNodes = make(idToNodeMapping, len(buses)) + // TECHDEBT: The order here is important in order for NodeIds to be set correctly below. // This logic will need to change once proper leader election is implemented. sort.Slice(buses, func(i, j int) bool { pk, err := cryptoPocket.NewPrivateKey(buses[i].GetRuntimeMgr().GetConfig().PrivateKey) @@ -89,21 +94,32 @@ func CreateTestConsensusPocketNodes( return pk.Address().String() < pk2.Address().String() }) - for i := range buses { - pocketNode := CreateTestConsensusPocketNode(t, buses[i], eventsChannel) - // TODO(olshansky): Figure this part out. - pocketNodes[typesCons.NodeId(i+1)] = pocketNode + blocks := &testingBlocks{} + + validatorPrivKeys := make(idToPrivKeyMapping, len(buses)) + for i, bus := range buses { + nodeId := typesCons.NodeId(i + 1) + + pocketNode := createTestConsensusPocketNode(t, bus, sharedNetworkChannel, blocks) + pocketNodes[nodeId] = pocketNode + + validatorPrivKey, err := cryptoPocket.NewPrivateKey(pocketNode.GetBus().GetRuntimeMgr().GetConfig().PrivateKey) + require.NoError(t, err) + + validatorPrivKeys[nodeId] = validatorPrivKey } + blocks.preparePlaceholderBlocks(t, buses[0], validatorPrivKeys, numMockedBlocks) return } // Creates a pocket node where all the primary modules, exception for consensus, are mocked -func CreateTestConsensusPocketNode( +func createTestConsensusPocketNode( t *testing.T, bus modules.Bus, - eventsChannel modules.EventsChannel, + sharedNetworkChannel modules.EventsChannel, + placeholderBlocks *testingBlocks, ) *shared.Node { - persistenceMock := basePersistenceMock(t, eventsChannel, bus) + persistenceMock := basePersistenceMock(t, sharedNetworkChannel, bus, placeholderBlocks) bus.RegisterModule(persistenceMock) consensusMod, err := consensus.Create(bus) @@ -117,11 +133,11 @@ func CreateTestConsensusPocketNode( runtimeMgr := (bus).GetRuntimeMgr() // TODO(olshansky): At the moment we are using the same base mocks for all the tests, // but note that they will need to be customized on a per test basis. - p2pMock := baseP2PMock(t, eventsChannel) - utilityMock := baseUtilityMock(t, eventsChannel, runtimeMgr.GetGenesis(), consensusModule) - telemetryMock := baseTelemetryMock(t, eventsChannel) - loggerMock := baseLoggerMock(t, eventsChannel) - rpcMock := baseRpcMock(t, eventsChannel) + p2pMock := baseP2PMock(t, sharedNetworkChannel) + utilityMock := baseUtilityMock(t, sharedNetworkChannel, runtimeMgr.GetGenesis(), consensusModule) + telemetryMock := baseTelemetryMock(t, sharedNetworkChannel) + loggerMock := baseLoggerMock(t, sharedNetworkChannel) + rpcMock := baseRpcMock(t, sharedNetworkChannel) ibcMock, hostMock := ibcUtils.IBCMockWithHost(t, bus) bus.RegisterModule(hostMock) @@ -148,22 +164,19 @@ func CreateTestConsensusPocketNode( return pocketNode } -func GenerateBuses(t *testing.T, runtimeMgrs []*runtime.Manager) (buses []modules.Bus) { +func generateBuses(t *testing.T, runtimeMgrs []*runtime.Manager, opts ...modules.BusOption) (buses []modules.Bus) { buses = make([]modules.Bus, len(runtimeMgrs)) for i := range runtimeMgrs { - bus, err := runtime.CreateBus(runtimeMgrs[i]) + bus, err := runtime.CreateBus(runtimeMgrs[i], opts...) require.NoError(t, err) buses[i] = bus } return } -// CLEANUP: Reduce package scope visibility in the consensus test module -func StartAllTestPocketNodes(t *testing.T, pocketNodes IdToNodeMapping) error { +func startAllTestPocketNodes(t *testing.T, pocketNodes idToNodeMapping) error { for _, pocketNode := range pocketNodes { go startNode(t, pocketNode) - startEvent := pocketNode.GetBus().GetBusEvent() - require.Equal(t, startEvent.GetContentType(), messaging.NodeStartedEventType) stateMachine := pocketNode.GetBus().GetStateMachineModule() if err := stateMachine.SendEvent(coreTypes.StateMachineEvent_Start); err != nil { return err @@ -177,27 +190,17 @@ func StartAllTestPocketNodes(t *testing.T, pocketNodes IdToNodeMapping) error { /*** Node Visibility/Reflection Helpers ***/ -// TODO(discuss): Should we use reflections inside the testing module as being done here or explicitly -// define the interfaces used for debug/development. The latter will probably scale more but will -// require more effort and pollute the source code with debugging information. -func GetConsensusNodeState(node *shared.Node) typesCons.ConsensusNodeState { - return GetConsensusModImpl(node).MethodByName("GetNodeState").Call([]reflect.Value{})[0].Interface().(typesCons.ConsensusNodeState) +// HACK: Look for ways to avoid using reflections in the testing package. It was a quick & dirty way to keep going. +func getConsensusNodeState(node *shared.Node) typesCons.ConsensusNodeState { + return getConsensusModImpl(node).MethodByName("GetNodeState").Call([]reflect.Value{})[0].Interface().(typesCons.ConsensusNodeState) } -func GetConsensusModElem(node *shared.Node) reflect.Value { - return reflect.ValueOf(node.GetBus().GetConsensusModule()).Elem() -} - -func GetConsensusModImpl(node *shared.Node) reflect.Value { +func getConsensusModImpl(node *shared.Node) reflect.Value { return reflect.ValueOf(node.GetBus().GetConsensusModule()) } /*** Debug/Development Message Helpers ***/ -func TriggerNextView(t *testing.T, node *shared.Node) { - triggerDebugMessage(t, node, messaging.DebugMessageAction_DEBUG_CONSENSUS_TRIGGER_NEXT_VIEW) -} - func triggerDebugMessage(t *testing.T, node *shared.Node, action messaging.DebugMessageAction) { debugMessage := &messaging.DebugMessage{ Action: action, @@ -212,14 +215,18 @@ func triggerDebugMessage(t *testing.T, node *shared.Node, action messaging.Debug /*** P2P Helpers ***/ -func P2PBroadcast(_ *testing.T, nodes IdToNodeMapping, any *anypb.Any) { +func broadcast(t *testing.T, nodes idToNodeMapping, any *anypb.Any) { + t.Helper() + e := &messaging.PocketEnvelope{Content: any} for _, node := range nodes { node.GetBus().PublishEventToBus(e) } } -func P2PSend(_ *testing.T, node *shared.Node, any *anypb.Any) { +func send(t *testing.T, node *shared.Node, any *anypb.Any) { + t.Helper() + e := &messaging.PocketEnvelope{Content: any} node.GetBus().PublishEventToBus(e) } @@ -233,10 +240,10 @@ func P2PSend(_ *testing.T, node *shared.Node, any *anypb.Any) { // For example, if the test expects to receive 5 messages within 2 seconds: // false: continue if 5 messages are received in 0.5 seconds // true: wait for another 1.5 seconds after 5 messages are received in 0.5 seconds, and fail if any additional messages are received. -func WaitForNetworkConsensusEvents( +func waitForNetworkConsensusEvents( t *testing.T, clck *clock.Mock, - eventsChannel modules.EventsChannel, + sharedNetworkChannel modules.EventsChannel, step typesCons.HotstuffStep, msgType typesCons.HotstuffMessageType, numExpectedMsgs int, @@ -254,38 +261,44 @@ func WaitForNetworkConsensusEvents( } errMsg := fmt.Sprintf("HotStuff step: %s, type: %s", typesCons.HotstuffStep_name[int32(step)], typesCons.HotstuffMessageType_name[int32(msgType)]) - return waitForEventsInternal(clck, eventsChannel, messaging.HotstuffMessageContentType, numExpectedMsgs, millis, includeFilter, errMsg, failOnExtraMessages) + return waitForEventsInternal(clck, sharedNetworkChannel, messaging.HotstuffMessageContentType, numExpectedMsgs, millis, includeFilter, errMsg, failOnExtraMessages) } // IMPROVE: Consider unifying this function with WaitForNetworkConsensusEvents // This is a helper for 'waitForEventsInternal' that creates the `includeFilter` function based on state sync message specific parameters. -func WaitForNetworkStateSyncEvents( +// +//nolint:unparam // failOnExtraMessages will be varied in the future +func waitForNetworkStateSyncEvents( t *testing.T, clck *clock.Mock, - eventsChannel modules.EventsChannel, + sharedNetworkChannel modules.EventsChannel, errMsg string, numExpectedMsgs int, maxWaitTime time.Duration, failOnExtraMessages bool, + stateSyncMsgType any, ) (messages []*anypb.Any, err error) { includeFilter := func(anyMsg *anypb.Any) bool { msg, err := codec.GetCodec().FromAny(anyMsg) require.NoError(t, err) - _, ok := msg.(*typesCons.StateSyncMessage) + stateSyncMsg, ok := msg.(*typesCons.StateSyncMessage) require.True(t, ok) + if stateSyncMsgType != nil { + return reflect.TypeOf(stateSyncMsg.Message) == stateSyncMsgType + } return true } - return waitForEventsInternal(clck, eventsChannel, messaging.StateSyncMessageContentType, numExpectedMsgs, maxWaitTime, includeFilter, errMsg, failOnExtraMessages) + return waitForEventsInternal(clck, sharedNetworkChannel, messaging.StateSyncMessageContentType, numExpectedMsgs, maxWaitTime, includeFilter, errMsg, failOnExtraMessages) } // RESEARCH(#462): Research ways to eliminate time-based non-determinism from the test framework // IMPROVE: This function can be extended to testing events outside of just the consensus module. func waitForEventsInternal( clck *clock.Mock, - eventsChannel modules.EventsChannel, + sharedNetworkChannel modules.EventsChannel, eventContentType string, numExpectedMsgs int, maxWaitTime time.Duration, @@ -322,7 +335,7 @@ func waitForEventsInternal( loop: for { select { - case nodeEvent := <-eventsChannel: + case nodeEvent := <-sharedNetworkChannel: if nodeEvent.GetContentType() != eventContentType { unusedEvents = append(unusedEvents, nodeEvent) continue @@ -349,7 +362,7 @@ loop: if numRemainingMsgs == 0 { break loop } else if numRemainingMsgs > 0 { - return expectedMsgs, fmt.Errorf("Missing '%s' messages; %d expected but %d received. (%s) \n\t DO_NOT_SKIP_ME(#462): Consider increasing `maxWaitTime` as a workaround", eventContentType, numExpectedMsgs, len(expectedMsgs), errMsg) + return expectedMsgs, fmt.Errorf("Missing '%s' messages; %d expected but %d received. (%s) \n\t !!!IMPORTANT(#462)!!!: Consider increasing `maxWaitTime` as a workaround", eventContentType, numExpectedMsgs, len(expectedMsgs), errMsg) } else { return expectedMsgs, fmt.Errorf("Too many '%s' messages; %d expected but %d received. (%s)", eventContentType, numExpectedMsgs, len(expectedMsgs), errMsg) } @@ -357,7 +370,7 @@ loop: } for _, u := range unusedEvents { - eventsChannel <- u + sharedNetworkChannel <- u } return } @@ -365,7 +378,7 @@ loop: /*** Module Mocking Helpers ***/ // Creates a persistence module mock with mock implementations of some basic functionality -func basePersistenceMock(t *testing.T, _ modules.EventsChannel, bus modules.Bus) *mockModules.MockPersistenceModule { +func basePersistenceMock(t *testing.T, _ modules.EventsChannel, bus modules.Bus, testBlocks *testingBlocks) *mockModules.MockPersistenceModule { ctrl := gomock.NewController(t) persistenceMock := mockModules.NewMockPersistenceModule(ctrl) persistenceReadContextMock := mockModules.NewMockPersistenceReadContext(ctrl) @@ -374,19 +387,16 @@ func basePersistenceMock(t *testing.T, _ modules.EventsChannel, bus modules.Bus) persistenceMock.EXPECT().Start().Return(nil).AnyTimes() persistenceMock.EXPECT().SetBus(gomock.Any()).Return().AnyTimes() persistenceMock.EXPECT().NewReadContext(gomock.Any()).Return(persistenceReadContextMock, nil).AnyTimes() - persistenceMock.EXPECT().ReleaseWriteContext().Return(nil).AnyTimes() blockStoreMock := persistenceMocks.NewMockBlockStore(ctrl) - - blockStoreMock. - EXPECT(). - StoreBlock(gomock.Any(), gomock.Any()). - DoAndReturn(func(height []byte, block *coreTypes.Block) error { - return nil - }). - AnyTimes() - + blockStoreMock.EXPECT().Get(gomock.Any()).DoAndReturn(func(height []byte) ([]byte, error) { + heightInt := utils.HeightFromBytes(height) + if bus.GetConsensusModule().CurrentHeight() < heightInt { + return nil, fmt.Errorf("requested height is higher than current height of the node's consensus module") + } + return codec.GetCodec().Marshal(testBlocks.getBlock(heightInt)) + }).AnyTimes() blockStoreMock. // NB: The business logic in this mock and below is vital for testing state-sync end-to-end EXPECT(). @@ -395,15 +405,19 @@ func basePersistenceMock(t *testing.T, _ modules.EventsChannel, bus modules.Bus) if bus.GetConsensusModule().CurrentHeight() < height { return nil, fmt.Errorf("requested height is higher than current height of the node's consensus module") } - blockWithHeight := &coreTypes.Block{ - BlockHeader: &coreTypes.BlockHeader{ - Height: height, - }, - } - return blockWithHeight, nil + return testBlocks.getBlock(height), nil }). AnyTimes() + persistenceReadContextMock.EXPECT().GetMaximumBlockHeight().DoAndReturn(func() (uint64, error) { + // Check that we are retrieving a block at a height that was mocked by our test suite + if int(bus.GetConsensusModule().CurrentHeight()) <= len(testBlocks.blocks) { + return bus.GetConsensusModule().CurrentHeight() - 1, nil + } + t.Error("Trying to retrieve a block at a height that was not mocked.") + return 0, nil + }).AnyTimes() + persistenceMock. EXPECT(). GetBlockStore(). @@ -462,11 +476,13 @@ func basePersistenceMock(t *testing.T, _ modules.EventsChannel, bus modules.Bus) Release(). AnyTimes() + persistenceReadContextMock.EXPECT().GetValidatorExists(gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes() + return persistenceMock } // Creates a p2p module mock with mock implementations of some basic functionality -func baseP2PMock(t *testing.T, eventsChannel modules.EventsChannel) *mockModules.MockP2PModule { +func baseP2PMock(t *testing.T, sharedNetworkChannel modules.EventsChannel) *mockModules.MockP2PModule { ctrl := gomock.NewController(t) p2pMock := mockModules.NewMockP2PModule(ctrl) @@ -476,7 +492,7 @@ func baseP2PMock(t *testing.T, eventsChannel modules.EventsChannel) *mockModules Broadcast(gomock.Any()). Do(func(msg *anypb.Any) { e := &messaging.PocketEnvelope{Content: msg} - eventsChannel <- e + sharedNetworkChannel <- e }). AnyTimes() // CONSIDERATION: Adding a check to not to send message to itself @@ -484,7 +500,7 @@ func baseP2PMock(t *testing.T, eventsChannel modules.EventsChannel) *mockModules Send(gomock.Any(), gomock.Any()). Do(func(addr cryptoPocket.Address, msg *anypb.Any) { e := &messaging.PocketEnvelope{Content: msg} - eventsChannel <- e + sharedNetworkChannel <- e }). AnyTimes() p2pMock.EXPECT().GetModuleName().Return(modules.P2PModuleName).AnyTimes() @@ -509,7 +525,8 @@ func baseUtilityMock(t *testing.T, _ modules.EventsChannel, genesisState *genesi } return baseReplicaUtilityUnitOfWorkMock(t, genesisState), nil }). - MaxTimes(4) + AnyTimes() + utilityMock.EXPECT().GetModuleName().Return(modules.UtilityModuleName).AnyTimes() return utilityMock @@ -526,7 +543,7 @@ func baseLeaderUtilityUnitOfWorkMock(t *testing.T, genesisState *genesis.Genesis utilityLeaderUnitOfWorkMock.EXPECT(). CreateProposalBlock(gomock.Any(), maxTxBytes). - Return(stateHash, make([][]byte, 0), nil). + Return(dummyStateHash, make([][]byte, 0), nil). AnyTimes() utilityLeaderUnitOfWorkMock.EXPECT(). ApplyBlock(). @@ -534,7 +551,7 @@ func baseLeaderUtilityUnitOfWorkMock(t *testing.T, genesisState *genesis.Genesis AnyTimes() utilityLeaderUnitOfWorkMock.EXPECT(). GetStateHash(). - Return(stateHash). + Return(dummyStateHash). AnyTimes() utilityLeaderUnitOfWorkMock.EXPECT().SetProposalBlock(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() utilityLeaderUnitOfWorkMock.EXPECT().Commit(gomock.Any()).Return(nil).AnyTimes() @@ -558,7 +575,7 @@ func baseReplicaUtilityUnitOfWorkMock(t *testing.T, genesisState *genesis.Genesi AnyTimes() utilityReplicaUnitOfWorkMock.EXPECT(). GetStateHash(). - Return(stateHash). + Return(dummyStateHash). AnyTimes() utilityReplicaUnitOfWorkMock.EXPECT().SetProposalBlock(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() utilityReplicaUnitOfWorkMock.EXPECT().Commit(gomock.Any()).Return(nil).AnyTimes() @@ -595,8 +612,8 @@ func baseRpcMock(t *testing.T, _ modules.EventsChannel) *mockModules.MockRPCModu func WaitForNextBlock( t *testing.T, clck *clock.Mock, - eventsChannel modules.EventsChannel, - pocketNodes IdToNodeMapping, + sharedNetworkChannel modules.EventsChannel, + pocketNodes idToNodeMapping, height uint64, round uint8, maxWaitTime time.Duration, @@ -604,50 +621,54 @@ func WaitForNextBlock( ) *coreTypes.Block { leaderId := typesCons.NodeId(pocketNodes[1].GetBus().GetConsensusModule().GetLeaderForView(height, uint64(round), uint8(consensus.NewRound))) + // Debug message to start consensus by triggering first view change + triggerNextView(t, pocketNodes) + advanceTime(t, clck, 10*time.Millisecond) + // 1. NewRound - newRoundMessages, err := waitForProposalMsgs(t, clck, eventsChannel, pocketNodes, height, uint8(consensus.NewRound), round, 0, numValidators*numValidators, maxWaitTime, failOnExtraMessages) + newRoundMessages, err := waitForProposalMsgs(t, clck, sharedNetworkChannel, pocketNodes, height, uint8(consensus.NewRound), round, 0, numValidators*numValidators, maxWaitTime, failOnExtraMessages) require.NoError(t, err) broadcastMessages(t, newRoundMessages, pocketNodes) advanceTime(t, clck, 10*time.Millisecond) // 2. Prepare - prepareProposals, err := waitForProposalMsgs(t, clck, eventsChannel, pocketNodes, height, uint8(consensus.Prepare), round, leaderId, numValidators, maxWaitTime, failOnExtraMessages) + prepareProposals, err := waitForProposalMsgs(t, clck, sharedNetworkChannel, pocketNodes, height, uint8(consensus.Prepare), round, leaderId, numValidators, maxWaitTime, failOnExtraMessages) require.NoError(t, err) broadcastMessages(t, prepareProposals, pocketNodes) advanceTime(t, clck, 10*time.Millisecond) // wait for prepare votes - prepareVotes, err := WaitForNetworkConsensusEvents(t, clck, eventsChannel, 2, consensus.Vote, numValidators, maxWaitTime, failOnExtraMessages) + prepareVotes, err := waitForNetworkConsensusEvents(t, clck, sharedNetworkChannel, 2, consensus.Vote, numValidators, maxWaitTime, failOnExtraMessages) require.NoError(t, err) broadcastMessages(t, prepareVotes, pocketNodes) advanceTime(t, clck, 10*time.Millisecond) // 3. PreCommit - preCommitProposals, err := waitForProposalMsgs(t, clck, eventsChannel, pocketNodes, height, uint8(consensus.PreCommit), round, leaderId, numValidators, maxWaitTime, failOnExtraMessages) + preCommitProposals, err := waitForProposalMsgs(t, clck, sharedNetworkChannel, pocketNodes, height, uint8(consensus.PreCommit), round, leaderId, numValidators, maxWaitTime, failOnExtraMessages) require.NoError(t, err) broadcastMessages(t, preCommitProposals, pocketNodes) advanceTime(t, clck, 10*time.Millisecond) // wait for preCommit votes - preCommitVotes, err := WaitForNetworkConsensusEvents(t, clck, eventsChannel, 3, consensus.Vote, numValidators, maxWaitTime, failOnExtraMessages) + preCommitVotes, err := waitForNetworkConsensusEvents(t, clck, sharedNetworkChannel, 3, consensus.Vote, numValidators, maxWaitTime, failOnExtraMessages) require.NoError(t, err) broadcastMessages(t, preCommitVotes, pocketNodes) advanceTime(t, clck, 10*time.Millisecond) // 4. Commit - commitProposals, err := waitForProposalMsgs(t, clck, eventsChannel, pocketNodes, height, uint8(consensus.Commit), round, leaderId, numValidators, maxWaitTime, failOnExtraMessages) + commitProposals, err := waitForProposalMsgs(t, clck, sharedNetworkChannel, pocketNodes, height, uint8(consensus.Commit), round, leaderId, numValidators, maxWaitTime, failOnExtraMessages) require.NoError(t, err) broadcastMessages(t, commitProposals, pocketNodes) advanceTime(t, clck, 10*time.Millisecond) // wait for commit votes - commitVotes, err := WaitForNetworkConsensusEvents(t, clck, eventsChannel, 4, consensus.Vote, numValidators, maxWaitTime, failOnExtraMessages) + commitVotes, err := waitForNetworkConsensusEvents(t, clck, sharedNetworkChannel, 4, consensus.Vote, numValidators, maxWaitTime, failOnExtraMessages) require.NoError(t, err) broadcastMessages(t, commitVotes, pocketNodes) advanceTime(t, clck, 10*time.Millisecond) // 5. Decide - decideProposals, err := waitForProposalMsgs(t, clck, eventsChannel, pocketNodes, height, uint8(consensus.Decide), round, leaderId, numValidators, maxWaitTime, failOnExtraMessages) + decideProposals, err := waitForProposalMsgs(t, clck, sharedNetworkChannel, pocketNodes, height, uint8(consensus.Decide), round, leaderId, numValidators, maxWaitTime, failOnExtraMessages) require.NoError(t, err) broadcastMessages(t, decideProposals, pocketNodes) advanceTime(t, clck, 10*time.Millisecond) @@ -662,8 +683,8 @@ func WaitForNextBlock( func waitForProposalMsgs( t *testing.T, clck *clock.Mock, - eventsChannel modules.EventsChannel, - pocketNodes IdToNodeMapping, + sharedNetworkChannel modules.EventsChannel, + pocketNodes idToNodeMapping, height uint64, step uint8, round uint8, @@ -672,13 +693,13 @@ func waitForProposalMsgs( maxWaitTime time.Duration, failOnExtraMessages bool, ) ([]*anypb.Any, error) { - proposalMsgs, err := WaitForNetworkConsensusEvents(t, clck, eventsChannel, typesCons.HotstuffStep(step), consensus.Propose, numExpectedMsgs, maxWaitTime, failOnExtraMessages) + proposalMsgs, err := waitForNetworkConsensusEvents(t, clck, sharedNetworkChannel, typesCons.HotstuffStep(step), consensus.Propose, numExpectedMsgs, maxWaitTime, failOnExtraMessages) if err != nil { return nil, err } for nodeId, pocketNode := range pocketNodes { - nodeState := GetConsensusNodeState(pocketNode) + nodeState := getConsensusNodeState(pocketNode) if (typesCons.HotstuffStep(step) == consensus.Decide) && (nodeId == leaderId) { assertNodeConsensusView(t, nodeId, typesCons.ConsensusNodeState{ @@ -702,90 +723,22 @@ func waitForProposalMsgs( return proposalMsgs, nil } -func broadcastMessages(t *testing.T, msgs []*anypb.Any, pocketNodes IdToNodeMapping) { +func broadcastMessages(t *testing.T, msgs []*anypb.Any, pocketNodes idToNodeMapping) { for _, message := range msgs { - P2PBroadcast(t, pocketNodes, message) + broadcast(t, pocketNodes, message) } } -// WaitForNodeToSync waits for a node to sync to a target height -// For every missing block for the unsynced node: -// -// first, waits for the node to request a missing block via `waitForNodeToRequestMissingBlock()` function, -// then, waits for the node to receive the missing block via `waitForNodeToReceiveMissingBlock()` function, -// finally, wait for the node to catch up to the target height via `waitForNodeToCatchUp()` function. -func WaitForNodeToSync( - t *testing.T, - clck *clock.Mock, - eventsChannel modules.EventsChannel, - unsyncedNode *shared.Node, - allNodes IdToNodeMapping, - targetHeight uint64, -) error { - currentHeight := unsyncedNode.GetBus().GetConsensusModule().CurrentHeight() - - for i := currentHeight; i <= targetHeight; i++ { - blockRequest, err := waitForNodeToRequestMissingBlock(t, clck, eventsChannel, allNodes, currentHeight, targetHeight) - if err != nil { - return err - } - - blockResponse, err := waitForNodeToReceiveMissingBlock(t, clck, eventsChannel, allNodes, blockRequest) - if err != nil { - return err - } - - err = waitForNodeToCatchUp(t, clck, eventsChannel, unsyncedNode, blockResponse, targetHeight) - if err != nil { - return err - } +func triggerNextView(t *testing.T, pocketNodes idToNodeMapping) { + for _, node := range pocketNodes { + triggerDebugMessage(t, node, messaging.DebugMessageAction_DEBUG_CONSENSUS_TRIGGER_NEXT_VIEW) } - return nil -} - -// TODO(#352): implement this function. -// waitForNodeToRequestMissingBlock waits for unsynced node to request missing block form the network -func waitForNodeToRequestMissingBlock( - t *testing.T, - clck *clock.Mock, - eventsChannel modules.EventsChannel, - allNodes IdToNodeMapping, - startingHeight uint64, - targetHeight uint64, -) (*anypb.Any, error) { - return &anypb.Any{}, nil -} - -// TODO(#352): implement this function. -// waitForNodeToReceiveMissingBlock requests block request of the unsynced node -// for given node to node to catch up to the target height by sending the requested block. -func waitForNodeToReceiveMissingBlock( - t *testing.T, - clck *clock.Mock, - eventsChannel modules.EventsChannel, - allNodes IdToNodeMapping, - blockReq *anypb.Any, -) (*anypb.Any, error) { - return &anypb.Any{}, nil -} - -// TODO(#352): implement this function. -// waitForNodeToCatchUp waits for given node to node to catch up to the target height by sending the requested block. -func waitForNodeToCatchUp( - t *testing.T, - clck *clock.Mock, - eventsChannel modules.EventsChannel, - unsyncedNode *shared.Node, - blockResponse *anypb.Any, - targetHeight uint64, -) error { - return nil } func generatePlaceholderBlock(height uint64, leaderAddrr cryptoPocket.Address) *coreTypes.Block { blockHeader := &coreTypes.BlockHeader{ Height: height, - StateHash: stateHash, + StateHash: dummyStateHash, PrevStateHash: "", ProposerAddress: leaderAddrr, QuorumCertificate: nil, @@ -821,6 +774,98 @@ func baseLoggerMock(t *testing.T, _ modules.EventsChannel) *mockModules.MockLogg return loggerMock } +/*** Placeholder Block Generation Helpers ***/ + +type testingBlocks struct { + blocks []*coreTypes.Block +} + +func (p *testingBlocks) getBlock(index uint64) *coreTypes.Block { + // returning block at index-1, because block 1 is stored at index 0 of the blocks array + return p.blocks[index-1] +} + +func (p *testingBlocks) preparePlaceholderBlocks(t *testing.T, bus modules.Bus, validatorPrivKeys idToPrivKeyMapping, numMockedBlocks uint64) { + t.Helper() + for i := uint64(1); i <= numMockedBlocks; i++ { + leaderId := bus.GetConsensusModule().GetLeaderForView(i, uint64(0), uint8(consensus.NewRound)) + leaderPivKey := validatorPrivKeys[typesCons.NodeId(leaderId)] + + // Construct the block + blockHeader := &coreTypes.BlockHeader{ + Height: i, + StateHash: dummyStateHash, + PrevStateHash: dummyStateHash, + ProposerAddress: leaderPivKey.Address(), + QuorumCertificate: nil, // inserted below + } + block := &coreTypes.Block{ + BlockHeader: blockHeader, + Transactions: make([][]byte, 0), // we don't care about the transactions in this context + } + + qc := generateQuorumCertificate(t, validatorPrivKeys, block) + qcBytes, err := codec.GetCodec().Marshal(qc) + require.NoError(t, err) + + block.BlockHeader.QuorumCertificate = qcBytes + + p.blocks = append(p.blocks, block) + } +} + +/*** Quorum certificate Generation Helpers ***/ + +func generateQuorumCertificate(t *testing.T, validatorPrivKeys idToPrivKeyMapping, block *coreTypes.Block) *typesCons.QuorumCertificate { + // Aggregate partial signatures + var pss []*typesCons.PartialSignature + for _, validatorPrivKey := range validatorPrivKeys { + pss = append(pss, generatePartialSignature(t, block, validatorPrivKey)) + } + + // Generate threshold signature + thresholdSig := new(typesCons.ThresholdSignature) + thresholdSig.Signatures = make([]*typesCons.PartialSignature, len(pss)) + copy(thresholdSig.Signatures, pss) + + return &typesCons.QuorumCertificate{ + Height: block.BlockHeader.Height, + Round: 1, // assume everything succeeds on the first round for now + Step: consensus.NewRound, + Block: block, + ThresholdSignature: thresholdSig, + } +} + +// generate partial signature for the validator +func generatePartialSignature(t *testing.T, block *coreTypes.Block, validatorPrivKey cryptoPocket.PrivateKey) *typesCons.PartialSignature { + return &typesCons.PartialSignature{ + Signature: getMessageSignature(t, block, validatorPrivKey), + Address: validatorPrivKey.PublicKey().Address().String(), + } +} + +// Generates partial signature with given private key +func getMessageSignature(t *testing.T, block *coreTypes.Block, privKey cryptoPocket.PrivateKey) []byte { + // Signature only over subset of fields in HotstuffMessage + // For reference, see section 4.3 of the the hotstuff whitepaper, partial signatures are + // computed over `tsignr(hm.type, m.viewNumber , m.nodei)`. https://arxiv.org/pdf/1803.05069.pdf + msgToSign := &typesCons.HotstuffMessage{ + Height: block.BlockHeader.Height, + Step: 1, + Round: 1, + Block: block, + } + + bytesToSign, err := codec.GetCodec().Marshal(msgToSign) + require.NoError(t, err) + + signature, err := privKey.Sign(bytesToSign) + require.NoError(t, err) + + return signature +} + func logTime(t *testing.T, clck *clock.Mock) { t.Helper() defer func() { @@ -885,3 +930,37 @@ func startNode(t *testing.T, pocketNode *shared.Node) { err := pocketNode.Start() require.NoError(t, err) } + +func prepareStateSyncGetBlockMessage(t *testing.T, peerAddress string, height uint64) *anypb.Any { + t.Helper() + + stateSyncGetBlockMessage := &typesCons.StateSyncMessage{ + Message: &typesCons.StateSyncMessage_GetBlockReq{ + GetBlockReq: &typesCons.GetBlockRequest{ + PeerAddress: peerAddress, + Height: height, + }, + }, + } + + anyProto, err := anypb.New(stateSyncGetBlockMessage) + require.NoError(t, err) + + return anyProto +} + +func prepareStateSyncGetMetadataMessage(t *testing.T, selfAddress string) *anypb.Any { + t.Helper() + + stateSyncMetaDataReqMessage := &typesCons.StateSyncMessage{ + Message: &typesCons.StateSyncMessage_MetadataReq{ + MetadataReq: &typesCons.StateSyncMetadataRequest{ + PeerAddress: selfAddress, + }, + }, + } + anyProto, err := anypb.New(stateSyncMetaDataReqMessage) + require.NoError(t, err) + + return anyProto +} diff --git a/consensus/event_handler.go b/consensus/event_handler.go new file mode 100644 index 000000000..51e7c89f0 --- /dev/null +++ b/consensus/event_handler.go @@ -0,0 +1,126 @@ +package consensus + +import ( + "fmt" + + typesCons "github.com/pokt-network/pocket/consensus/types" + "github.com/pokt-network/pocket/shared/codec" + coreTypes "github.com/pokt-network/pocket/shared/core/types" + "github.com/pokt-network/pocket/shared/messaging" + "google.golang.org/protobuf/types/known/anypb" +) + +// RESEARCH(#816): Research whether the E2E state sync business logic can be simplified by not using the FSM module at all. +// We were originally intending to make heavier use of the FSM module to handle state sync, but we ended up not using it much as +// seen below. + +const ( + consensusFSMHandlerSource = "ConsensusFSMHandler" +) + +// Implements the `HandleEvent` function in the `ConsensusModule` interface +func (m *consensusModule) HandleEvent(event *anypb.Any) error { + m.m.Lock() + defer m.m.Unlock() + + msg, err := codec.GetCodec().FromAny(event) + if err != nil { + return err + } + + switch event.MessageName() { + + case messaging.StateMachineTransitionEventType: + stateTransitionMessage, ok := msg.(*messaging.StateMachineTransitionEvent) + if !ok { + return fmt.Errorf("failed to cast message to StateSyncMessage") + } + return m.handleStateTransitionEvent(stateTransitionMessage) + + case messaging.ConsensusNewHeightEventType: + blockCommittedEvent, ok := msg.(*messaging.ConsensusNewHeightEvent) + if !ok { + return fmt.Errorf("failed to cast event to ConsensusNewHeightEvent") + } + m.stateSync.HandleBlockCommittedEvent(blockCommittedEvent) + return nil + default: + return typesCons.ErrUnknownStateSyncMessageType(event.MessageName()) + } +} + +// handleStateTransitionEvent handles the state transition event from the state machine module +func (m *consensusModule) handleStateTransitionEvent(msg *messaging.StateMachineTransitionEvent) error { + m.logger.Info().Fields(messaging.TransitionEventToMap(msg)).Msg("Received state machine transition msg") + + switch coreTypes.StateMachineState(msg.NewState) { + case coreTypes.StateMachineState_P2P_Bootstrapped: + return m.HandleBootstrapped(msg) + + case coreTypes.StateMachineState_Consensus_Unsynced: + return m.HandleUnsynced(msg) + + case coreTypes.StateMachineState_Consensus_SyncMode: + return m.HandleSyncMode(msg) + + case coreTypes.StateMachineState_Consensus_Synced: + return m.HandleSynced(msg) + + case coreTypes.StateMachineState_Consensus_Pacemaker: + return m.HandlePacemaker(msg) + + default: + m.logger.Warn().Msgf("Consensus module not handling this event: %s", msg.Event) + + } + + return nil +} + +// HandleBootstrapped handles the FSM event P2P_IsBootstrapped, and when P2P_Bootstrapped is the destination state. +// Bootstrapped mode is when the node (validator or non) is first coming online. +// This is a transition mode from node bootstrapping to a node being out-of-sync. +func (m *consensusModule) HandleBootstrapped(msg *messaging.StateMachineTransitionEvent) error { + m.logger.Info().Str("source", consensusFSMHandlerSource).Msg("Node is in the bootstrapped state. Transitioning to IsUnsynched mode...") + return m.GetBus().GetStateMachineModule().SendEvent(coreTypes.StateMachineEvent_Consensus_IsUnsynced) +} + +// HandleUnsynced handles the FSM event Consensus_IsUnsynced, and when Unsynced is the destination state. +// In Unsynced mode, the node (validator or not) is out of sync with the rest of the network. +// This mode is a transition mode from the node being up-to-date (i.e. Pacemaker mode, Synced mode) with the latest network height to being out-of-sync. +// As soon as a node transitions to this mode, it will transition to the synching mode. +func (m *consensusModule) HandleUnsynced(msg *messaging.StateMachineTransitionEvent) error { + m.logger.Info().Str("source", consensusFSMHandlerSource).Msg("Node is in an Unsynced state. Transitioning to IsSyncing mode...") + return m.GetBus().GetStateMachineModule().SendEvent(coreTypes.StateMachineEvent_Consensus_IsSyncing) +} + +// HandleSyncMode handles the FSM event Consensus_IsSyncing, and when SyncMode is the destination state. +// In Sync mode, the node (validator or not starts syncing with the rest of the network. +func (m *consensusModule) HandleSyncMode(msg *messaging.StateMachineTransitionEvent) error { + m.logger.Info().Str("source", consensusFSMHandlerSource).Msg("Node is in Sync Mode. About to start synchronous sync loop...") + go m.stateSync.StartSynchronousStateSync() + m.logger.Info().Str("source", consensusFSMHandlerSource).Msg("Node is in Sync Mode. Finished synchronous sync loop!!!") + return nil +} + +// HandleSynced handles the FSM event IsSyncedNonValidator for Non-Validators, and Synced is the destination state. +// Currently, FSM never transition to this state and a non-validator node always stays in SyncMode. +// CONSIDER: when a non-validator sync is implemented, maybe there is a case that requires transitioning to this state. +func (m *consensusModule) HandleSynced(msg *messaging.StateMachineTransitionEvent) error { + m.logger.Info().Str("source", consensusFSMHandlerSource).Msg("Node (non-validator) is Synced. NOOP") + return nil +} + +// HandlePacemaker handles the FSM event IsSyncedValidator, and Pacemaker is the destination state. +// Execution of this state means the validator node is synced and it will stay in this mode until +// it receives a new block proposal that has a higher height than the current consensus height. +func (m *consensusModule) HandlePacemaker(msg *messaging.StateMachineTransitionEvent) error { + m.logger.Info().Str("source", consensusFSMHandlerSource).Msg("Node (validator) node is Synced and entering Pacemaker mode. About to starting participating in consensus...") + + // if a validator is just bootstrapped and finished state sync, it will not have a nodeId yet, which is 0. Set correct nodeId here. + if m.nodeId == 0 { + return m.updateNodeId() + } + + return nil +} diff --git a/consensus/fsm_handler.go b/consensus/fsm_handler.go deleted file mode 100644 index 1a410c477..000000000 --- a/consensus/fsm_handler.go +++ /dev/null @@ -1,106 +0,0 @@ -package consensus - -import ( - "fmt" - - typesCons "github.com/pokt-network/pocket/consensus/types" - "github.com/pokt-network/pocket/shared/codec" - coreTypes "github.com/pokt-network/pocket/shared/core/types" - "github.com/pokt-network/pocket/shared/messaging" - "google.golang.org/protobuf/types/known/anypb" -) - -// HandleEvent handles FSM state transition events. -func (m *consensusModule) HandleEvent(transitionMessageAny *anypb.Any) error { - m.m.Lock() - defer m.m.Unlock() - - switch transitionMessageAny.MessageName() { - case messaging.StateMachineTransitionEventType: - msg, err := codec.GetCodec().FromAny(transitionMessageAny) - if err != nil { - return err - } - - stateTransitionMessage, ok := msg.(*messaging.StateMachineTransitionEvent) - if !ok { - return fmt.Errorf("failed to cast message to StateSyncMessage") - } - return m.handleStateTransitionEvent(stateTransitionMessage) - default: - return typesCons.ErrUnknownStateSyncMessageType(transitionMessageAny.MessageName()) - } -} - -func (m *consensusModule) handleStateTransitionEvent(msg *messaging.StateMachineTransitionEvent) error { - fsm_state := msg.NewState - - m.logger.Debug().Fields(messaging.TransitionEventToMap(msg)).Msg("Received state machine transition msg") - - switch coreTypes.StateMachineState(fsm_state) { - case coreTypes.StateMachineState_P2P_Bootstrapped: - return m.HandleBootstrapped(msg) - - case coreTypes.StateMachineState_Consensus_Unsynced: - return m.HandleUnsynced(msg) - - case coreTypes.StateMachineState_Consensus_SyncMode: - return m.HandleSyncMode(msg) - - case coreTypes.StateMachineState_Consensus_Synced: - return m.HandleSynced(msg) - - case coreTypes.StateMachineState_Consensus_Pacemaker: - return m.HandlePacemaker(msg) - - default: - m.logger.Warn().Msgf("Consensus module not handling this event: %s", msg.Event) - - } - - return nil -} - -// HandleBootstrapped handles FSM event P2P_IsBootstrapped, and P2P_Bootstrapped is the destination state. -// Bootrstapped mode is when the node (validator or non-validator) is first coming online. -// This is a transition mode from node bootstrapping to a node being out-of-sync. -func (m *consensusModule) HandleBootstrapped(msg *messaging.StateMachineTransitionEvent) error { - m.logger.Debug().Msg("Node is in bootstrapped state") - return nil -} - -// HandleUnsynced handles FSM event Consensus_IsUnsynced, and Unsynced is the destination state. -// In Unsynced mode node (validator or non-validator) is out of sync with the rest of the network. -// This mode is a transition mode from the node being up-to-date (i.e. Pacemaker mode, Synced mode) with the latest network height to being out-of-sync. -// As soon as node transitions to this mode, it will transition to the sync mode. -func (m *consensusModule) HandleUnsynced(msg *messaging.StateMachineTransitionEvent) error { - m.logger.Debug().Msg("Node is in Unsyched state, as node is out of sync sending syncmode event to start syncing") - - return m.GetBus().GetStateMachineModule().SendEvent(coreTypes.StateMachineEvent_Consensus_IsSyncing) -} - -// HandleSyncMode handles FSM event Consensus_IsSyncing, and SyncMode is the destination state. -// In Sync mode node (validator or non-validator) starts syncing with the rest of the network. -func (m *consensusModule) HandleSyncMode(msg *messaging.StateMachineTransitionEvent) error { - m.logger.Debug().Msg("Node is in Sync Mode, starting to sync...") - - return m.stateSync.Start() -} - -// HandleSynced handles FSM event IsSyncedNonValidator for Non-Validators, and Synced is the destination state. -// Currently, FSM never transition to this state and a non-validator node always stays in syncmode. -// CONSIDER: when a non-validator sync is implemented, maybe there is a case that requires transitioning to this state. -// TODO: Add check that this never happens when IsValidator() is false, i.e. node is not validator. -func (m *consensusModule) HandleSynced(msg *messaging.StateMachineTransitionEvent) error { - m.logger.Debug().Msg("Non-validator node is in Synced mode") - return nil -} - -// HandlePacemaker handles FSM event IsSyncedValidator, and Pacemaker is the destination state. -// Execution of this state means the validator node is synced. -func (m *consensusModule) HandlePacemaker(msg *messaging.StateMachineTransitionEvent) error { - m.logger.Debug().Msg("Validator node is Synced and in Pacemaker mode. It will stay in this mode until it receives a new block proposal that has a higher height than the current block height") - // validator receives a new block proposal, and it understands that it doesn't have block and it transitions to unsycnhed state - // transitioning out of this state happens when a new block proposal is received by the hotstuff_replica - return nil -} diff --git a/consensus/helpers.go b/consensus/helpers.go index f3ef915f3..05c4bf82b 100644 --- a/consensus/helpers.go +++ b/consensus/helpers.go @@ -177,7 +177,6 @@ func (m *consensusModule) sendToLeader(msg *typesCons.HotstuffMessage) { } // Star-like (O(n)) broadcast - send to all nodes directly -// INVESTIGATE: Re-evaluate if we should be using our structured broadcast (RainTree O(log3(n))) algorithm instead func (m *consensusModule) broadcastToValidators(msg *typesCons.HotstuffMessage) { m.logger.Info().Fields(hotstuffMsgToLoggingFields(msg)).Msg("📣 Broadcasting message 📣") @@ -187,11 +186,11 @@ func (m *consensusModule) broadcastToValidators(msg *typesCons.HotstuffMessage) return } + // Not using Broadcast because this is a direct message to all validators only validators, err := m.getValidatorsAtHeight(m.CurrentHeight()) if err != nil { m.logger.Error().Err(err).Msg(typesCons.ErrPersistenceGetAllValidators.Error()) } - for _, val := range validators { if err := m.GetBus().GetP2PModule().Send(cryptoPocket.AddressFromString(val.GetAddress()), anyConsensusMessage); err != nil { m.logger.Error().Err(err).Msg(typesCons.ErrBroadcastMessage.Error()) @@ -271,22 +270,6 @@ func (m *consensusModule) getValidatorsAtHeight(height uint64) ([]*coreTypes.Act return readCtx.GetAllValidators(int64(height)) } -// TODO: This is a temporary solution, cache this in Consensus module. This field will be populated once with a single query to the persistence module. -func (m *consensusModule) IsValidator() (bool, error) { - validators, err := m.getValidatorsAtHeight(m.CurrentHeight()) - if err != nil { - return false, err - } - - for _, actor := range validators { - if actor.Address == m.nodeAddress { - return true, nil - } - } - - return false, nil -} - func hotstuffMsgToLoggingFields(msg *typesCons.HotstuffMessage) map[string]any { return map[string]any{ "height": msg.GetHeight(), @@ -294,3 +277,19 @@ func hotstuffMsgToLoggingFields(msg *typesCons.HotstuffMessage) map[string]any { "step": typesCons.StepToString[msg.GetStep()], } } + +func (m *consensusModule) maxPersistedBlockHeight() (uint64, error) { + // TECHDEBT: We don't need to pass the height here to retrieve the maximum block height. + readCtx, err := m.GetBus().GetPersistenceModule().NewReadContext(int64(m.CurrentHeight())) + if err != nil { + return 0, err + } + defer readCtx.Release() + + maxHeight, err := readCtx.GetMaximumBlockHeight() + if err != nil { + return 0, err + } + + return maxHeight, nil +} diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index 1a3520870..aa478e944 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -30,7 +30,7 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *consensusM defer m.paceMaker.RestartTimer() handler.emitTelemetryEvent(m, msg) - if err := handler.anteHandle(m, msg); err != nil { + if err := handler.isMessageValidBasic(m, msg); err != nil { m.logger.Error().Msg(typesCons.ErrHotstuffValidation.Error()) return } @@ -108,7 +108,7 @@ func (handler *HotstuffLeaderMessageHandler) HandlePrepareMessage(m *consensusMo defer m.paceMaker.RestartTimer() handler.emitTelemetryEvent(m, msg) - if err := handler.anteHandle(m, msg); err != nil { + if err := handler.isMessageValidBasic(m, msg); err != nil { m.logger.Error().Msg(typesCons.ErrHotstuffValidation.Error()) return } @@ -159,7 +159,7 @@ func (handler *HotstuffLeaderMessageHandler) HandlePrecommitMessage(m *consensus defer m.paceMaker.RestartTimer() handler.emitTelemetryEvent(m, msg) - if err := handler.anteHandle(m, msg); err != nil { + if err := handler.isMessageValidBasic(m, msg); err != nil { m.logger.Error().Err(err).Msg(typesCons.ErrHotstuffValidation.Error()) return } @@ -210,7 +210,7 @@ func (handler *HotstuffLeaderMessageHandler) HandleCommitMessage(m *consensusMod defer m.paceMaker.RestartTimer() handler.emitTelemetryEvent(m, msg) - if err := handler.anteHandle(m, msg); err != nil { + if err := handler.isMessageValidBasic(m, msg); err != nil { m.logger.Error().Err(err).Msg(typesCons.ErrHotstuffValidation.Error()) return } @@ -273,17 +273,17 @@ func (handler *HotstuffLeaderMessageHandler) HandleDecideMessage(m *consensusMod defer m.paceMaker.RestartTimer() handler.emitTelemetryEvent(m, msg) - if err := handler.anteHandle(m, msg); err != nil { + if err := handler.isMessageValidBasic(m, msg); err != nil { m.logger.Error().Err(err).Msg(typesCons.ErrHotstuffValidation.Error()) return } } -// anteHandle is the general handler called for every before every specific HotstuffLeaderMessageHandler handler -func (handler *HotstuffLeaderMessageHandler) anteHandle(m *consensusModule, msg *typesCons.HotstuffMessage) error { +// isMessageValidBasic is the general handler called for every before every specific HotstuffLeaderMessageHandler handler +func (handler *HotstuffLeaderMessageHandler) isMessageValidBasic(m *consensusModule, msg *typesCons.HotstuffMessage) error { // Basic block metadata validation - if valid, err := m.isValidMessageBlock(msg); !valid { + if valid, err := m.isBlockInMessageValidBasic(msg); !valid { return err } diff --git a/consensus/hotstuff_replica.go b/consensus/hotstuff_replica.go index 4b86bf113..15df2f262 100644 --- a/consensus/hotstuff_replica.go +++ b/consensus/hotstuff_replica.go @@ -28,7 +28,7 @@ func (handler *HotstuffReplicaMessageHandler) HandleNewRoundMessage(m *consensus defer m.paceMaker.RestartTimer() handler.emitTelemetryEvent(m, msg) - if err := handler.anteHandle(m, msg); err != nil { + if err := handler.isMessageValidBasic(m, msg); err != nil { m.logger.Error().Err(err).Msg(typesCons.ErrHotstuffValidation.Error()) return } @@ -48,7 +48,7 @@ func (handler *HotstuffReplicaMessageHandler) HandlePrepareMessage(m *consensusM defer m.paceMaker.RestartTimer() handler.emitTelemetryEvent(m, msg) - if err := handler.anteHandle(m, msg); err != nil { + if err := handler.isMessageValidBasic(m, msg); err != nil { m.logger.Error().Err(err).Msg(typesCons.ErrHotstuffValidation.Error()) return } @@ -82,7 +82,7 @@ func (handler *HotstuffReplicaMessageHandler) HandlePrecommitMessage(m *consensu defer m.paceMaker.RestartTimer() handler.emitTelemetryEvent(m, msg) - if err := handler.anteHandle(m, msg); err != nil { + if err := handler.isMessageValidBasic(m, msg); err != nil { m.logger.Error().Err(err).Msg(typesCons.ErrHotstuffValidation.Error()) return } @@ -111,7 +111,7 @@ func (handler *HotstuffReplicaMessageHandler) HandleCommitMessage(m *consensusMo defer m.paceMaker.RestartTimer() handler.emitTelemetryEvent(m, msg) - if err := handler.anteHandle(m, msg); err != nil { + if err := handler.isMessageValidBasic(m, msg); err != nil { m.logger.Error().Err(err).Msg(typesCons.ErrHotstuffValidation.Error()) return } @@ -140,7 +140,7 @@ func (handler *HotstuffReplicaMessageHandler) HandleDecideMessage(m *consensusMo defer m.paceMaker.RestartTimer() handler.emitTelemetryEvent(m, msg) - if err := handler.anteHandle(m, msg); err != nil { + if err := handler.isMessageValidBasic(m, msg); err != nil { m.logger.Error().Err(err).Msg(typesCons.ErrHotstuffValidation.Error()) return } @@ -157,21 +157,22 @@ func (handler *HotstuffReplicaMessageHandler) HandleDecideMessage(m *consensusMo m.logger.Error().Err(err).Msg("Failed to convert the quorum certificate to bytes") return } + m.block.BlockHeader.QuorumCertificate = quorumCertBytes if err := m.commitBlock(m.block); err != nil { m.logger.Error().Err(err).Msg("Could not commit block") m.paceMaker.InterruptRound("failed to commit block") - return + return } m.paceMaker.NewHeight() } -// anteHandle is the handler called on every replica message before specific handler -func (handler *HotstuffReplicaMessageHandler) anteHandle(m *consensusModule, msg *typesCons.HotstuffMessage) error { +// isMessageValidBasic is the handler called on every replica message before specific handler +func (handler *HotstuffReplicaMessageHandler) isMessageValidBasic(m *consensusModule, msg *typesCons.HotstuffMessage) error { // Basic block metadata validation - if valid, err := m.isValidMessageBlock(msg); !valid { + if valid, err := m.isBlockInMessageValidBasic(msg); !valid { return err } diff --git a/consensus/leader_election/module.go b/consensus/leader_election/module.go index 25a38024c..b1e35be2d 100644 --- a/consensus/leader_election/module.go +++ b/consensus/leader_election/module.go @@ -58,7 +58,7 @@ func (m *leaderElectionModule) electNextLeaderDeterministicRoundRobin(message *t return typesCons.NodeId(0), err } - value := int64(message.Height) + int64(message.Round) + int64(message.Step) - 1 + value := int64(message.Height) + int64(message.Round) - 1 numVals := int64(len(vals)) return typesCons.NodeId(value%numVals + 1), nil diff --git a/consensus/module.go b/consensus/module.go index 2c1697aef..0ce5a8590 100644 --- a/consensus/module.go +++ b/consensus/module.go @@ -25,27 +25,23 @@ import ( var _ modules.ConsensusModule = &consensusModule{} -// TODO: This should be configurable -const ( - metadataChannelSize = 1000 - blocksChannelSize = 1000 -) - type consensusModule struct { base_modules.IntegrableModule - privateKey cryptoPocket.Ed25519PrivateKey + logger *modules.Logger + // General configs consCfg *configs.ConsensusConfig genesisState *genesis.GenesisState - logger *modules.Logger + // The key used for participating in consensus + privateKey cryptoPocket.Ed25519PrivateKey + nodeAddress string // m is a mutex used to control synchronization when multiple goroutines are accessing the struct and its fields / properties. - // - // The idea is that you want to acquire a Lock when you are writing values and a RLock when you want to make sure that no other goroutine is changing the values you are trying to read concurrently. - // - // Locking context should be the smallest possible but not smaller than a single "unit of work". + // The idea is that you want to acquire a Lock when you are writing values and a RLock when you want to make sure that no other + // goroutine is changing the values you are trying to read concurrently. Locking context should be the smallest possible but not + // smaller than a single "unit of work". m sync.RWMutex // Hotstuff @@ -53,9 +49,11 @@ type consensusModule struct { round uint64 step typesCons.HotstuffStep block *coreTypes.Block // The current block being proposed / voted on; it has not been committed to finality - // TODO(#315): Move the statefulness of `IndexedTransaction` to the persistence module - IndexedTransactions []coreTypes.IndexedTransaction // The current block applied transaction results / voted on; it has not been committed to finality + // Stores messages aggregated during a single consensus round from other validators + hotstuffMempool map[typesCons.HotstuffStep]*hotstuffFIFOMempool + + // Hotstuff safety prepareQC *typesCons.QuorumCertificate // Highest QC for which replica voted PRECOMMIT lockedQC *typesCons.QuorumCertificate // Highest QC for which replica voted COMMIT @@ -63,27 +61,13 @@ type consensusModule struct { leaderId *typesCons.NodeId nodeId typesCons.NodeId - nodeAddress string - // Module Dependencies - // IMPROVE(#283): Investigate whether the current approach to how the `utilityUnitOfWork` should be - // managed or changed. Also consider exposing a function that exposes the context - // to streamline how its accessed in the module (see the ticket). utilityUnitOfWork modules.UtilityUnitOfWork paceMaker pacemaker.Pacemaker leaderElectionMod leader_election.LeaderElectionModule + // State Sync stateSync state_sync.StateSyncModule - - hotstuffMempool map[typesCons.HotstuffStep]*hotstuffFIFOMempool - - // block responses received from peers are collected in this channel - blocksReceived chan *typesCons.GetBlockResponse - - // metadata responses received from peers are collected in this channel - metadataReceived chan *typesCons.StateSyncMetadataResponse - - serverModeEnabled bool } func Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { @@ -96,13 +80,13 @@ func (*consensusModule) Create(bus modules.Bus, options ...modules.ModuleOption) return nil, err } - paceMakerMod, err := pacemaker.CreatePacemaker(bus) + paceMakerMod, err := pacemaker.Create(bus) if err != nil { return nil, err } pm := paceMakerMod.(pacemaker.Pacemaker) - stateSyncMod, err := state_sync.CreateStateSync(bus) + stateSyncMod, err := state_sync.Create(bus) if err != nil { return nil, err } @@ -130,7 +114,6 @@ func (*consensusModule) Create(bus modules.Bus, options ...modules.ModuleOption) for _, option := range options { option(m) } - bus.RegisterModule(m) // Ensure `CurrentHeightProvider` submodule is registered. @@ -139,39 +122,26 @@ func (*consensusModule) Create(bus modules.Bus, options ...modules.ModuleOption) } runtimeMgr := bus.GetRuntimeMgr() - - consensusCfg := runtimeMgr.GetConfig().Consensus - - m.serverModeEnabled = consensusCfg.ServerModeEnabled + m.consCfg = runtimeMgr.GetConfig().Consensus genesisState := runtimeMgr.GetGenesis() if err := m.ValidateGenesis(genesisState); err != nil { return nil, fmt.Errorf("genesis validation failed: %w", err) } + m.genesisState = genesisState - privateKey, err := cryptoPocket.NewPrivateKey(consensusCfg.GetPrivateKey()) + // TECHDEBT: Should we use the same private key everywhere (top level config, consensus config, etc...) or should we consolidate them? + privateKey, err := cryptoPocket.NewPrivateKey(m.consCfg.GetPrivateKey()) if err != nil { return nil, err } - address := privateKey.Address().String() + m.privateKey = privateKey.(cryptoPocket.Ed25519PrivateKey) - validators, err := m.getValidatorsAtHeight(m.CurrentHeight()) - if err != nil { + m.nodeAddress = privateKey.Address().String() + if m.updateNodeId() != nil { return nil, err } - valAddrToIdMap := typesCons.NewActorMapper(validators).GetValAddrToIdMap() - - m.privateKey = privateKey.(cryptoPocket.Ed25519PrivateKey) - m.consCfg = consensusCfg - m.genesisState = genesisState - - m.nodeId = valAddrToIdMap[address] - m.nodeAddress = address - - m.metadataReceived = make(chan *typesCons.StateSyncMetadataResponse, metadataChannelSize) - m.blocksReceived = make(chan *typesCons.GetBlockResponse, blocksChannelSize) - m.initMessagesPool() return m, nil @@ -200,13 +170,15 @@ func (m *consensusModule) Start() error { return err } - go m.metadataSyncLoop() - go m.blockApplicationLoop() + if err := m.stateSync.Start(); err != nil { + return err + } return nil } func (m *consensusModule) Stop() error { + m.logger.Info().Msg("Stopping consensus module") return nil } @@ -287,6 +259,20 @@ func (m *consensusModule) CurrentStep() uint64 { return uint64(m.step) } +func (m *consensusModule) GetNodeAddress() string { + return m.nodeAddress +} + +func (m *consensusModule) updateNodeId() error { + validators, err := m.getValidatorsAtHeight(m.CurrentHeight()) + if err != nil { + return err + } + valAddrToIdMap := typesCons.NewActorMapper(validators).GetValAddrToIdMap() + m.nodeId = valAddrToIdMap[m.nodeAddress] + return nil +} + // TODO: Populate the entire state from the persistence module: validator set, quorum cert, last block hash, etc... func (m *consensusModule) loadPersistedState() error { readCtx, err := m.GetBus().GetPersistenceModule().NewReadContext(-1) // Unknown height @@ -296,7 +282,7 @@ func (m *consensusModule) loadPersistedState() error { defer readCtx.Release() latestHeight, err := readCtx.GetMaximumBlockHeight() - if err != nil || latestHeight == 0 { + if err != nil { // TODO: Proper state sync not implemented yet return nil } diff --git a/consensus/module_consensus_debugging.go b/consensus/module_consensus_debugging.go index 137643df6..0af34ad83 100644 --- a/consensus/module_consensus_debugging.go +++ b/consensus/module_consensus_debugging.go @@ -1,8 +1,10 @@ package consensus +// All the code below is used for debugging & testing purposes only and should not be used in prod. +// TECHDEBT: Add debug/test tags to avoid accidental production usage. + import ( typesCons "github.com/pokt-network/pocket/consensus/types" - coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/messaging" "github.com/pokt-network/pocket/shared/modules" ) @@ -47,10 +49,6 @@ func (m *consensusModule) SetStep(step uint8) { m.step = typesCons.HotstuffStep(step) } -func (m *consensusModule) SetBlock(block *coreTypes.Block) { - m.block = block -} - func (m *consensusModule) SetUtilityUnitOfWork(utilityUnitOfWork modules.UtilityUnitOfWork) { m.utilityUnitOfWork = utilityUnitOfWork } @@ -67,8 +65,3 @@ func (m *consensusModule) GetLeaderForView(height, round uint64, step uint8) uin } return uint64(leaderId) } - -// TODO(#609): Refactor to use the test-only package and remove reflection -func (m *consensusModule) PushStateSyncMetadataResponse(metadataRes *typesCons.StateSyncMetadataResponse) { - m.metadataReceived <- metadataRes -} diff --git a/consensus/module_consensus_pacemaker.go b/consensus/module_consensus_pacemaker.go index 6ac886b6d..e856f9ec7 100644 --- a/consensus/module_consensus_pacemaker.go +++ b/consensus/module_consensus_pacemaker.go @@ -54,7 +54,8 @@ func (m *consensusModule) BroadcastMessageToValidators(msg *anypb.Any) error { } func (m *consensusModule) IsLeader() bool { - return m.leaderId != nil && *m.leaderId == m.nodeId + valMod, err := m.GetBus().GetUtilityModule().GetValidatorModule() + return err == nil && valMod != nil && m.leaderId != nil && *m.leaderId == m.nodeId } func (m *consensusModule) IsLeaderSet() bool { diff --git a/consensus/module_consensus_state_sync.go b/consensus/module_consensus_state_sync.go deleted file mode 100644 index 3763c08a0..000000000 --- a/consensus/module_consensus_state_sync.go +++ /dev/null @@ -1,41 +0,0 @@ -package consensus - -import ( - typesCons "github.com/pokt-network/pocket/consensus/types" - "github.com/pokt-network/pocket/shared/modules" -) - -var _ modules.ConsensusStateSync = &consensusModule{} - -func (m *consensusModule) GetNodeIdFromNodeAddress(peerId string) (uint64, error) { - validators, err := m.getValidatorsAtHeight(m.CurrentHeight()) - if err != nil { - // REFACTOR(#434): As per issue #434, once the new id is sorted out, this return statement must be changed - return 0, err - } - - valAddrToIdMap := typesCons.NewActorMapper(validators).GetValAddrToIdMap() - return uint64(valAddrToIdMap[peerId]), nil -} - -func (m *consensusModule) GetNodeAddress() string { - return m.nodeAddress -} - -// TODO(#352): Implement this function, currently a placeholder. -// blockApplicationLoop commits the blocks received from the blocksReceived channel -// it is intended to be run as a background process -func (m *consensusModule) blockApplicationLoop() { - // runs as a background process in consensus module - // listens on the blocksReceived channel - // commits the received block -} - -// TODO(#352): Implement this function, currently a placeholder. -// metadataSyncLoop periodically sends metadata requests to its peers -// it is intended to be run as a background process -func (m *consensusModule) metadataSyncLoop() { - // runs as a background process in consensus module - // requests metadata from peers - // sends received metadata to the metadataReceived channel -} diff --git a/consensus/pacemaker/module.go b/consensus/pacemaker/module.go index 3a9a16db1..28da8678d 100644 --- a/consensus/pacemaker/module.go +++ b/consensus/pacemaker/module.go @@ -60,37 +60,36 @@ type pacemaker struct { logPrefix string } -func CreatePacemaker(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { +func Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { return new(pacemaker).Create(bus, options...) } func (*pacemaker) Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { + runtimeMgr := bus.GetRuntimeMgr() + cfg := runtimeMgr.GetConfig() + pacemakerCfg := cfg.Consensus.PacemakerConfig + m := &pacemaker{ logPrefix: defaultLogPrefix, + debug: pacemakerDebug{ + manualMode: pacemakerCfg.GetManual(), + debugTimeBetweenStepsMsec: pacemakerCfg.GetDebugTimeBetweenStepsMsec(), + quorumCertificate: nil, + }, + pacemakerCfg: pacemakerCfg, } + m.roundTimeout = m.getRoundTimeout() + m.logger = logger.Global.CreateLoggerForModule(m.GetModuleName()) for _, option := range options { option(m) } - bus.RegisterModule(m) - runtimeMgr := bus.GetRuntimeMgr() - cfg := runtimeMgr.GetConfig() - - m.pacemakerCfg = cfg.Consensus.PacemakerConfig - m.roundTimeout = m.getRoundTimeout() - m.debug = pacemakerDebug{ - manualMode: m.pacemakerCfg.GetManual(), - debugTimeBetweenStepsMsec: m.pacemakerCfg.GetDebugTimeBetweenStepsMsec(), - quorumCertificate: nil, - } - return m, nil } func (m *pacemaker) Start() error { - m.logger = logger.Global.CreateLoggerForModule(m.GetModuleName()) m.RestartTimer() return nil } @@ -101,7 +100,6 @@ func (*pacemaker) GetModuleName() string { func (m *pacemaker) ShouldHandleMessage(msg *typesCons.HotstuffMessage) (bool, error) { consensusMod := m.GetBus().GetConsensusModule() - currentHeight := consensusMod.CurrentHeight() currentRound := consensusMod.CurrentRound() currentStep := typesCons.HotstuffStep(consensusMod.CurrentStep()) @@ -113,11 +111,11 @@ func (m *pacemaker) ShouldHandleMessage(msg *typesCons.HotstuffMessage) (bool, e } // If this case happens, there are two possibilities: - // 1. The node is behind and needs to catch up, node must start syncing, - // 2. The leader is sending a malicious proposal. + // 1. The node is behind and needs to catch up, node must start syncing, + // 2. The leader is sending a malicious proposal. // There, for both cases, node rejects the proposal, because: - // 1. If node is out of sync, node can't verify the block proposal, so rejects it. But node will eventually sync with the rest of the network and add the block. - // 2. If node is synced, node must reject the proposal because proposal is not valid. + // 1. If node is out of sync, node can't verify the block proposal, so rejects it. But node will eventually sync with the rest of the network and add the block. + // 2. If node is synced, node must reject the proposal because proposal is not valid. if msg.Height > currentHeight { m.logger.Info().Msgf("⚠️ [WARN] ⚠️ Node at height %d < message height %d", currentHeight, msg.Height) err := m.GetBus().GetStateMachineModule().SendEvent(coreTypes.StateMachineEvent_Consensus_IsUnsynced) @@ -228,8 +226,10 @@ func (m *pacemaker) NewHeight() { consensusMod := m.GetBus().GetConsensusModule() consensusMod.ResetRound(true) + newHeight := consensusMod.CurrentHeight() + 1 consensusMod.SetHeight(newHeight) + m.logger.Info().Uint64("height", newHeight).Msg("🏁 Starting 1st round at new height 🏁") // CONSIDERATION: We are omitting CommitQC and TimeoutQC here for simplicity, but should we add them? diff --git a/consensus/state_sync/helpers.go b/consensus/state_sync/helpers.go index 3a7bb1afc..6d2ba6078 100644 --- a/consensus/state_sync/helpers.go +++ b/consensus/state_sync/helpers.go @@ -8,23 +8,35 @@ import ( // SendStateSyncMessage sends a state sync message after converting to any proto, to the given peer func (m *stateSync) sendStateSyncMessage(msg *typesCons.StateSyncMessage, dst cryptoPocket.Address) error { - anyMsg, err := anypb.New(msg) - if err != nil { + if anyMsg, err := anypb.New(msg); err != nil { return err - } - if err := m.GetBus().GetP2PModule().Send(dst, anyMsg); err != nil { + } else if err := m.GetBus().GetP2PModule().Send(dst, anyMsg); err != nil { m.logger.Error().Err(err).Msg(typesCons.ErrSendMessage.Error()) return err } return nil } -func (m *stateSync) stateSyncLogHelper(receiverPeerAddress string) map[string]any { - consensusMod := m.GetBus().GetConsensusModule() +// TECHDEBT(#686): This should be an ongoing background passive state sync process. +// For now, aggregating the messages when requests is good enough. +func (m *stateSync) getAggregatedStateSyncMetadata() (minHeight, maxHeight uint64) { + chanLen := len(m.metadataReceived) + m.logger.Info(). + Int16("num_state_sync_metadata_messages", int16(chanLen)). + Msgf("About to loop overstate sync metadata responses") - return map[string]any{ - "height": consensusMod.CurrentHeight(), - "senderPeerAddress": consensusMod.GetNodeAddress(), - "receiverPeerAddress": receiverPeerAddress, + for i := 0; i < chanLen; i++ { + metadata := <-m.metadataReceived + if metadata.MaxHeight > maxHeight { + maxHeight = metadata.MaxHeight + } + if metadata.MinHeight < minHeight { + minHeight = metadata.MinHeight + } } + m.logger.Info().Fields(map[string]any{ + "min_height": minHeight, + "max_height": maxHeight, + }).Msg("Finished aggregating state sync metadata") + return } diff --git a/consensus/state_sync/interfaces.go b/consensus/state_sync/interfaces.go deleted file mode 100644 index f6772ac45..000000000 --- a/consensus/state_sync/interfaces.go +++ /dev/null @@ -1,104 +0,0 @@ -package state_sync - -import ( - coreTypes "github.com/pokt-network/pocket/shared/core/types" - cryptoPocket "github.com/pokt-network/pocket/shared/crypto" -) - -// REFACTOR: Remove interface definitions from this file to their respective source code files, -// keep interface definitions in the same file with the implementation as in server.go - -type SyncState interface { - // latest local height - LatestHeight() int64 - // latest network height (from the aggregation of Peer Sync Meta) - LatestNetworkHeight() int64 - // retrieve peer meta (actively updated through churn management) - GetPeers() []PeerSyncMeta - // returns ordered array of missing block heights - GetMissingBlockHeights() []int64 -} - -type BlockRequestMessage interface { - // the height the peer wants from the block store - GetHeight() int64 -} - -type BlockResponseMessage interface { - // the bytes of the requested block from the block store - GetBlockBytes() []byte -} - -// TODO: needs to be shared between P2P as the Churn Management Process updates this information -type PeerSyncMeta interface { - // the unique identifier associated with the peer - GetPeerID() string - // the maximum height the peer has in the block store - GetMaxHeight() int64 - // the minimum height the peer has in the block store - GetMinHeight() int64 -} - -// LEGACY interface definition -// TODO(#352): delete this once state sync module is ready. -type StateSyncModuleLEGACY interface { - // -- Constructor Setter Functions -- - - // `HandleStateSync` function: - // - Create a Utility Unit Of Work - // - Block.ValidateBasic() - // - Consensus Module Replica Path - // - Prepare Block: utilityUnitOfWork.SetProposalBlock(block) - // - Apply Block: utilityUnitOfWork.ApplyBlock(block) - // - Validate Block: utilityUnitOfWork.StateHash == Block.StateHash - // - Store Block: consensusModule.CommitBlock() - HandleStateSyncMessage(msg BlockResponseMessage) - - // `GetPeerSyncMeta` function: - // - Retrieve a list of active peers with their metadata (identified and retrieved through P2P's `Churn Management`) - GetPeerMetadata(GetPeerSyncMeta func() (peers []PeerSyncMeta, err error)) - - // `typesP2P.Router#Send()` function contract: - // - sends data to an address via P2P network - NetworkSend(NetworkSend func(data []byte, address cryptoPocket.Address) error) - - // -- Sync modes -- - - // In the StateSync protocol, the Node fields valid BlockRequests from its peers to help them CatchUp to be Synced. - // This sub-protocol is continuous throughout the lifecycle of StateSync. - RunServerMode() - - // In SyncedMode, the Node is caught up to the latest block and is listening & waiting for the latest block to be passed - // to maintain a synchronous state with the global SyncState. - // - UpdatePeerMetadata from P2P module - // - UpdateSyncState - // - Rely on new blocks to be propagated via the P2P network after Validators reach consensus - // - If `localSyncState.Height < globalNetworkSyncState.Height` -> RunSyncMode() // careful about race-conditions - RunSyncedMode() - - // Runs sync mode 'service' that continuously runs while `localSyncState.Height < globalNetworkSyncState.Height` - // - UpdatePeerMetadata from P2P module - // - Retrieve missing blocks from peers - // - Process retrieved blocks - // - UpdateSyncState - // - If `localSyncState.Height == globalNetworkSyncState.Height` -> RunSyncedMode() - RunSyncMode() - - // Returns the `highest priority aka lowest height` missing block heights up to `max` heights - GetMissingBlockHeights(state SyncState, max int) (blockHeights []int64, err error) - - // Random selection of eligilbe peers enables a fair distribution of blockRequests over time via law of large numbers - // An eligible peer is when `PeerMeta.MinHeight <= blockHeight <= PeerMeta.MaxHeight` - GetRandomEligiblePeersForHeight(blockHeight int64) (eligiblePeer PeerSyncMeta, err error) - - // Uses `typesP2P.Router#Send()` to send a `BlockRequestMessage` to a specific peer - SendBlockRequest(peerId string) error - - // Uses 'typesP2P.Router#Send()' to send a `BlockResponseMessage` to a specific peer - // This function is used in 'ServerMode()' - HandleBlockRequest(message BlockRequestMessage) error - - // Uses `HandleBlock` to process retrieved blocks from peers - // Must update sync state using `SetMissingBlockHeight` - ProcessBlock(block *coreTypes.Block) error -} diff --git a/consensus/state_sync/module.go b/consensus/state_sync/module.go index a35a24318..968cf5af3 100644 --- a/consensus/state_sync/module.go +++ b/consensus/state_sync/module.go @@ -1,18 +1,37 @@ package state_sync import ( + "context" + "encoding/hex" + "time" + + typesCons "github.com/pokt-network/pocket/consensus/types" "github.com/pokt-network/pocket/logger" coreTypes "github.com/pokt-network/pocket/shared/core/types" + "github.com/pokt-network/pocket/shared/messaging" "github.com/pokt-network/pocket/shared/modules" + "google.golang.org/protobuf/types/known/anypb" ) const ( stateSyncModuleName = "stateSyncModule" + // TODO: Make these configurable + blockWaitingPeriod = 30 * time.Second + committedBlocsChannelSize = 100 + metadataChannelSize = 1000 + blocksChannelSize = 1000 + metadataSyncPeriod = 10 * time.Second ) type StateSyncModule interface { modules.Module StateSyncServerModule + + HandleBlockCommittedEvent(*messaging.ConsensusNewHeightEvent) + HandleStateSyncMetadataResponse(*typesCons.StateSyncMetadataResponse) + + // TECHDEBT: This function can be removed once the dependency of state sync on the FSM module is removed. + StartSynchronousStateSync() } var ( @@ -24,41 +43,168 @@ var ( type stateSync struct { bus modules.Bus logger *modules.Logger + + // metadata responses received from peers are collected in this channel + metadataReceived chan *typesCons.StateSyncMetadataResponse + + committedBlocksChannel chan uint64 } -func CreateStateSync(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { +func Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { return new(stateSync).Create(bus, options...) } func (*stateSync) Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { - m := &stateSync{} + m := &stateSync{ + metadataReceived: make(chan *typesCons.StateSyncMetadataResponse, metadataChannelSize), + committedBlocksChannel: make(chan uint64, committedBlocsChannelSize), + } + m.logger = logger.Global.CreateLoggerForModule(m.GetModuleName()) for _, option := range options { option(m) } - bus.RegisterModule(m) - m.logger = logger.Global.CreateLoggerForModule(m.GetModuleName()) - return m, nil } -// TODO(#352): implement this function -// Start performs state sync func (m *stateSync) Start() error { - // processes and aggregates all metadata collected in metadataReceived channel, - // requests missing blocks starting from its current height to the aggregated metadata's maxHeight, - // once the requested block is received and committed by consensus module, sends the next request for the next block, - // when all blocks are received and committed, stops the state sync process by calling its `Stop()` function. + go m.metadataSyncLoop() return nil } -// TODO(#352): check if node is a valdiator, if not send Consensus_IsSyncedNonValidator event -// Stop stops the state sync process, and sends `Consensus_IsSyncedValidator` FSM event -func (m *stateSync) Stop() error { +// Start a synchronous state sync process to catch up to the network +// 1. Processes and aggregates all metadata collected in metadataReceived channel +// 2. Requests missing blocks until the maximum seen block is retrieved +// 3. Perform (2) one-by-one, applying and validating each block while doing so +// 4. Once all blocks are received and committed, stop the synchronous state sync process +func (m *stateSync) StartSynchronousStateSync() { + consensusMod := m.bus.GetConsensusModule() + currentHeight := consensusMod.CurrentHeight() + nodeAddress := consensusMod.GetNodeAddress() + nodeAddressBz, err := hex.DecodeString(nodeAddress) + if err != nil { + m.logger.Error().Err(err).Msg("Failed to decode node address") + return + } + + readCtx, err := m.GetBus().GetPersistenceModule().NewReadContext(int64(currentHeight)) + if err != nil { + m.logger.Error().Err(err).Msg("Failed to create read context") + return + } + defer readCtx.Release() + + // Get a view into the state of the network + _, maxHeight := m.getAggregatedStateSyncMetadata() + + m.logger.Info(). + Uint64("current_height", currentHeight). + Uint64("max_height", maxHeight). + Msg("Synchronous state sync is requesting blocks...") + + // Synchronously request block requests from the current height to the aggregated metadata height + // Note that we are using `<=` because: + // - maxHeight is the max * committed * height of the network + // - currentHeight is the latest * committing * height of the node + + // We do not need to request the genesis block from anyone + if currentHeight == 0 { + currentHeight += 1 + } + + for currentHeight <= maxHeight { + m.logger.Info().Msgf("Synchronous state sync is requesting block: %d, ending height: %d", currentHeight, maxHeight) + + // form the get block request message + stateSyncGetBlockMsg := &typesCons.StateSyncMessage{ + Message: &typesCons.StateSyncMessage_GetBlockReq{ + GetBlockReq: &typesCons.GetBlockRequest{ + PeerAddress: nodeAddress, + Height: currentHeight, + }, + }, + } + anyProtoStateSyncMsg, err := anypb.New(stateSyncGetBlockMsg) + if err != nil { + m.logger.Error().Err(err).Msg("Failed to create Any proto") + return + } + + // Broadcast the block request + if err := m.GetBus().GetP2PModule().Broadcast(anyProtoStateSyncMsg); err != nil { + m.logger.Error().Err(err).Msg("Failed to broadcast state sync message") + return + } + + // Wait for the consensus module to commit the requested block and re-try on timeout + select { + case blockHeight := <-m.committedBlocksChannel: + m.logger.Info().Msgf("State sync received event that block %d is committed!", blockHeight) + case <-time.After(blockWaitingPeriod): + m.logger.Error().Msgf("Timed out waiting for block %d to be committed...", currentHeight) + } + + // Update the height and continue catching up to the latest known state + currentHeight = consensusMod.CurrentHeight() + } - return m.GetBus().GetStateMachineModule().SendEvent(coreTypes.StateMachineEvent_Consensus_IsSyncedValidator) + // Checked if the synched node is a validator or not + isValidator, err := readCtx.GetValidatorExists(nodeAddressBz, int64(currentHeight)) + if err != nil { + m.logger.Error().Err(err).Msg("Failed to check if validator exists") + return + } + + // Send out the appropriate FSM event now that the node is caught up + if isValidator { + err = m.GetBus().GetStateMachineModule().SendEvent(coreTypes.StateMachineEvent_Consensus_IsSyncedValidator) + } else { + err = m.GetBus().GetStateMachineModule().SendEvent(coreTypes.StateMachineEvent_Consensus_IsSyncedNonValidator) + } + if err != nil { + m.logger.Error().Err(err).Msg("Failed to send state machine event") + return + } +} + +func (m *stateSync) HandleStateSyncMetadataResponse(res *typesCons.StateSyncMetadataResponse) { + m.logger.Info().Fields(map[string]any{ + "peer_address": res.PeerAddress, + "min_height": res.MinHeight, + "max_height": res.MaxHeight, + }).Msg("Handling state sync metadata response") + m.metadataReceived <- res + + if res.MaxHeight > 0 && m.GetBus().GetConsensusModule().CurrentHeight() <= res.MaxHeight { + if err := m.GetBus().GetStateMachineModule().SendEvent(coreTypes.StateMachineEvent_Consensus_IsUnsynced); err != nil { + m.logger.Error().Err(err).Msg("Failed to send state machine event") + } + } +} + +func (m *stateSync) HandleBlockCommittedEvent(msg *messaging.ConsensusNewHeightEvent) { + m.logger.Info().Msg("Handling state sync block committed event") + m.committedBlocksChannel <- msg.Height +} + +func (m *stateSync) Stop() error { + m.logger.Log().Msg("Draining and closing metadataReceived and blockResponse channels") + for { + select { + case metaData, ok := <-m.metadataReceived: + if ok { + m.logger.Info().Msgf("Drained metadata message: %s", metaData) + } else { + close(m.metadataReceived) + return nil + } + default: + close(m.metadataReceived) + return nil + } + } } func (m *stateSync) SetBus(pocketBus modules.Bus) { @@ -75,3 +221,45 @@ func (m *stateSync) GetBus() modules.Bus { func (m *stateSync) GetModuleName() string { return stateSyncModuleName } + +// metadataSyncLoop periodically sends metadata requests to its peers to collect & +// aggregate metadata related to synching the state. +// It is intended to be run as a background process via a goroutine. +func (m *stateSync) metadataSyncLoop() { + metaSyncLoopLogger := m.logger.With().Str("source", "metadataSyncLoop").Logger() + ctx := context.TODO() + + ticker := time.NewTicker(metadataSyncPeriod) + for { + select { + case <-ticker.C: + metaSyncLoopLogger.Info().Msg("Background metadata sync check triggered") + if err := m.broadcastMetadataRequests(); err != nil { + metaSyncLoopLogger.Error().Err(err).Msg("Failed to send metadata requests") + } + + case <-ctx.Done(): + ticker.Stop() + } + } +} + +// broadcastMetadataRequests sends a metadata request to all peers in the network to understand +// the state of the network and determine if the node is behind. +func (m *stateSync) broadcastMetadataRequests() error { + stateSyncMetadataReqMessage := &typesCons.StateSyncMessage{ + Message: &typesCons.StateSyncMessage_MetadataReq{ + MetadataReq: &typesCons.StateSyncMetadataRequest{ + PeerAddress: m.GetBus().GetConsensusModule().GetNodeAddress(), + }, + }, + } + anyMsg, err := anypb.New(stateSyncMetadataReqMessage) + if err != nil { + return err + } + if err := m.GetBus().GetP2PModule().Broadcast(anyMsg); err != nil { + return err + } + return nil +} diff --git a/consensus/state_sync/server.go b/consensus/state_sync/server.go index 062792f56..7b2e6e4e3 100644 --- a/consensus/state_sync/server.go +++ b/consensus/state_sync/server.go @@ -1,81 +1,112 @@ package state_sync import ( - "fmt" - typesCons "github.com/pokt-network/pocket/consensus/types" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" ) -// This module is responsible for handling requests and business logic that advertises and shares -// local state metadata with other peers syncing to the latest block. +// StateSyncServerModule is responsible for handling requests and business logic that +// advertise and share local state metadata with other peers syncing to the latest block. type StateSyncServerModule interface { // Advertise (send) the local state sync metadata to the requesting peer - HandleStateSyncMetadataRequest(*typesCons.StateSyncMetadataRequest) error + HandleStateSyncMetadataRequest(*typesCons.StateSyncMetadataRequest) // Advertise (send) the block being requested by the peer - HandleGetBlockRequest(*typesCons.GetBlockRequest) error + HandleGetBlockRequest(*typesCons.GetBlockRequest) } -func (m *stateSync) HandleStateSyncMetadataRequest(metadataReq *typesCons.StateSyncMetadataRequest) error { +// HandleStateSyncMetadataRequest processes a request from another peer to get a view into the +// state stored in this node +func (m *stateSync) HandleStateSyncMetadataRequest(metadataReq *typesCons.StateSyncMetadataRequest) { + logger := m.logger.With().Str("source", "HandleStateSyncMetadataRequest").Logger() + consensusMod := m.GetBus().GetConsensusModule() serverNodePeerAddress := consensusMod.GetNodeAddress() clientPeerAddress := metadataReq.PeerAddress - m.logger.Info().Fields(m.stateSyncLogHelper(clientPeerAddress)).Msgf("Received StateSyncMetadataRequest %s", metadataReq) + // No blocks or metadata to share at genesis + if consensusMod.CurrentHeight() == 0 { + return + } // current height is the height of the block that is being processed, so we need to subtract 1 for the last finalized block prevPersistedBlockHeight := consensusMod.CurrentHeight() - 1 readCtx, err := m.GetBus().GetPersistenceModule().NewReadContext(int64(prevPersistedBlockHeight)) if err != nil { - return nil + logger.Err(err).Msg("Error creating read context") + return } defer readCtx.Release() + // What is the maximum block height this node can share with others? maxHeight, err := readCtx.GetMaximumBlockHeight() if err != nil { - return err + logger.Err(err).Msg("Error getting max height") + return } + // What is the minimum block height this node can share with others? minHeight, err := readCtx.GetMinimumBlockHeight() if err != nil { - return err + logger.Err(err).Msg("Error getting min height") + return } + // Prepare state sync message to send to peer stateSyncMessage := typesCons.StateSyncMessage{ Message: &typesCons.StateSyncMessage_MetadataRes{ MetadataRes: &typesCons.StateSyncMetadataResponse{ PeerAddress: serverNodePeerAddress, MinHeight: minHeight, - MaxHeight: uint64(maxHeight), + MaxHeight: maxHeight, }, }, } - return m.sendStateSyncMessage(&stateSyncMessage, cryptoPocket.AddressFromString(clientPeerAddress)) + fields := map[string]interface{}{ + "max_height": maxHeight, + "min_height": minHeight, + "self": serverNodePeerAddress, + "peer": clientPeerAddress, + } + + if err = m.sendStateSyncMessage(&stateSyncMessage, cryptoPocket.AddressFromString(clientPeerAddress)); err != nil { + logger.Err(err).Fields(fields).Msg("Error responding to state sync metadata request") + } + logger.Debug().Fields(fields).Msg("Successfully responded to state sync metadata request") } -func (m *stateSync) HandleGetBlockRequest(blockReq *typesCons.GetBlockRequest) error { +// HandleGetBlockRequest processes a request from another to share a specific block at a specific node +// that this node likely has available. +func (m *stateSync) HandleGetBlockRequest(blockReq *typesCons.GetBlockRequest) { + logger := m.logger.With().Str("source", "HandleGetBlockRequest").Logger() + consensusMod := m.GetBus().GetConsensusModule() serverNodePeerAddress := consensusMod.GetNodeAddress() clientPeerAddress := blockReq.PeerAddress - m.logger.Info().Fields(m.stateSyncLogHelper(clientPeerAddress)).Msgf("Received StateSync GetBlockRequest") - prevPersistedBlockHeight := consensusMod.CurrentHeight() - 1 + // No blocks or metadata to share at genesis + if consensusMod.CurrentHeight() == 0 { + return + } + // Check if the block should be retrievable based on the node's consensus height + prevPersistedBlockHeight := consensusMod.CurrentHeight() - 1 if prevPersistedBlockHeight < blockReq.Height { - return fmt.Errorf("requested block height: %d is higher than current persisted block height: %d", blockReq.Height, prevPersistedBlockHeight) + logger.Error().Msgf("The requested block height (%d) is higher than current persisted block height (%d)", blockReq.Height, prevPersistedBlockHeight) + return } - // get block from the persistence module + // Try to get block from the block store blockStore := m.GetBus().GetPersistenceModule().GetBlockStore() block, err := blockStore.GetBlock(blockReq.Height) if err != nil { - m.logger.Error().Err(err).Msgf("failed to get block at height %d", blockReq.Height) - return err + logger.Error().Err(err).Msgf("failed to get block at height %d", blockReq.Height) + return } + // Prepare state sync message to send to peer stateSyncMessage := typesCons.StateSyncMessage{ Message: &typesCons.StateSyncMessage_GetBlockRes{ GetBlockRes: &typesCons.GetBlockResponse{ @@ -85,5 +116,14 @@ func (m *stateSync) HandleGetBlockRequest(blockReq *typesCons.GetBlockRequest) e }, } - return m.sendStateSyncMessage(&stateSyncMessage, cryptoPocket.AddressFromString(clientPeerAddress)) + fields := map[string]interface{}{ + "height": blockReq.Height, + "self": serverNodePeerAddress, + "peer": clientPeerAddress, + } + + if err = m.sendStateSyncMessage(&stateSyncMessage, cryptoPocket.AddressFromString(clientPeerAddress)); err != nil { + logger.Err(err).Fields(fields).Msg("Error responding to state sync block request") + } + logger.Debug().Fields(fields).Msg("Successfully responded to state sync block request") } diff --git a/consensus/state_sync_handler.go b/consensus/state_sync_handler.go index d67e2832a..f33d809db 100644 --- a/consensus/state_sync_handler.go +++ b/consensus/state_sync_handler.go @@ -5,29 +5,25 @@ import ( typesCons "github.com/pokt-network/pocket/consensus/types" "github.com/pokt-network/pocket/shared/codec" + coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/messaging" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" ) func (m *consensusModule) HandleStateSyncMessage(stateSyncMessageAny *anypb.Any) error { - m.m.Lock() - defer m.m.Unlock() - - m.logger.Info().Msg("Handling StateSyncMessage") - switch stateSyncMessageAny.MessageName() { case messaging.StateSyncMessageContentType: msg, err := codec.GetCodec().FromAny(stateSyncMessageAny) if err != nil { return err } - stateSyncMessage, ok := msg.(*typesCons.StateSyncMessage) if !ok { return fmt.Errorf("failed to cast message to StateSyncMessage") } - return m.handleStateSyncMessage(stateSyncMessage) + default: return typesCons.ErrUnknownStateSyncMessageType(stateSyncMessageAny.MessageName()) } @@ -35,29 +31,146 @@ func (m *consensusModule) HandleStateSyncMessage(stateSyncMessageAny *anypb.Any) func (m *consensusModule) handleStateSyncMessage(stateSyncMessage *typesCons.StateSyncMessage) error { switch stateSyncMessage.Message.(type) { + case *typesCons.StateSyncMessage_MetadataReq: - m.logger.Info().Str("proto_type", "MetadataRequest").Msg("Handling StateSyncMessage MetadataReq") - if !m.serverModeEnabled { + // m.logger.Info().Str("proto_type", "MetadataRequest").Msg("Handling StateSyncMessage MetadataReq") + if !m.consCfg.ServerModeEnabled { m.logger.Warn().Msg("Node's server module is not enabled") return nil } - return m.stateSync.HandleStateSyncMetadataRequest(stateSyncMessage.GetMetadataReq()) - case *typesCons.StateSyncMessage_MetadataRes: - m.logger.Info().Str("proto_type", "MetadataResponse").Msg("Handling StateSyncMessage MetadataRes") - m.metadataReceived <- stateSyncMessage.GetMetadataRes() + go m.stateSync.HandleStateSyncMetadataRequest(stateSyncMessage.GetMetadataReq()) return nil + case *typesCons.StateSyncMessage_GetBlockReq: - m.logger.Info().Str("proto_type", "GetBlockRequest").Msg("Handling StateSyncMessage GetBlockRequest") - if !m.serverModeEnabled { + // m.logger.Info().Str("proto_type", "GetBlockRequest").Msg("Handling StateSyncMessage GetBlockRequest") + if !m.consCfg.ServerModeEnabled { m.logger.Warn().Msg("Node's server module is not enabled") return nil } - return m.stateSync.HandleGetBlockRequest(stateSyncMessage.GetGetBlockReq()) + go m.stateSync.HandleGetBlockRequest(stateSyncMessage.GetGetBlockReq()) + return nil + + case *typesCons.StateSyncMessage_MetadataRes: + m.logger.Info().Str("proto_type", "MetadataResponse").Msg("Handling StateSyncMessage MetadataRes") + go m.stateSync.HandleStateSyncMetadataResponse(stateSyncMessage.GetMetadataRes()) + return nil + + // NB: Note that this is the only case that calls a function in the consensus module (not the state sync submodule) since + // consensus is the one responsible for calling business logic to apply and commit the blocks. State sync listens for events + // that are a result of it. case *typesCons.StateSyncMessage_GetBlockRes: m.logger.Info().Str("proto_type", "GetBlockResponse").Msg("Handling StateSyncMessage GetBlockResponse") - m.blocksReceived <- stateSyncMessage.GetGetBlockRes() + go m.tryToApplyRequestedBlock(stateSyncMessage.GetGetBlockRes()) return nil + default: return fmt.Errorf("unspecified state sync message type") } } + +// tryToApplyRequestedBlock tries to commit the requested Block received from a peer. +// Intended to be called via a background goroutine. +// CLEANUP: Investigate whether this should be part of `Consensus` or part of `StateSync` +func (m *consensusModule) tryToApplyRequestedBlock(blockResponse *typesCons.GetBlockResponse) { + logger := m.logger.With().Str("source", "tryToApplyRequestedBlock").Logger() + + // Retrieve the block we're about to try and apply + block := blockResponse.Block + if block == nil { + logger.Error().Msg("Received nil block in GetBlockResponse") + return + } + logger.Info().Msgf("Received new block at height %d.", block.BlockHeader.Height) + + // Check what the current latest committed block height is + maxPersistedHeight, err := m.maxPersistedBlockHeight() + if err != nil { + logger.Err(err).Msg("couldn't query max persisted height") + return + } + + // Check if the block being synched is behind the current height + if block.BlockHeader.Height <= maxPersistedHeight { + logger.Debug().Msgf("Discarding block height %d, since node is ahead at height %d", block.BlockHeader.Height, maxPersistedHeight) + return + } + + // Check if the block being synched is ahead of the current height + if block.BlockHeader.Height > m.CurrentHeight() { + // IMPROVE: we need to store block responses that we are not yet ready to validate so we can validate them on a subsequent iteration of this loop + logger.Info().Bool("TODO", true).Msgf("Received block at height %d, discarding as it is higher than the current height", block.BlockHeader.Height) + return + } + + // Perform basic validation on the block + if err = m.basicValidateBlock(block); err != nil { + logger.Err(err).Msg("failed to validate block") + return + } + + // Update the leader proposing the block + // TECHDEBT: This ID logic could potentially be simplified in the future but needs a SPIKE + leaderIdInt, err := m.getNodeIdFromNodeAddress(string(block.BlockHeader.ProposerAddress)) + if err != nil { + m.logger.Error().Err(err).Msg("Could not get leader id from leader address") + return + } + m.leaderId = typesCons.NewNodeId(leaderIdInt) + + // Prepare the utility UOW of work to apply a new block + if err := m.refreshUtilityUnitOfWork(); err != nil { + m.logger.Error().Err(err).Msg("Could not refresh utility context") + return + } + + // Try to apply the block by validating the transactions in the block + if err := m.applyBlock(block); err != nil { + m.logger.Error().Err(err).Msg("Could not apply block") + return + } + + // Try to commit the block to persistence + if err := m.commitBlock(block); err != nil { + m.logger.Error().Err(err).Msg("Could not commit block") + return + } + logger.Info().Int64("height", int64(block.BlockHeader.Height)).Msgf("State sync committed block at height %d!", block.BlockHeader.Height) + + m.paceMaker.NewHeight() +} + +// REFACTOR(#434): Once we consolidated NodeIds/PeerIds, this could potentially be removed +func (m *consensusModule) getNodeIdFromNodeAddress(peerId string) (uint64, error) { + validators, err := m.getValidatorsAtHeight(m.CurrentHeight()) + if err != nil { + m.logger.Warn().Err(err).Msgf("Could not get validators at height %d when checking if peer %s is a validator", m.CurrentHeight(), peerId) + return 0, fmt.Errorf("Could determine if peer %s is a validator or not: %w", peerId, err) + } + + valAddrToIdMap := typesCons.NewActorMapper(validators).GetValAddrToIdMap() + return uint64(valAddrToIdMap[peerId]), nil +} + +// basicValidateBlock performs basic validation of the block, its metadata, signatures, +// but not the transactions themselves +func (m *consensusModule) basicValidateBlock(block *coreTypes.Block) error { + blockHeader := block.BlockHeader + qcBytes := blockHeader.GetQuorumCertificate() + + if qcBytes == nil { + m.logger.Error().Err(typesCons.ErrNoQcInReceivedBlock).Msg(typesCons.DisregardBlock) + return typesCons.ErrNoQcInReceivedBlock + } + + qc := typesCons.QuorumCertificate{} + if err := proto.Unmarshal(qcBytes, &qc); err != nil { + return err + } + + if err := m.validateQuorumCertificate(&qc); err != nil { + m.logger.Error().Err(err).Msg("Couldn't apply block, invalid QC") + return err + } + + return nil +} diff --git a/consensus/types/messages.go b/consensus/types/messages.go index 8faceb2ba..8f30259db 100644 --- a/consensus/types/messages.go +++ b/consensus/types/messages.go @@ -116,7 +116,6 @@ var ( ErrSendMessage = errors.New(sendMessageError) ErrBroadcastMessage = errors.New(broadcastMessageError) ErrCreateConsensusMessage = errors.New(createConsensusMessageError) - ErrCreateStateSyncMessage = errors.New(createStateSyncMessageError) ErrNoQcInReceivedBlock = errors.New(noQcInReceivedBlockError) ErrBlockRetrievalMessage = errors.New(blockRetrievalError) ErrHotstuffValidation = errors.New(anteValidationError) diff --git a/consensus/types/proto/hotstuff.proto b/consensus/types/proto/hotstuff.proto index 8cc9a8334..8bee5d73a 100644 --- a/consensus/types/proto/hotstuff.proto +++ b/consensus/types/proto/hotstuff.proto @@ -40,6 +40,7 @@ message QuorumCertificate { uint64 height = 1; uint64 round = 2; HotstuffStep step = 3; + // TECHDEBT: Note that there is a circular dependency between the `Block` and `QuorumCertificate` types which we have to think about and resolve. core.Block block = 4; ThresholdSignature threshold_signature = 5; } diff --git a/consensus/types/types.go b/consensus/types/types.go index 19517d3cd..cbb9f0e98 100644 --- a/consensus/types/types.go +++ b/consensus/types/types.go @@ -30,3 +30,8 @@ func ActorListToValidatorMap(actors []*coreTypes.Actor) (m ValidatorMap) { } return } + +func NewNodeId(id uint64) *NodeId { + n := NodeId(id) + return &n +} diff --git a/e2e/tests/node.go b/e2e/tests/node.go index 422e6e009..9e570a5d1 100644 --- a/e2e/tests/node.go +++ b/e2e/tests/node.go @@ -21,7 +21,8 @@ var ( ) func init() { - defaultRPCHost := runtime.GetEnv("RPC_HOST", defaults.RandomValidatorEndpointK8SHostname) + // defaultRPCHost := runtime.GetEnv("RPC_HOST", defaults.RandomValidatorEndpointK8SHostname) + defaultRPCHost := runtime.GetEnv("RPC_HOST", defaults.FullNode1EndpointK8SHostname) defaultRPCURL = fmt.Sprintf("http://%s:%s", defaultRPCHost, defaults.DefaultRPCPort) } diff --git a/p2p/event_handler.go b/p2p/event_handler.go index b1184e860..b39ad7323 100644 --- a/p2p/event_handler.go +++ b/p2p/event_handler.go @@ -10,7 +10,6 @@ import ( "github.com/pokt-network/pocket/shared/messaging" ) -// CONSIDERATION(#576): making this part of some new `ConnManager`. func (m *p2pModule) HandleEvent(event *anypb.Any) error { evt, err := codec.GetCodec().FromAny(event) if err != nil { diff --git a/p2p/utils_test.go b/p2p/utils_test.go index bebab237f..897a75437 100644 --- a/p2p/utils_test.go +++ b/p2p/utils_test.go @@ -40,8 +40,8 @@ import ( // ~~~~~~ RainTree Unit Test Configurations ~~~~~~ const ( - serviceURLFormat = "node%d.consensus:42069" - eventsChannelSize = 10000 + // TECHDEBT: Look into ways to remove `serviceURLFormat` from the test suite + serviceURLFormat = "node%d.consensus:42069" // Since we simulate up to a 27 node network, we will pre-generate a n >= 27 number of keys to avoid generation // every time. The genesis config seed start is set for deterministic key generation and 42 was chosen arbitrarily. genesisConfigSeedStart = 42 diff --git a/runtime/bus.go b/runtime/bus.go index 9d7a6e05f..3efaf8bba 100644 --- a/runtime/bus.go +++ b/runtime/bus.go @@ -21,23 +21,32 @@ type bus struct { // Node events channel modules.EventsChannel + // A secondary channel that receives all the same events as the main bus, + // but does not pull events when `GetBusEvent` is called + debugChannel modules.EventsChannel + modulesRegistry modules.ModulesRegistry runtimeMgr modules.RuntimeMgr } -func CreateBus(runtimeMgr modules.RuntimeMgr) (modules.Bus, error) { - return new(bus).Create(runtimeMgr) +func CreateBus(runtimeMgr modules.RuntimeMgr, opts ...modules.BusOption) (modules.Bus, error) { + return new(bus).Create(runtimeMgr, opts...) } -func (b *bus) Create(runtimeMgr modules.RuntimeMgr) (modules.Bus, error) { +func (b *bus) Create(runtimeMgr modules.RuntimeMgr, opts ...modules.BusOption) (modules.Bus, error) { bus := &bus{ - channel: make(modules.EventsChannel, defaults.DefaultBusBufferSize), + channel: make(modules.EventsChannel, defaults.DefaultBusBufferSize), + debugChannel: nil, runtimeMgr: runtimeMgr, modulesRegistry: NewModulesRegistry(), } + for _, o := range opts { + o(bus) + } + return bus, nil } @@ -52,6 +61,9 @@ func (m *bus) RegisterModule(module modules.Submodule) { func (m *bus) PublishEventToBus(e *messaging.PocketEnvelope) { m.channel <- e + if m.debugChannel != nil { + m.debugChannel <- e + } } func (m *bus) GetBusEvent() *messaging.PocketEnvelope { @@ -63,6 +75,11 @@ func (m *bus) GetEventBus() modules.EventsChannel { return m.channel } +// GetDebugEventBus returns the debug event bus +func (m *bus) GetDebugEventBus() modules.EventsChannel { + return m.debugChannel +} + func (m *bus) GetRuntimeMgr() modules.RuntimeMgr { return m.runtimeMgr } diff --git a/runtime/debug_helpers.go b/runtime/debug_helpers.go new file mode 100644 index 000000000..62e354192 --- /dev/null +++ b/runtime/debug_helpers.go @@ -0,0 +1,27 @@ +// +built test debug + +package runtime + +import ( + "github.com/pokt-network/pocket/runtime/defaults" + "github.com/pokt-network/pocket/shared/modules" +) + +// WithDebugEventsChannel is used initialize a secondary (debug) bus that receives all the same events +// as the main bus, but does pull events when `GetBusEvent` is called +func WithDebugEventsChannel(eventsChannel modules.EventsChannel) modules.BusOption { + return func(m modules.Bus) { + if m, ok := m.(*bus); ok { + m.debugChannel = eventsChannel + } + } +} + +// WithNewDebugEventsChannel is used initialize a secondary (debug) bus that receives all the same events +func WithNewDebugEventsChannel() modules.BusOption { + return func(m modules.Bus) { + if m, ok := m.(*bus); ok { + m.debugChannel = make(modules.EventsChannel, defaults.DefaultBusBufferSize) + } + } +} diff --git a/runtime/defaults/defaults.go b/runtime/defaults/defaults.go index 6a66726b3..a01f0ff58 100644 --- a/runtime/defaults/defaults.go +++ b/runtime/defaults/defaults.go @@ -28,6 +28,7 @@ const ( DefaultRPCHost = "localhost" Validator1EndpointDockerComposeHostname = "validator1" Validator1EndpointK8SHostname = "validator-001-pocket" + FullNode1EndpointK8SHostname = "full-node-001-pocket" RandomValidatorEndpointK8SHostname = "pocket-validators" ) diff --git a/shared/core/types/proto/block.proto b/shared/core/types/proto/block.proto index a0e2ed6c3..eef67e961 100644 --- a/shared/core/types/proto/block.proto +++ b/shared/core/types/proto/block.proto @@ -7,12 +7,16 @@ option go_package = "github.com/pokt-network/pocket/shared/core/types"; import "google/protobuf/timestamp.proto"; message BlockHeader { - uint64 height = 1; - string networkId = 2; // used to differentiate what network the chain is on (Tendermint legacy) - string stateHash = 3; // the state committment at this blocks height - string prevStateHash = 4; // the state committment at this block height-1 - bytes proposerAddress = 5; // the address of the proposer of this block; TECHDEBT: Change this to an string + uint64 height = 1; // the block height + // TECHDEBT: This is Tendermint legacy and we have to decide if we want to keep it + string networkId = 2; // used to differentiate what network the chain is on such as MainNet/TestNet + string stateHash = 3; // the state committment (i.e. root hash) at this block height + string prevStateHash = 4; // the state committment (i.e. root hash) for the previous block (height-1) + // TECHDEBT: Change the proposer address to a string + bytes proposerAddress = 5; // the address of the proposer/leader of this block + // CONSIDERATION: Should we use `QuorumCertificate` directly here? bytes quorumCertificate = 6; // the quorum certificate containing signature from 2/3+ validators at this height + // CONSIDERATION: Decide if this is needed google.protobuf.Timestamp timestamp = 7; // unixnano timestamp of when the block was created map state_tree_hashes = 8; // map[TreeName]hex(TreeRootHash) string val_set_hash = 9; // the hash of the current validator set who were able to sign the current block diff --git a/shared/messaging/events.go b/shared/messaging/events.go index a726a55a0..6b19785be 100644 --- a/shared/messaging/events.go +++ b/shared/messaging/events.go @@ -6,8 +6,10 @@ const ( ConsensusNewHeightEventType = "pocket.ConsensusNewHeightEvent" StateMachineTransitionEventType = "pocket.StateMachineTransitionEvent" - // Consensus - HotstuffMessageContentType = "consensus.HotstuffMessage" + // Consensus - HotPOKT + HotstuffMessageContentType = "consensus.HotstuffMessage" + + // Consensus - State Sync StateSyncMessageContentType = "consensus.StateSyncMessage" // Utility diff --git a/shared/messaging/proto/events.proto b/shared/messaging/proto/events.proto index 13931be93..8fc6ff3e5 100644 --- a/shared/messaging/proto/events.proto +++ b/shared/messaging/proto/events.proto @@ -6,12 +6,14 @@ option go_package = "github.com/pokt-network/pocket/shared/messaging"; message NodeStartedEvent {} +// Notifies the node that the consensus module has started a new height message ConsensusNewHeightEvent { uint64 height = 1; } +// Notifies the node that the state of the node has transitioned through an event trigger message StateMachineTransitionEvent { string event = 1; string previous_state = 2; string new_state = 3; -} +} \ No newline at end of file diff --git a/shared/modules/bus_module.go b/shared/modules/bus_module.go index 14b329643..af5fa464a 100644 --- a/shared/modules/bus_module.go +++ b/shared/modules/bus_module.go @@ -13,11 +13,14 @@ const BusModuleName = "bus" // it, which could potentially be a feature rather than a bug. type EventsChannel chan *messaging.PocketEnvelope +type BusOption func(Bus) + type Bus interface { // Bus Events PublishEventToBus(e *messaging.PocketEnvelope) GetBusEvent() *messaging.PocketEnvelope GetEventBus() EventsChannel + GetDebugEventBus() EventsChannel // Dependency Injection / Service Discovery GetModulesRegistry() ModulesRegistry diff --git a/shared/modules/consensus_module.go b/shared/modules/consensus_module.go index 591e006dd..636721671 100644 --- a/shared/modules/consensus_module.go +++ b/shared/modules/consensus_module.go @@ -1,9 +1,8 @@ package modules -//go:generate mockgen -destination=./mocks/consensus_module_mock.go github.com/pokt-network/pocket/shared/modules ConsensusModule,ConsensusPacemaker,ConsensusStateSync,ConsensusDebugModule +//go:generate mockgen -destination=./mocks/consensus_module_mock.go github.com/pokt-network/pocket/shared/modules ConsensusModule,ConsensusPacemaker,ConsensusDebugModule import ( - "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/messaging" "google.golang.org/protobuf/types/known/anypb" ) @@ -21,22 +20,28 @@ type ConsensusModule interface { Module KeyholderModule - ConsensusStateSync ConsensusPacemaker ConsensusDebugModule - // Consensus Engine Handlers // TODO: Rename `HandleMessage` to a more specific name that is consistent with its business logic. + // Consensus message handlers HandleMessage(*anypb.Any) error + // State Sync message handlers HandleStateSyncMessage(*anypb.Any) error - // FSM transition event handler + + // Internal event handler such as FSM transition events HandleEvent(transitionMessageAny *anypb.Any) error // Consensus State Accessors + // CLEANUP: Add `Get` prefixes to these functions CurrentHeight() uint64 CurrentRound() uint64 CurrentStep() uint64 + + // Returns The cryptographic address associated with the node's private key. + // TECHDEBT: Consider removing this function altogether when we consolidate node identities + GetNodeAddress() string } // ConsensusPacemaker represents functions exposed by the Consensus module for Pacemaker specific business logic. @@ -67,24 +72,15 @@ type ConsensusPacemaker interface { GetNodeId() uint64 } -// ConsensusStateSync exposes functionality of the Consensus module for StateSync specific business logic. -// These functions are intended to only be called by the StateSync module. -// INVESTIGATE: This interface enable a fast implementation of state sync but look into a way of removing it in the future -type ConsensusStateSync interface { - GetNodeIdFromNodeAddress(string) (uint64, error) - GetNodeAddress() string -} - // ConsensusDebugModule exposes functionality used for testing & development purposes. // Not to be used in production. -// TODO: Add a flag so this is not compiled in the prod binary. +// TECHDEBT: Move this into a separate file with the `//go:build debug test` tags type ConsensusDebugModule interface { HandleDebugMessage(*messaging.DebugMessage) error SetHeight(uint64) SetRound(uint64) - SetStep(uint8) // REFACTOR: This should accept typesCons.HotstuffStep - SetBlock(*types.Block) + SetStep(uint8) SetUtilityUnitOfWork(UtilityUnitOfWork) diff --git a/shared/node.go b/shared/node.go index 39b905e38..c1cda79a1 100644 --- a/shared/node.go +++ b/shared/node.go @@ -163,36 +163,46 @@ func (m *Node) GetBus() modules.Bus { // TODO: Move all message types this is dependant on to the `messaging` package func (node *Node) handleEvent(message *messaging.PocketEnvelope) error { contentType := message.GetContentType() - logger.Global.Debug().Fields(map[string]any{ - "message": message, - "contentType": contentType, - }).Msg("node handling event") + // logger.Global.Debug().Fields(map[string]any{ + // "message": message, + // "contentType": contentType, + // }).Msg("node handling event") switch contentType { + case messaging.NodeStartedEventType: logger.Global.Info().Msg("Received NodeStartedEvent") if err := node.GetBus().GetStateMachineModule().SendEvent(coreTypes.StateMachineEvent_Start); err != nil { return err } + case messaging.HotstuffMessageContentType: return node.GetBus().GetConsensusModule().HandleMessage(message.Content) + case messaging.StateSyncMessageContentType: return node.GetBus().GetConsensusModule().HandleStateSyncMessage(message.Content) + case messaging.TxGossipMessageContentType: return node.GetBus().GetUtilityModule().HandleUtilityMessage(message.Content) - case messaging.DebugMessageEventType: - return node.handleDebugMessage(message) + case messaging.ConsensusNewHeightEventType: + err_consensus := node.GetBus().GetConsensusModule().HandleEvent(message.Content) err_p2p := node.GetBus().GetP2PModule().HandleEvent(message.Content) err_ibc := node.GetBus().GetIBCModule().HandleEvent(message.Content) - return errors.Join(err_p2p, err_ibc) + return errors.Join(err_p2p, err_ibc, err_consensus) + case messaging.StateMachineTransitionEventType: err_consensus := node.GetBus().GetConsensusModule().HandleEvent(message.Content) err_p2p := node.GetBus().GetP2PModule().HandleEvent(message.Content) return errors.Join(err_consensus, err_p2p) + + case messaging.DebugMessageEventType: + return node.handleDebugMessage(message) + default: logger.Global.Warn().Msgf("Unsupported message content type: %s", contentType) } + return nil } diff --git a/state_machine/docs/README.md b/state_machine/docs/README.md index d0e3a03ec..b816e89c0 100644 --- a/state_machine/docs/README.md +++ b/state_machine/docs/README.md @@ -66,7 +66,7 @@ These are the main building blocks: - **P2P_Bootstrapped**: The Consensus module handles `P2P_Bootstrapped` -> triggers a `Consensus_IsUnsynced` event -> transitions to `Consensus_Unsynced`. - **Consensus_Unsynced**: Node is out of sync, the Consensus module sends `Consensus_IsSyncing` event -> transitions to `Consensus_SyncMode` to start syncing with the rest of the network. -- **Consensus_SyncMode**: The Consensus module runs `StartSyncing()` and requests blocks one by one from peers in its address book. +- **Consensus_SyncMode**: The Consensus module runs `Start()` and requests blocks one by one from peers in its address book. - **Node finishes syncing**: When node completes syncing: - if the node is a validator, the Consensus module sends `Consensus_IsSyncedValidator` event -> transitions to `Consensus_Pacemaker`. - if the node is not a validator, the Consensus module sends `Consensus_IsSyncedNonValidator` event -> transitions to `Consensus_Synced`. diff --git a/state_machine/fsm.go b/state_machine/fsm.go index 08ab7c6a2..0c5227203 100644 --- a/state_machine/fsm.go +++ b/state_machine/fsm.go @@ -1,5 +1,7 @@ package state_machine +// TECHDEBT(#821): Remove the dependency of state sync on FSM, as well as the FSM in general. + import ( "github.com/looplab/fsm" coreTypes "github.com/pokt-network/pocket/shared/core/types" @@ -50,6 +52,7 @@ func NewNodeFSM(callbacks *fsm.Callbacks, options ...func(*fsm.FSM)) *fsm.FSM { Name: string(coreTypes.StateMachineEvent_Consensus_IsSyncedNonValidator), Src: []string{ string(coreTypes.StateMachineState_Consensus_SyncMode), + // string(coreTypes.StateMachineState_Consensus_Synced), }, Dst: string(coreTypes.StateMachineState_Consensus_Synced), }, @@ -58,6 +61,8 @@ func NewNodeFSM(callbacks *fsm.Callbacks, options ...func(*fsm.FSM)) *fsm.FSM { Src: []string{ string(coreTypes.StateMachineState_Consensus_Pacemaker), string(coreTypes.StateMachineState_Consensus_Synced), + // string(coreTypes.StateMachineState_Consensus_Unsynced), + string(coreTypes.StateMachineState_Consensus_SyncMode), string(coreTypes.StateMachineState_P2P_Bootstrapped), }, Dst: string(coreTypes.StateMachineState_Consensus_Unsynced), diff --git a/state_machine/module.go b/state_machine/module.go index 5ed8531af..9a8b6b505 100644 --- a/state_machine/module.go +++ b/state_machine/module.go @@ -1,5 +1,7 @@ package state_machine +// TECHDEBT(#821): Remove the dependency of state sync on FSM, as well as the FSM in general. + import ( "context" @@ -45,7 +47,6 @@ func (*stateMachineModule) Create(bus modules.Bus, options ...modules.ModuleOpti if err != nil { m.logger.Fatal().Err(err).Msg("failed to pack state machine transition event") } - bus.PublishEventToBus(newStateMachineTransitionEvent) }, }) diff --git a/utility/unit_of_work/actor.go b/utility/unit_of_work/actor.go index 4f99a0e70..75a523c6c 100644 --- a/utility/unit_of_work/actor.go +++ b/utility/unit_of_work/actor.go @@ -307,19 +307,19 @@ func (u *baseUtilityUnitOfWork) getActorExists(actorType coreTypes.ActorType, ad // IMPROVE: Need to re-evaluate the design of `Output Address` to support things like "rev-share" // and multiple output addresses. -func (u *baseUtilityUnitOfWork) getActorOutputAddress(actorType coreTypes.ActorType, operator []byte) ([]byte, coreTypes.Error) { +func (uow *baseUtilityUnitOfWork) getActorOutputAddress(actorType coreTypes.ActorType, operator []byte) ([]byte, coreTypes.Error) { var outputAddr []byte var err error switch actorType { case coreTypes.ActorType_ACTOR_TYPE_APP: - outputAddr, err = u.persistenceReadContext.GetAppOutputAddress(operator, u.height) + outputAddr, err = uow.persistenceReadContext.GetAppOutputAddress(operator, uow.height) case coreTypes.ActorType_ACTOR_TYPE_FISH: - outputAddr, err = u.persistenceReadContext.GetFishermanOutputAddress(operator, u.height) + outputAddr, err = uow.persistenceReadContext.GetFishermanOutputAddress(operator, uow.height) case coreTypes.ActorType_ACTOR_TYPE_SERVICER: - outputAddr, err = u.persistenceReadContext.GetServicerOutputAddress(operator, u.height) + outputAddr, err = uow.persistenceReadContext.GetServicerOutputAddress(operator, uow.height) case coreTypes.ActorType_ACTOR_TYPE_VAL: - outputAddr, err = u.persistenceReadContext.GetValidatorOutputAddress(operator, u.height) + outputAddr, err = uow.persistenceReadContext.GetValidatorOutputAddress(operator, uow.height) default: err = coreTypes.ErrUnknownActorType(actorType.String()) } diff --git a/utility/unit_of_work/module.go b/utility/unit_of_work/module.go index a654218ac..60df5f5aa 100644 --- a/utility/unit_of_work/module.go +++ b/utility/unit_of_work/module.go @@ -15,6 +15,7 @@ const ( var _ modules.UtilityUnitOfWork = &baseUtilityUnitOfWork{} +// TODO: Rename all `u * baseUtilityUnitOfWork` to `uow * baseUtilityUnitOfWork` for consistency type baseUtilityUnitOfWork struct { base_modules.IntegrableModule @@ -117,6 +118,8 @@ func (uow *baseUtilityUnitOfWork) Commit(quorumCert []byte) error { } func (uow *baseUtilityUnitOfWork) Release() error { + uow.logger.Info().Msg("releasing the unit of work...") + rwCtx := uow.persistenceRWContext if rwCtx != nil { uow.persistenceRWContext = nil