From 12d65377df38c1cfc911408e85f0031af5892b9e Mon Sep 17 00:00:00 2001 From: Bas van Kervel Date: Mon, 8 Jul 2024 16:15:33 -0400 Subject: [PATCH] node: Make stream sync more resilient --- core/node/events/stream.go | 4 + core/node/events/stream_cache_test.go | 5 + core/node/protocol/protocol.pb.go | 404 ++++++------ core/node/rpc/server.go | 15 +- core/node/rpc/service.go | 3 +- core/node/rpc/service_sync_streams.go | 55 ++ core/node/rpc/service_test.go | 507 +++++++++++++- core/node/rpc/sync/client/local.go | 136 ++++ core/node/rpc/sync/client/remote.go | 230 +++++++ core/node/rpc/sync/client/syncer_set.go | 234 +++++++ core/node/rpc/sync/handler.go | 139 ++++ core/node/rpc/sync/operation.go | 246 +++++++ core/node/rpc/sync/util.go | 28 + core/node/rpc/sync_receiver.go | 84 --- core/node/rpc/sync_streams.go | 843 ------------------------ core/node/rpc/sync_subscription.go | 423 ------------ core/node/rpc/tester_test.go | 14 +- packages/proto/src/gen/protocol_pb.ts | 14 + protocol/protocol.proto | 2 + 19 files changed, 1815 insertions(+), 1571 deletions(-) create mode 100644 core/node/rpc/service_sync_streams.go create mode 100644 core/node/rpc/sync/client/local.go create mode 100644 core/node/rpc/sync/client/remote.go create mode 100644 core/node/rpc/sync/client/syncer_set.go create mode 100644 core/node/rpc/sync/handler.go create mode 100644 core/node/rpc/sync/operation.go create mode 100644 core/node/rpc/sync/util.go delete mode 100644 core/node/rpc/sync_receiver.go delete mode 100644 core/node/rpc/sync_streams.go delete mode 100644 core/node/rpc/sync_subscription.go diff --git a/core/node/events/stream.go b/core/node/events/stream.go index fc8a391513..d965084536 100644 --- a/core/node/events/stream.go +++ b/core/node/events/stream.go @@ -33,8 +33,12 @@ type Stream interface { } type SyncResultReceiver interface { + // OnUpdate is called each time a new cookie is available for a stream OnUpdate(r *StreamAndCookie) + // OnSyncError is called when a sync subscription failed unrecoverable OnSyncError(err error) + // OnStreamSyncDown is called when updates for a stream could not be given. + OnStreamSyncDown(StreamId) } // TODO: refactor interfaces. diff --git a/core/node/events/stream_cache_test.go b/core/node/events/stream_cache_test.go index 711bc3dd82..7595d95fc5 100644 --- a/core/node/events/stream_cache_test.go +++ b/core/node/events/stream_cache_test.go @@ -199,6 +199,7 @@ func TestCacheEvictionWithFilledMiniBlockPool(t *testing.T) { type testStreamCacheViewEvictionSub struct { receivedStreamAndCookies []*protocol.StreamAndCookie receivedErrors []error + streamErrors []shared.StreamId } func (sub *testStreamCacheViewEvictionSub) OnUpdate(sac *protocol.StreamAndCookie) { @@ -209,6 +210,10 @@ func (sub *testStreamCacheViewEvictionSub) OnSyncError(err error) { sub.receivedErrors = append(sub.receivedErrors, err) } +func (sub *testStreamCacheViewEvictionSub) OnStreamSyncDown(streamID shared.StreamId) { + sub.streamErrors = append(sub.streamErrors, streamID) +} + func (sub *testStreamCacheViewEvictionSub) eventsReceived() int { count := 0 for _, sac := range sub.receivedStreamAndCookies { diff --git a/core/node/protocol/protocol.pb.go b/core/node/protocol/protocol.pb.go index ccfacd58de..7d0f3ea479 100644 --- a/core/node/protocol/protocol.pb.go +++ b/core/node/protocol/protocol.pb.go @@ -30,6 +30,7 @@ const ( SyncOp_SYNC_CLOSE SyncOp = 2 // close the sync SyncOp_SYNC_UPDATE SyncOp = 3 // update from server SyncOp_SYNC_PONG SyncOp = 4 // respond to the ping message from the client. + SyncOp_SYNC_DOWN SyncOp = 5 // indication that stream updates could (temporarily) not be provided ) // Enum value maps for SyncOp. @@ -40,6 +41,7 @@ var ( 2: "SYNC_CLOSE", 3: "SYNC_UPDATE", 4: "SYNC_PONG", + 5: "SYNC_DOWN", } SyncOp_value = map[string]int32{ "SYNC_UNSPECIFIED": 0, @@ -47,6 +49,7 @@ var ( "SYNC_CLOSE": 2, "SYNC_UPDATE": 3, "SYNC_PONG": 4, + "SYNC_DOWN": 5, } ) @@ -3424,6 +3427,7 @@ type SyncStreamsResponse struct { SyncOp SyncOp `protobuf:"varint,2,opt,name=sync_op,json=syncOp,proto3,enum=river.SyncOp" json:"sync_op,omitempty"` Stream *StreamAndCookie `protobuf:"bytes,3,opt,name=stream,proto3" json:"stream,omitempty"` PongNonce string `protobuf:"bytes,4,opt,name=pong_nonce,json=pongNonce,proto3" json:"pong_nonce,omitempty"` + StreamId []byte `protobuf:"bytes,5,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` } func (x *SyncStreamsResponse) Reset() { @@ -3486,6 +3490,13 @@ func (x *SyncStreamsResponse) GetPongNonce() string { return "" } +func (x *SyncStreamsResponse) GetStreamId() []byte { + if x != nil { + return x.StreamId + } + return nil +} + type AddStreamToSyncRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -7296,7 +7307,7 @@ var file_protocol_proto_rawDesc = []byte{ 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2c, 0x0a, 0x08, 0x73, 0x79, 0x6e, 0x63, 0x5f, 0x70, 0x6f, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x43, 0x6f, 0x6f, - 0x6b, 0x69, 0x65, 0x52, 0x07, 0x73, 0x79, 0x6e, 0x63, 0x50, 0x6f, 0x73, 0x22, 0xa5, 0x01, 0x0a, + 0x6b, 0x69, 0x65, 0x52, 0x07, 0x73, 0x79, 0x6e, 0x63, 0x50, 0x6f, 0x73, 0x22, 0xc2, 0x01, 0x0a, 0x13, 0x53, 0x79, 0x6e, 0x63, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x79, 0x6e, 0x63, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x79, 0x6e, 0x63, 0x49, 0x64, 0x12, 0x26, 0x0a, @@ -7307,203 +7318,206 @@ var file_protocol_proto_rawDesc = []byte{ 0x72, 0x65, 0x61, 0x6d, 0x41, 0x6e, 0x64, 0x43, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x52, 0x06, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x6f, 0x6e, 0x67, 0x5f, 0x6e, 0x6f, 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x6f, 0x6e, 0x67, 0x4e, - 0x6f, 0x6e, 0x63, 0x65, 0x22, 0x5f, 0x0a, 0x16, 0x41, 0x64, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x54, 0x6f, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, - 0x0a, 0x07, 0x73, 0x79, 0x6e, 0x63, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x73, 0x79, 0x6e, 0x63, 0x49, 0x64, 0x12, 0x2c, 0x0a, 0x08, 0x73, 0x79, 0x6e, 0x63, 0x5f, - 0x70, 0x6f, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x72, 0x69, 0x76, 0x65, - 0x72, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x43, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x52, 0x07, 0x73, 0x79, - 0x6e, 0x63, 0x50, 0x6f, 0x73, 0x22, 0x19, 0x0a, 0x17, 0x41, 0x64, 0x64, 0x53, 0x74, 0x72, 0x65, - 0x61, 0x6d, 0x54, 0x6f, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x53, 0x0a, 0x1b, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, - 0x46, 0x72, 0x6f, 0x6d, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x17, 0x0a, 0x07, 0x73, 0x79, 0x6e, 0x63, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x73, 0x79, 0x6e, 0x63, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x74, 0x72, 0x65, - 0x61, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x73, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x49, 0x64, 0x22, 0x1e, 0x0a, 0x1c, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, - 0x74, 0x72, 0x65, 0x61, 0x6d, 0x46, 0x72, 0x6f, 0x6d, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x2c, 0x0a, 0x11, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x53, - 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x79, - 0x6e, 0x63, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x79, 0x6e, - 0x63, 0x49, 0x64, 0x22, 0x14, 0x0a, 0x12, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x53, 0x79, 0x6e, - 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x40, 0x0a, 0x0f, 0x50, 0x69, 0x6e, - 0x67, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, + 0x6f, 0x6e, 0x63, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x69, + 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x49, + 0x64, 0x22, 0x5f, 0x0a, 0x16, 0x41, 0x64, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, 0x6f, + 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x73, + 0x79, 0x6e, 0x63, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x79, + 0x6e, 0x63, 0x49, 0x64, 0x12, 0x2c, 0x0a, 0x08, 0x73, 0x79, 0x6e, 0x63, 0x5f, 0x70, 0x6f, 0x73, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x53, + 0x79, 0x6e, 0x63, 0x43, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x52, 0x07, 0x73, 0x79, 0x6e, 0x63, 0x50, + 0x6f, 0x73, 0x22, 0x19, 0x0a, 0x17, 0x41, 0x64, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, + 0x6f, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x53, 0x0a, + 0x1b, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x46, 0x72, 0x6f, + 0x6d, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x79, 0x6e, 0x63, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, - 0x79, 0x6e, 0x63, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x6f, 0x6e, 0x63, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6e, 0x6f, 0x6e, 0x63, 0x65, 0x22, 0x12, 0x0a, 0x10, 0x50, - 0x69, 0x6e, 0x67, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x23, 0x0a, 0x0b, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, - 0x0a, 0x05, 0x64, 0x65, 0x62, 0x75, 0x67, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x64, - 0x65, 0x62, 0x75, 0x67, 0x22, 0x7f, 0x0a, 0x0c, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x67, 0x72, 0x61, 0x66, 0x66, 0x69, 0x74, 0x69, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x67, 0x72, 0x61, 0x66, 0x66, 0x69, 0x74, 0x69, - 0x12, 0x39, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x2a, 0x5c, 0x0a, 0x06, 0x53, 0x79, 0x6e, 0x63, 0x4f, 0x70, 0x12, - 0x14, 0x0a, 0x10, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, - 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x4e, 0x45, - 0x57, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x43, 0x4c, 0x4f, 0x53, - 0x45, 0x10, 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x55, 0x50, 0x44, 0x41, - 0x54, 0x45, 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x50, 0x4f, 0x4e, - 0x47, 0x10, 0x04, 0x2a, 0x4c, 0x0a, 0x0c, 0x4d, 0x65, 0x6d, 0x62, 0x65, 0x72, 0x73, 0x68, 0x69, - 0x70, 0x4f, 0x70, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x4f, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, - 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x4f, 0x5f, 0x49, 0x4e, - 0x56, 0x49, 0x54, 0x45, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x4f, 0x5f, 0x4a, 0x4f, 0x49, - 0x4e, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x4f, 0x5f, 0x4c, 0x45, 0x41, 0x56, 0x45, 0x10, - 0x03, 0x2a, 0x4f, 0x0a, 0x09, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x4f, 0x70, 0x12, 0x12, - 0x0a, 0x0e, 0x43, 0x4f, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, - 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x43, 0x4f, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, - 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x43, 0x4f, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x44, - 0x10, 0x02, 0x12, 0x0e, 0x0a, 0x0a, 0x43, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x44, - 0x10, 0x04, 0x2a, 0xdd, 0x0a, 0x0a, 0x03, 0x45, 0x72, 0x72, 0x12, 0x13, 0x0a, 0x0f, 0x45, 0x52, - 0x52, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, - 0x0c, 0x0a, 0x08, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0b, 0x0a, - 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x02, 0x12, 0x14, 0x0a, 0x10, 0x49, 0x4e, - 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x41, 0x52, 0x47, 0x55, 0x4d, 0x45, 0x4e, 0x54, 0x10, 0x03, - 0x12, 0x15, 0x0a, 0x11, 0x44, 0x45, 0x41, 0x44, 0x4c, 0x49, 0x4e, 0x45, 0x5f, 0x45, 0x58, 0x43, - 0x45, 0x45, 0x44, 0x45, 0x44, 0x10, 0x04, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x54, 0x5f, 0x46, - 0x4f, 0x55, 0x4e, 0x44, 0x10, 0x05, 0x12, 0x12, 0x0a, 0x0e, 0x41, 0x4c, 0x52, 0x45, 0x41, 0x44, - 0x59, 0x5f, 0x45, 0x58, 0x49, 0x53, 0x54, 0x53, 0x10, 0x06, 0x12, 0x15, 0x0a, 0x11, 0x50, 0x45, - 0x52, 0x4d, 0x49, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x44, 0x45, 0x4e, 0x49, 0x45, 0x44, 0x10, - 0x07, 0x12, 0x16, 0x0a, 0x12, 0x52, 0x45, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x45, 0x58, - 0x48, 0x41, 0x55, 0x53, 0x54, 0x45, 0x44, 0x10, 0x08, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x41, 0x49, - 0x4c, 0x45, 0x44, 0x5f, 0x50, 0x52, 0x45, 0x43, 0x4f, 0x4e, 0x44, 0x49, 0x54, 0x49, 0x4f, 0x4e, - 0x10, 0x09, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x42, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x10, 0x0a, 0x12, - 0x10, 0x0a, 0x0c, 0x4f, 0x55, 0x54, 0x5f, 0x4f, 0x46, 0x5f, 0x52, 0x41, 0x4e, 0x47, 0x45, 0x10, - 0x0b, 0x12, 0x11, 0x0a, 0x0d, 0x55, 0x4e, 0x49, 0x4d, 0x50, 0x4c, 0x45, 0x4d, 0x45, 0x4e, 0x54, - 0x45, 0x44, 0x10, 0x0c, 0x12, 0x0c, 0x0a, 0x08, 0x49, 0x4e, 0x54, 0x45, 0x52, 0x4e, 0x41, 0x4c, - 0x10, 0x0d, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x41, 0x56, 0x41, 0x49, 0x4c, 0x41, 0x42, 0x4c, - 0x45, 0x10, 0x0e, 0x12, 0x0d, 0x0a, 0x09, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x4c, 0x4f, 0x53, 0x53, - 0x10, 0x0f, 0x12, 0x13, 0x0a, 0x0f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, - 0x43, 0x41, 0x54, 0x45, 0x44, 0x10, 0x10, 0x12, 0x0f, 0x0a, 0x0b, 0x44, 0x45, 0x42, 0x55, 0x47, - 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x11, 0x12, 0x11, 0x0a, 0x0d, 0x42, 0x41, 0x44, 0x5f, - 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, 0x49, 0x44, 0x10, 0x12, 0x12, 0x1e, 0x0a, 0x1a, 0x42, - 0x41, 0x44, 0x5f, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x49, - 0x4f, 0x4e, 0x5f, 0x50, 0x41, 0x52, 0x41, 0x4d, 0x53, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x49, - 0x4e, 0x54, 0x45, 0x52, 0x4e, 0x41, 0x4c, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x53, 0x57, - 0x49, 0x54, 0x43, 0x48, 0x10, 0x14, 0x12, 0x10, 0x0a, 0x0c, 0x42, 0x41, 0x44, 0x5f, 0x45, 0x56, - 0x45, 0x4e, 0x54, 0x5f, 0x49, 0x44, 0x10, 0x15, 0x12, 0x17, 0x0a, 0x13, 0x42, 0x41, 0x44, 0x5f, - 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x53, 0x49, 0x47, 0x4e, 0x41, 0x54, 0x55, 0x52, 0x45, 0x10, - 0x16, 0x12, 0x13, 0x0a, 0x0f, 0x42, 0x41, 0x44, 0x5f, 0x48, 0x41, 0x53, 0x48, 0x5f, 0x46, 0x4f, - 0x52, 0x4d, 0x41, 0x54, 0x10, 0x17, 0x12, 0x1b, 0x0a, 0x17, 0x42, 0x41, 0x44, 0x5f, 0x50, 0x52, - 0x45, 0x56, 0x5f, 0x4d, 0x49, 0x4e, 0x49, 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x5f, 0x48, 0x41, 0x53, - 0x48, 0x10, 0x18, 0x12, 0x16, 0x0a, 0x12, 0x4e, 0x4f, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, - 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x19, 0x12, 0x0d, 0x0a, 0x09, 0x42, - 0x41, 0x44, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x10, 0x1a, 0x12, 0x12, 0x0a, 0x0e, 0x55, 0x53, - 0x45, 0x52, 0x5f, 0x43, 0x41, 0x4e, 0x54, 0x5f, 0x50, 0x4f, 0x53, 0x54, 0x10, 0x1b, 0x12, 0x15, - 0x0a, 0x11, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, 0x42, 0x41, 0x44, 0x5f, 0x48, 0x41, 0x53, - 0x48, 0x45, 0x53, 0x10, 0x1c, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, - 0x45, 0x4d, 0x50, 0x54, 0x59, 0x10, 0x1d, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x54, 0x52, 0x45, 0x41, - 0x4d, 0x5f, 0x42, 0x41, 0x44, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x10, 0x1e, 0x12, 0x14, 0x0a, - 0x10, 0x42, 0x41, 0x44, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x47, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x49, - 0x47, 0x10, 0x1f, 0x12, 0x12, 0x0a, 0x0e, 0x42, 0x41, 0x44, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, - 0x43, 0x5f, 0x4b, 0x45, 0x59, 0x10, 0x20, 0x12, 0x0f, 0x0a, 0x0b, 0x42, 0x41, 0x44, 0x5f, 0x50, - 0x41, 0x59, 0x4c, 0x4f, 0x41, 0x44, 0x10, 0x21, 0x12, 0x12, 0x0a, 0x0e, 0x42, 0x41, 0x44, 0x5f, - 0x48, 0x45, 0x58, 0x5f, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x22, 0x12, 0x12, 0x0a, 0x0e, - 0x42, 0x41, 0x44, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x48, 0x41, 0x53, 0x48, 0x10, 0x23, - 0x12, 0x13, 0x0a, 0x0f, 0x42, 0x41, 0x44, 0x5f, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x43, 0x4f, 0x4f, - 0x4b, 0x49, 0x45, 0x10, 0x24, 0x12, 0x13, 0x0a, 0x0f, 0x44, 0x55, 0x50, 0x4c, 0x49, 0x43, 0x41, - 0x54, 0x45, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x10, 0x25, 0x12, 0x0d, 0x0a, 0x09, 0x42, 0x41, - 0x44, 0x5f, 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x10, 0x26, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x54, 0x52, - 0x45, 0x41, 0x4d, 0x5f, 0x4e, 0x4f, 0x5f, 0x49, 0x4e, 0x43, 0x45, 0x50, 0x54, 0x49, 0x4f, 0x4e, - 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x10, 0x27, 0x12, 0x14, 0x0a, 0x10, 0x42, 0x41, 0x44, 0x5f, - 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, 0x52, 0x10, 0x28, 0x12, 0x15, - 0x0a, 0x11, 0x42, 0x41, 0x44, 0x5f, 0x4d, 0x49, 0x4e, 0x49, 0x50, 0x4f, 0x4f, 0x4c, 0x5f, 0x53, - 0x4c, 0x4f, 0x54, 0x10, 0x29, 0x12, 0x17, 0x0a, 0x13, 0x42, 0x41, 0x44, 0x5f, 0x43, 0x52, 0x45, - 0x41, 0x54, 0x4f, 0x52, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, 0x10, 0x2a, 0x12, 0x12, - 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x4c, 0x45, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x47, 0x41, 0x54, 0x45, - 0x10, 0x2b, 0x12, 0x21, 0x0a, 0x1d, 0x42, 0x41, 0x44, 0x5f, 0x4c, 0x49, 0x4e, 0x4b, 0x5f, 0x57, - 0x41, 0x4c, 0x4c, 0x45, 0x54, 0x5f, 0x42, 0x41, 0x44, 0x5f, 0x53, 0x49, 0x47, 0x4e, 0x41, 0x54, - 0x55, 0x52, 0x45, 0x10, 0x2c, 0x12, 0x13, 0x0a, 0x0f, 0x42, 0x41, 0x44, 0x5f, 0x52, 0x4f, 0x4f, - 0x54, 0x5f, 0x4b, 0x45, 0x59, 0x5f, 0x49, 0x44, 0x10, 0x2d, 0x12, 0x10, 0x0a, 0x0c, 0x55, 0x4e, - 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x4e, 0x4f, 0x44, 0x45, 0x10, 0x2e, 0x12, 0x18, 0x0a, 0x14, - 0x44, 0x42, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x46, 0x41, 0x49, - 0x4c, 0x55, 0x52, 0x45, 0x10, 0x2f, 0x12, 0x1e, 0x0a, 0x1a, 0x4d, 0x49, 0x4e, 0x49, 0x42, 0x4c, - 0x4f, 0x43, 0x4b, 0x53, 0x5f, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x46, 0x41, 0x49, - 0x4c, 0x55, 0x52, 0x45, 0x10, 0x30, 0x12, 0x0f, 0x0a, 0x0b, 0x42, 0x41, 0x44, 0x5f, 0x41, 0x44, - 0x44, 0x52, 0x45, 0x53, 0x53, 0x10, 0x31, 0x12, 0x0f, 0x0a, 0x0b, 0x42, 0x55, 0x46, 0x46, 0x45, - 0x52, 0x5f, 0x46, 0x55, 0x4c, 0x4c, 0x10, 0x32, 0x12, 0x0e, 0x0a, 0x0a, 0x42, 0x41, 0x44, 0x5f, - 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x10, 0x33, 0x12, 0x10, 0x0a, 0x0c, 0x42, 0x41, 0x44, 0x5f, - 0x43, 0x4f, 0x4e, 0x54, 0x52, 0x41, 0x43, 0x54, 0x10, 0x34, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x41, - 0x4e, 0x4e, 0x4f, 0x54, 0x5f, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x10, 0x35, 0x12, 0x1d, - 0x0a, 0x19, 0x43, 0x41, 0x4e, 0x4e, 0x4f, 0x54, 0x5f, 0x47, 0x45, 0x54, 0x5f, 0x4c, 0x49, 0x4e, - 0x4b, 0x45, 0x44, 0x5f, 0x57, 0x41, 0x4c, 0x4c, 0x45, 0x54, 0x53, 0x10, 0x36, 0x12, 0x1d, 0x0a, - 0x19, 0x43, 0x41, 0x4e, 0x4e, 0x4f, 0x54, 0x5f, 0x43, 0x48, 0x45, 0x43, 0x4b, 0x5f, 0x45, 0x4e, - 0x54, 0x49, 0x54, 0x4c, 0x45, 0x4d, 0x45, 0x4e, 0x54, 0x53, 0x10, 0x37, 0x12, 0x18, 0x0a, 0x14, - 0x43, 0x41, 0x4e, 0x4e, 0x4f, 0x54, 0x5f, 0x43, 0x41, 0x4c, 0x4c, 0x5f, 0x43, 0x4f, 0x4e, 0x54, - 0x52, 0x41, 0x43, 0x54, 0x10, 0x38, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x50, 0x41, 0x43, 0x45, 0x5f, - 0x44, 0x49, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x44, 0x10, 0x39, 0x12, 0x14, 0x0a, 0x10, 0x43, 0x48, - 0x41, 0x4e, 0x4e, 0x45, 0x4c, 0x5f, 0x44, 0x49, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x44, 0x10, 0x3a, - 0x12, 0x15, 0x0a, 0x11, 0x57, 0x52, 0x4f, 0x4e, 0x47, 0x5f, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x10, 0x3b, 0x12, 0x1b, 0x0a, 0x17, 0x4d, 0x49, 0x4e, 0x49, 0x50, - 0x4f, 0x4f, 0x4c, 0x5f, 0x4d, 0x49, 0x53, 0x53, 0x49, 0x4e, 0x47, 0x5f, 0x45, 0x56, 0x45, 0x4e, - 0x54, 0x53, 0x10, 0x3c, 0x12, 0x1e, 0x0a, 0x1a, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, 0x4c, - 0x41, 0x53, 0x54, 0x5f, 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x5f, 0x4d, 0x49, 0x53, 0x4d, 0x41, 0x54, - 0x43, 0x48, 0x10, 0x3d, 0x12, 0x1c, 0x0a, 0x18, 0x44, 0x4f, 0x57, 0x4e, 0x53, 0x54, 0x52, 0x45, - 0x41, 0x4d, 0x5f, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, - 0x10, 0x3e, 0x32, 0xf6, 0x06, 0x0a, 0x0d, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x12, 0x47, 0x0a, 0x0c, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x74, - 0x72, 0x65, 0x61, 0x6d, 0x12, 0x1a, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x43, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1b, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, - 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, - 0x09, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x17, 0x2e, 0x72, 0x69, 0x76, - 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x53, - 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, - 0x0b, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x45, 0x78, 0x12, 0x19, 0x2e, 0x72, - 0x69, 0x76, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x45, 0x78, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, - 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x45, 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x4a, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4d, 0x69, 0x6e, 0x69, - 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x12, 0x1b, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x47, - 0x65, 0x74, 0x4d, 0x69, 0x6e, 0x69, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x4d, - 0x69, 0x6e, 0x69, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x5f, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x4c, 0x61, 0x73, 0x74, 0x4d, 0x69, 0x6e, 0x69, - 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x61, 0x73, 0x68, 0x12, 0x22, 0x2e, 0x72, 0x69, 0x76, 0x65, + 0x79, 0x6e, 0x63, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, + 0x49, 0x64, 0x22, 0x1e, 0x0a, 0x1c, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x46, 0x72, 0x6f, 0x6d, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x2c, 0x0a, 0x11, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x53, 0x79, 0x6e, 0x63, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x79, 0x6e, 0x63, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x79, 0x6e, 0x63, 0x49, 0x64, + 0x22, 0x14, 0x0a, 0x12, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x40, 0x0a, 0x0f, 0x50, 0x69, 0x6e, 0x67, 0x53, 0x79, + 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x79, 0x6e, + 0x63, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x79, 0x6e, 0x63, + 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x6f, 0x6e, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x6e, 0x6f, 0x6e, 0x63, 0x65, 0x22, 0x12, 0x0a, 0x10, 0x50, 0x69, 0x6e, 0x67, + 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x23, 0x0a, 0x0b, + 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x64, + 0x65, 0x62, 0x75, 0x67, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x64, 0x65, 0x62, 0x75, + 0x67, 0x22, 0x7f, 0x0a, 0x0c, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x67, 0x72, 0x61, 0x66, 0x66, 0x69, 0x74, 0x69, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x67, 0x72, 0x61, 0x66, 0x66, 0x69, 0x74, 0x69, 0x12, 0x39, 0x0a, + 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, + 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x2a, 0x6b, 0x0a, 0x06, 0x53, 0x79, 0x6e, 0x63, 0x4f, 0x70, 0x12, 0x14, 0x0a, 0x10, + 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, + 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x4e, 0x45, 0x57, 0x10, 0x01, + 0x12, 0x0e, 0x0a, 0x0a, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x02, + 0x12, 0x0f, 0x0a, 0x0b, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, + 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x50, 0x4f, 0x4e, 0x47, 0x10, 0x04, + 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x10, 0x05, 0x2a, + 0x4c, 0x0a, 0x0c, 0x4d, 0x65, 0x6d, 0x62, 0x65, 0x72, 0x73, 0x68, 0x69, 0x70, 0x4f, 0x70, 0x12, + 0x12, 0x0a, 0x0e, 0x53, 0x4f, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, + 0x44, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x4f, 0x5f, 0x49, 0x4e, 0x56, 0x49, 0x54, 0x45, + 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x4f, 0x5f, 0x4a, 0x4f, 0x49, 0x4e, 0x10, 0x02, 0x12, + 0x0c, 0x0a, 0x08, 0x53, 0x4f, 0x5f, 0x4c, 0x45, 0x41, 0x56, 0x45, 0x10, 0x03, 0x2a, 0x4f, 0x0a, + 0x09, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x4f, 0x70, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x4f, + 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0e, + 0x0a, 0x0a, 0x43, 0x4f, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0e, + 0x0a, 0x0a, 0x43, 0x4f, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x02, 0x12, 0x0e, + 0x0a, 0x0a, 0x43, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x44, 0x10, 0x04, 0x2a, 0xdd, + 0x0a, 0x0a, 0x03, 0x45, 0x72, 0x72, 0x12, 0x13, 0x0a, 0x0f, 0x45, 0x52, 0x52, 0x5f, 0x55, 0x4e, + 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x43, + 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, + 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x02, 0x12, 0x14, 0x0a, 0x10, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, + 0x44, 0x5f, 0x41, 0x52, 0x47, 0x55, 0x4d, 0x45, 0x4e, 0x54, 0x10, 0x03, 0x12, 0x15, 0x0a, 0x11, + 0x44, 0x45, 0x41, 0x44, 0x4c, 0x49, 0x4e, 0x45, 0x5f, 0x45, 0x58, 0x43, 0x45, 0x45, 0x44, 0x45, + 0x44, 0x10, 0x04, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x54, 0x5f, 0x46, 0x4f, 0x55, 0x4e, 0x44, + 0x10, 0x05, 0x12, 0x12, 0x0a, 0x0e, 0x41, 0x4c, 0x52, 0x45, 0x41, 0x44, 0x59, 0x5f, 0x45, 0x58, + 0x49, 0x53, 0x54, 0x53, 0x10, 0x06, 0x12, 0x15, 0x0a, 0x11, 0x50, 0x45, 0x52, 0x4d, 0x49, 0x53, + 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x44, 0x45, 0x4e, 0x49, 0x45, 0x44, 0x10, 0x07, 0x12, 0x16, 0x0a, + 0x12, 0x52, 0x45, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x45, 0x58, 0x48, 0x41, 0x55, 0x53, + 0x54, 0x45, 0x44, 0x10, 0x08, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x5f, + 0x50, 0x52, 0x45, 0x43, 0x4f, 0x4e, 0x44, 0x49, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x09, 0x12, 0x0b, + 0x0a, 0x07, 0x41, 0x42, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x10, 0x0a, 0x12, 0x10, 0x0a, 0x0c, 0x4f, + 0x55, 0x54, 0x5f, 0x4f, 0x46, 0x5f, 0x52, 0x41, 0x4e, 0x47, 0x45, 0x10, 0x0b, 0x12, 0x11, 0x0a, + 0x0d, 0x55, 0x4e, 0x49, 0x4d, 0x50, 0x4c, 0x45, 0x4d, 0x45, 0x4e, 0x54, 0x45, 0x44, 0x10, 0x0c, + 0x12, 0x0c, 0x0a, 0x08, 0x49, 0x4e, 0x54, 0x45, 0x52, 0x4e, 0x41, 0x4c, 0x10, 0x0d, 0x12, 0x0f, + 0x0a, 0x0b, 0x55, 0x4e, 0x41, 0x56, 0x41, 0x49, 0x4c, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x0e, 0x12, + 0x0d, 0x0a, 0x09, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x4c, 0x4f, 0x53, 0x53, 0x10, 0x0f, 0x12, 0x13, + 0x0a, 0x0f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, 0x54, 0x45, + 0x44, 0x10, 0x10, 0x12, 0x0f, 0x0a, 0x0b, 0x44, 0x45, 0x42, 0x55, 0x47, 0x5f, 0x45, 0x52, 0x52, + 0x4f, 0x52, 0x10, 0x11, 0x12, 0x11, 0x0a, 0x0d, 0x42, 0x41, 0x44, 0x5f, 0x53, 0x54, 0x52, 0x45, + 0x41, 0x4d, 0x5f, 0x49, 0x44, 0x10, 0x12, 0x12, 0x1e, 0x0a, 0x1a, 0x42, 0x41, 0x44, 0x5f, 0x53, + 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x50, + 0x41, 0x52, 0x41, 0x4d, 0x53, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x49, 0x4e, 0x54, 0x45, 0x52, + 0x4e, 0x41, 0x4c, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x53, 0x57, 0x49, 0x54, 0x43, 0x48, + 0x10, 0x14, 0x12, 0x10, 0x0a, 0x0c, 0x42, 0x41, 0x44, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, + 0x49, 0x44, 0x10, 0x15, 0x12, 0x17, 0x0a, 0x13, 0x42, 0x41, 0x44, 0x5f, 0x45, 0x56, 0x45, 0x4e, + 0x54, 0x5f, 0x53, 0x49, 0x47, 0x4e, 0x41, 0x54, 0x55, 0x52, 0x45, 0x10, 0x16, 0x12, 0x13, 0x0a, + 0x0f, 0x42, 0x41, 0x44, 0x5f, 0x48, 0x41, 0x53, 0x48, 0x5f, 0x46, 0x4f, 0x52, 0x4d, 0x41, 0x54, + 0x10, 0x17, 0x12, 0x1b, 0x0a, 0x17, 0x42, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x45, 0x56, 0x5f, 0x4d, + 0x49, 0x4e, 0x49, 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x5f, 0x48, 0x41, 0x53, 0x48, 0x10, 0x18, 0x12, + 0x16, 0x0a, 0x12, 0x4e, 0x4f, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x53, 0x50, 0x45, 0x43, + 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x19, 0x12, 0x0d, 0x0a, 0x09, 0x42, 0x41, 0x44, 0x5f, 0x45, + 0x56, 0x45, 0x4e, 0x54, 0x10, 0x1a, 0x12, 0x12, 0x0a, 0x0e, 0x55, 0x53, 0x45, 0x52, 0x5f, 0x43, + 0x41, 0x4e, 0x54, 0x5f, 0x50, 0x4f, 0x53, 0x54, 0x10, 0x1b, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x54, + 0x52, 0x45, 0x41, 0x4d, 0x5f, 0x42, 0x41, 0x44, 0x5f, 0x48, 0x41, 0x53, 0x48, 0x45, 0x53, 0x10, + 0x1c, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, 0x45, 0x4d, 0x50, 0x54, + 0x59, 0x10, 0x1d, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, 0x42, 0x41, + 0x44, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x10, 0x1e, 0x12, 0x14, 0x0a, 0x10, 0x42, 0x41, 0x44, + 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x47, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x49, 0x47, 0x10, 0x1f, 0x12, + 0x12, 0x0a, 0x0e, 0x42, 0x41, 0x44, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x4b, 0x45, + 0x59, 0x10, 0x20, 0x12, 0x0f, 0x0a, 0x0b, 0x42, 0x41, 0x44, 0x5f, 0x50, 0x41, 0x59, 0x4c, 0x4f, + 0x41, 0x44, 0x10, 0x21, 0x12, 0x12, 0x0a, 0x0e, 0x42, 0x41, 0x44, 0x5f, 0x48, 0x45, 0x58, 0x5f, + 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x22, 0x12, 0x12, 0x0a, 0x0e, 0x42, 0x41, 0x44, 0x5f, + 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x48, 0x41, 0x53, 0x48, 0x10, 0x23, 0x12, 0x13, 0x0a, 0x0f, + 0x42, 0x41, 0x44, 0x5f, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x43, 0x4f, 0x4f, 0x4b, 0x49, 0x45, 0x10, + 0x24, 0x12, 0x13, 0x0a, 0x0f, 0x44, 0x55, 0x50, 0x4c, 0x49, 0x43, 0x41, 0x54, 0x45, 0x5f, 0x45, + 0x56, 0x45, 0x4e, 0x54, 0x10, 0x25, 0x12, 0x0d, 0x0a, 0x09, 0x42, 0x41, 0x44, 0x5f, 0x42, 0x4c, + 0x4f, 0x43, 0x4b, 0x10, 0x26, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, + 0x4e, 0x4f, 0x5f, 0x49, 0x4e, 0x43, 0x45, 0x50, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x45, 0x56, 0x45, + 0x4e, 0x54, 0x10, 0x27, 0x12, 0x14, 0x0a, 0x10, 0x42, 0x41, 0x44, 0x5f, 0x42, 0x4c, 0x4f, 0x43, + 0x4b, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, 0x52, 0x10, 0x28, 0x12, 0x15, 0x0a, 0x11, 0x42, 0x41, + 0x44, 0x5f, 0x4d, 0x49, 0x4e, 0x49, 0x50, 0x4f, 0x4f, 0x4c, 0x5f, 0x53, 0x4c, 0x4f, 0x54, 0x10, + 0x29, 0x12, 0x17, 0x0a, 0x13, 0x42, 0x41, 0x44, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x4f, 0x52, + 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, 0x10, 0x2a, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, + 0x41, 0x4c, 0x45, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x47, 0x41, 0x54, 0x45, 0x10, 0x2b, 0x12, 0x21, + 0x0a, 0x1d, 0x42, 0x41, 0x44, 0x5f, 0x4c, 0x49, 0x4e, 0x4b, 0x5f, 0x57, 0x41, 0x4c, 0x4c, 0x45, + 0x54, 0x5f, 0x42, 0x41, 0x44, 0x5f, 0x53, 0x49, 0x47, 0x4e, 0x41, 0x54, 0x55, 0x52, 0x45, 0x10, + 0x2c, 0x12, 0x13, 0x0a, 0x0f, 0x42, 0x41, 0x44, 0x5f, 0x52, 0x4f, 0x4f, 0x54, 0x5f, 0x4b, 0x45, + 0x59, 0x5f, 0x49, 0x44, 0x10, 0x2d, 0x12, 0x10, 0x0a, 0x0c, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, + 0x4e, 0x5f, 0x4e, 0x4f, 0x44, 0x45, 0x10, 0x2e, 0x12, 0x18, 0x0a, 0x14, 0x44, 0x42, 0x5f, 0x4f, + 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, + 0x10, 0x2f, 0x12, 0x1e, 0x0a, 0x1a, 0x4d, 0x49, 0x4e, 0x49, 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x53, + 0x5f, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, + 0x10, 0x30, 0x12, 0x0f, 0x0a, 0x0b, 0x42, 0x41, 0x44, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, + 0x53, 0x10, 0x31, 0x12, 0x0f, 0x0a, 0x0b, 0x42, 0x55, 0x46, 0x46, 0x45, 0x52, 0x5f, 0x46, 0x55, + 0x4c, 0x4c, 0x10, 0x32, 0x12, 0x0e, 0x0a, 0x0a, 0x42, 0x41, 0x44, 0x5f, 0x43, 0x4f, 0x4e, 0x46, + 0x49, 0x47, 0x10, 0x33, 0x12, 0x10, 0x0a, 0x0c, 0x42, 0x41, 0x44, 0x5f, 0x43, 0x4f, 0x4e, 0x54, + 0x52, 0x41, 0x43, 0x54, 0x10, 0x34, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x41, 0x4e, 0x4e, 0x4f, 0x54, + 0x5f, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x10, 0x35, 0x12, 0x1d, 0x0a, 0x19, 0x43, 0x41, + 0x4e, 0x4e, 0x4f, 0x54, 0x5f, 0x47, 0x45, 0x54, 0x5f, 0x4c, 0x49, 0x4e, 0x4b, 0x45, 0x44, 0x5f, + 0x57, 0x41, 0x4c, 0x4c, 0x45, 0x54, 0x53, 0x10, 0x36, 0x12, 0x1d, 0x0a, 0x19, 0x43, 0x41, 0x4e, + 0x4e, 0x4f, 0x54, 0x5f, 0x43, 0x48, 0x45, 0x43, 0x4b, 0x5f, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x4c, + 0x45, 0x4d, 0x45, 0x4e, 0x54, 0x53, 0x10, 0x37, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x41, 0x4e, 0x4e, + 0x4f, 0x54, 0x5f, 0x43, 0x41, 0x4c, 0x4c, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x52, 0x41, 0x43, 0x54, + 0x10, 0x38, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x50, 0x41, 0x43, 0x45, 0x5f, 0x44, 0x49, 0x53, 0x41, + 0x42, 0x4c, 0x45, 0x44, 0x10, 0x39, 0x12, 0x14, 0x0a, 0x10, 0x43, 0x48, 0x41, 0x4e, 0x4e, 0x45, + 0x4c, 0x5f, 0x44, 0x49, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x44, 0x10, 0x3a, 0x12, 0x15, 0x0a, 0x11, + 0x57, 0x52, 0x4f, 0x4e, 0x47, 0x5f, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x10, 0x3b, 0x12, 0x1b, 0x0a, 0x17, 0x4d, 0x49, 0x4e, 0x49, 0x50, 0x4f, 0x4f, 0x4c, 0x5f, + 0x4d, 0x49, 0x53, 0x53, 0x49, 0x4e, 0x47, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x53, 0x10, 0x3c, + 0x12, 0x1e, 0x0a, 0x1a, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, 0x4c, 0x41, 0x53, 0x54, 0x5f, + 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x5f, 0x4d, 0x49, 0x53, 0x4d, 0x41, 0x54, 0x43, 0x48, 0x10, 0x3d, + 0x12, 0x1c, 0x0a, 0x18, 0x44, 0x4f, 0x57, 0x4e, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, 0x4e, + 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x3e, 0x32, 0xf6, + 0x06, 0x0a, 0x0d, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x12, 0x47, 0x0a, 0x0c, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, + 0x12, 0x1a, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x72, + 0x69, 0x76, 0x65, 0x72, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x09, 0x47, 0x65, 0x74, + 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x17, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x47, + 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x18, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x0b, 0x47, 0x65, 0x74, + 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x45, 0x78, 0x12, 0x19, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, + 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x45, 0x78, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x53, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x45, 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, + 0x01, 0x12, 0x4a, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4d, 0x69, 0x6e, 0x69, 0x62, 0x6c, 0x6f, 0x63, + 0x6b, 0x73, 0x12, 0x1b, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x69, + 0x6e, 0x69, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1c, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x69, 0x6e, 0x69, 0x62, + 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5f, 0x0a, + 0x14, 0x47, 0x65, 0x74, 0x4c, 0x61, 0x73, 0x74, 0x4d, 0x69, 0x6e, 0x69, 0x62, 0x6c, 0x6f, 0x63, + 0x6b, 0x48, 0x61, 0x73, 0x68, 0x12, 0x22, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x47, 0x65, + 0x74, 0x4c, 0x61, 0x73, 0x74, 0x4d, 0x69, 0x6e, 0x69, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x61, + 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x61, 0x73, 0x74, 0x4d, 0x69, 0x6e, 0x69, 0x62, 0x6c, 0x6f, - 0x63, 0x6b, 0x48, 0x61, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, - 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x61, 0x73, 0x74, 0x4d, 0x69, 0x6e, - 0x69, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x61, 0x73, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x08, 0x41, 0x64, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x16, - 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x41, 0x64, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x41, - 0x64, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x46, 0x0a, 0x0b, 0x53, 0x79, 0x6e, 0x63, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x19, - 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x53, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x72, 0x69, 0x76, 0x65, - 0x72, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x50, 0x0a, 0x0f, 0x41, 0x64, 0x64, 0x53, 0x74, - 0x72, 0x65, 0x61, 0x6d, 0x54, 0x6f, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x1d, 0x2e, 0x72, 0x69, 0x76, - 0x65, 0x72, 0x2e, 0x41, 0x64, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, 0x6f, 0x53, 0x79, - 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x72, 0x69, 0x76, 0x65, - 0x72, 0x2e, 0x41, 0x64, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, 0x6f, 0x53, 0x79, 0x6e, - 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x0a, 0x43, 0x61, 0x6e, - 0x63, 0x65, 0x6c, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x18, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, - 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x19, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, - 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5f, 0x0a, 0x14, - 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x46, 0x72, 0x6f, 0x6d, - 0x53, 0x79, 0x6e, 0x63, 0x12, 0x22, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6d, + 0x63, 0x6b, 0x48, 0x61, 0x73, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, + 0x0a, 0x08, 0x41, 0x64, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x16, 0x2e, 0x72, 0x69, 0x76, + 0x65, 0x72, 0x2e, 0x41, 0x64, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x41, 0x64, 0x64, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x0b, 0x53, + 0x79, 0x6e, 0x63, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x19, 0x2e, 0x72, 0x69, 0x76, + 0x65, 0x72, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x53, 0x79, + 0x6e, 0x63, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x30, 0x01, 0x12, 0x50, 0x0a, 0x0f, 0x41, 0x64, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, + 0x54, 0x6f, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x1d, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x41, + 0x64, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, 0x6f, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x41, 0x64, + 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, 0x6f, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x0a, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x53, + 0x79, 0x6e, 0x63, 0x12, 0x18, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x43, 0x61, 0x6e, 0x63, + 0x65, 0x6c, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, + 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x53, 0x79, 0x6e, 0x63, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5f, 0x0a, 0x14, 0x52, 0x65, 0x6d, 0x6f, + 0x76, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x46, 0x72, 0x6f, 0x6d, 0x53, 0x79, 0x6e, 0x63, + 0x12, 0x22, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x46, 0x72, 0x6f, 0x6d, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x46, 0x72, 0x6f, 0x6d, 0x53, 0x79, 0x6e, - 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, - 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x46, 0x72, 0x6f, - 0x6d, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, - 0x04, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x49, 0x6e, - 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x72, 0x69, 0x76, 0x65, - 0x72, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, - 0x0a, 0x08, 0x50, 0x69, 0x6e, 0x67, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x16, 0x2e, 0x72, 0x69, 0x76, - 0x65, 0x72, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x53, - 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x31, 0x5a, 0x2f, 0x67, - 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2d, - 0x62, 0x75, 0x69, 0x6c, 0x64, 0x2f, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2f, 0x63, 0x6f, 0x72, 0x65, - 0x2f, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x04, 0x49, 0x6e, 0x66, + 0x6f, 0x12, 0x12, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x49, 0x6e, + 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x08, 0x50, 0x69, + 0x6e, 0x67, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x16, 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x50, + 0x69, 0x6e, 0x67, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, + 0x2e, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x53, 0x79, 0x6e, 0x63, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x31, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2d, 0x62, 0x75, 0x69, 0x6c, + 0x64, 0x2f, 0x72, 0x69, 0x76, 0x65, 0x72, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x6e, 0x6f, 0x64, + 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( diff --git a/core/node/rpc/server.go b/core/node/rpc/server.go index 3d3efd0360..d82d1bd807 100644 --- a/core/node/rpc/server.go +++ b/core/node/rpc/server.go @@ -16,11 +16,6 @@ import ( "connectrpc.com/connect" "github.com/ethereum/go-ethereum/common" - "github.com/rs/cors" - "golang.org/x/net/http2" - "golang.org/x/net/http2/h2c" - httptrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http" - "github.com/river-build/river/core/config" "github.com/river-build/river/core/node/auth" . "github.com/river-build/river/core/node/base" @@ -32,8 +27,13 @@ import ( . "github.com/river-build/river/core/node/protocol" "github.com/river-build/river/core/node/protocol/protocolconnect" "github.com/river-build/river/core/node/registries" + "github.com/river-build/river/core/node/rpc/sync" "github.com/river-build/river/core/node/storage" "github.com/river-build/river/core/xchain/entitlement" + "github.com/rs/cors" + "golang.org/x/net/http2" + "golang.org/x/net/http2/h2c" + httptrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http" ) const ( @@ -502,11 +502,10 @@ func (s *Service) initCacheAndSync() error { s.mbProducer = events.NewMiniblockProducer(s.serverCtx, s.cache, nil) - s.syncHandler = NewSyncHandler( - s.wallet, + s.syncHandler = sync.NewHandler( + s.wallet.Address, s.cache, s.nodeRegistry, - s.streamRegistry, ) return nil diff --git a/core/node/rpc/service.go b/core/node/rpc/service.go index e1054ceec6..baa378f344 100644 --- a/core/node/rpc/service.go +++ b/core/node/rpc/service.go @@ -2,6 +2,7 @@ package rpc import ( "context" + river_sync "github.com/river-build/river/core/node/rpc/sync" "log/slog" "net" "net/http" @@ -44,7 +45,7 @@ type Service struct { // Streams cache events.StreamCache mbProducer events.MiniblockProducer - syncHandler SyncHandler + syncHandler river_sync.Handler // River chain riverChain *crypto.Blockchain diff --git a/core/node/rpc/service_sync_streams.go b/core/node/rpc/service_sync_streams.go new file mode 100644 index 0000000000..dc4e039811 --- /dev/null +++ b/core/node/rpc/service_sync_streams.go @@ -0,0 +1,55 @@ +package rpc + +import ( + "connectrpc.com/connect" + "context" + . "github.com/river-build/river/core/node/protocol" +) + +// TODO: wire metrics. +// var ( +// syncStreamsRequests = infra.NewSuccessMetrics("sync_streams_requests", serviceRequests) +// syncStreamsResultSize = infra.NewCounter("sync_streams_result_size", "The total number of events returned by sync streams") +// ) + +// func addUpdatesToCounter(updates []*StreamAndCookie) { +// for _, stream := range updates { +// syncStreamsResultSize.Add(float64(len(stream.Events))) +// } +// } + +func (s *Service) SyncStreams( + ctx context.Context, + req *connect.Request[SyncStreamsRequest], + res *connect.ServerStream[SyncStreamsResponse], +) error { + return s.syncHandler.SyncStreams(ctx, req, res) +} + +func (s *Service) AddStreamToSync( + ctx context.Context, + req *connect.Request[AddStreamToSyncRequest], +) (*connect.Response[AddStreamToSyncResponse], error) { + return s.syncHandler.AddStreamToSync(ctx, req) +} + +func (s *Service) RemoveStreamFromSync( + ctx context.Context, + req *connect.Request[RemoveStreamFromSyncRequest], +) (*connect.Response[RemoveStreamFromSyncResponse], error) { + return s.syncHandler.RemoveStreamFromSync(ctx, req) +} + +func (s *Service) CancelSync( + ctx context.Context, + req *connect.Request[CancelSyncRequest], +) (*connect.Response[CancelSyncResponse], error) { + return s.syncHandler.CancelSync(ctx, req) +} + +func (s *Service) PingSync( + ctx context.Context, + req *connect.Request[PingSyncRequest], +) (*connect.Response[PingSyncResponse], error) { + return s.syncHandler.PingSync(ctx, req) +} diff --git a/core/node/rpc/service_test.go b/core/node/rpc/service_test.go index aeaedc46b5..30f3bedda1 100644 --- a/core/node/rpc/service_test.go +++ b/core/node/rpc/service_test.go @@ -1,17 +1,24 @@ package rpc import ( + "bytes" "context" "crypto/tls" "fmt" + "math/rand" "net" "net/http" "os" "strconv" + "sync" + "sync/atomic" "testing" + "time" - "golang.org/x/net/http2" - + "connectrpc.com/connect" + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/common" + eth_crypto "github.com/ethereum/go-ethereum/crypto" "github.com/river-build/river/core/node/crypto" "github.com/river-build/river/core/node/dlog" "github.com/river-build/river/core/node/events" @@ -20,12 +27,8 @@ import ( "github.com/river-build/river/core/node/protocol/protocolconnect" . "github.com/river-build/river/core/node/shared" "github.com/river-build/river/core/node/testutils" - - "connectrpc.com/connect" - "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/common" - eth_crypto "github.com/ethereum/go-ethereum/crypto" "github.com/stretchr/testify/require" + "golang.org/x/net/http2" "google.golang.org/protobuf/proto" ) @@ -749,7 +752,7 @@ func testAddStreamsToSync(tester *serviceTester) { }, ), ) - require.Nilf(err, "error calling AddStreamsToSync: %v", err) + require.NoError(err, "error calling AddStreamsToSync") // wait for the sync syncRes.Receive() msg := syncRes.Msg() @@ -761,6 +764,7 @@ func testAddStreamsToSync(tester *serviceTester) { */ require.NotEmpty(syncId, "expected non-empty sync id") require.NotNil(msg.Stream, "expected 1 stream") + require.Equal(len(msg.Stream.Events), 1, "expected 1 event") require.Equal(syncId, msg.SyncId, "expected sync id to match") } @@ -839,7 +843,7 @@ func testRemoveStreamsFromSync(tester *serviceTester) { }, ), ) - require.Nilf(err, "error calling AddStreamsToSync: %v", err) + require.NoError(err, "AddStreamsToSync") log.Info("AddStreamToSync", "resp", resp) // When AddEvent is called, node calls streamImpl.notifyToSubscribers() twice // for different events. See hnt-3683 for explanation. First event is for @@ -898,13 +902,20 @@ OuterLoop: ) require.Nilf(err, "error calling AddEvent: %v", err) - /** - For debugging only. Uncomment to see syncRes.Receive() block. - bobClient's syncRes no longer receives the latest events from alice. + gotUnexpectedMsg := make(chan *protocol.SyncStreamsResponse) + go func() { + if syncRes.Receive() { + gotUnexpectedMsg <- syncRes.Msg() + } + }() + + select { + case <-time.After(3 * time.Second): + break + case <-gotUnexpectedMsg: + require.Fail("received message after stream was removed from sync") + } - // wait to see if we got a message. We shouldn't. - // uncomment: syncRes.Receive() - */ syncCancel() /** @@ -1042,3 +1053,469 @@ func TestForwardingWithRetries(t *testing.T) { }) } } + +// TestNodeDownStreamsSync ensures that after a node goes down the sync for affected streams are stopped and the client +// receives a down msg +func TestNodeDownStreamsSync(t *testing.T) { + var ( + req = require.New(t) + services = newServiceTester(t, serviceTesterOpts{numNodes: 5, start: true}) + client0 = services.testClient(0) + client1 = services.testClient(1) + node2 = services.nodes[2] // node that will be stopped + ctx = services.ctx + mu sync.Mutex + wallets []*crypto.Wallet + usersMu sync.Mutex + users []*protocol.SyncCookie + channelsMu sync.Mutex + channels []*protocol.SyncCookie + ) + + for i, node := range services.nodes { + t.Logf("node[%d] = %s\n", i, node.address) + } + + // create users that will join and add messages to channels. + for range 3 { + // Create user streams + wallet, err := crypto.NewWallet(ctx) + req.NoError(err, "new wallet") + syncCookie, _, err := createUser(ctx, wallet, client0, nil) + req.NoError(err, "create user") + + _, _, err = createUserDeviceKeyStream(ctx, wallet, client0, nil) + req.NoError(err) + + wallets = append(wallets, wallet) + users = append(users, syncCookie) + } + + // create a space and several channels in it + spaceID := testutils.FakeStreamId(STREAM_SPACE_BIN) + resspace, _, err := createSpace(ctx, wallets[0], client0, spaceID, nil) + req.NoError(err) + req.NotNil(resspace, "create space sync cookie") + + // create enough channels that they will be distributed among nodes and at least 1 end up on node5 + for range TestStreams { + channelId := testutils.FakeStreamId(STREAM_CHANNEL_BIN) + channel, _, err := createChannel(ctx, wallets[0], client0, spaceID, channelId, nil) + req.NoError(err) + req.NotNil(channel, "nil create channel sync cookie") + + channelsMu.Lock() + channels = append(channels, channel) + channelsMu.Unlock() + } + + // subscribe to channel updates + syncPos := append(users, channels...) + syncRes, err := client1.SyncStreams(ctx, connect.NewRequest(&protocol.SyncStreamsRequest{SyncPos: syncPos})) + req.NoError(err, "sync streams") + + syncRes.Receive() + syncID := syncRes.Msg().SyncId + t.Logf("subscription %s created on node: %s", syncID, services.nodes[1].address) + + // collect sync cookie updates for channels + var ( + messages = make(chan string, 512) + down = make(chan StreamId, 512) + node2StreamsCount atomic.Int64 + node2Streams sync.Map + nextCanBeDownMsg sync.Map + ) + + go func() { + for syncRes.Receive() { + msg := syncRes.Msg() + + switch msg.GetSyncOp() { + case protocol.SyncOp_SYNC_NEW: + syncID := msg.GetSyncId() + t.Logf("start stream sync %s ", syncID) + case protocol.SyncOp_SYNC_UPDATE: + req.NotNil(msg.GetStream(), "stream") + req.NotNil(msg.GetStream().GetNextSyncCookie(), "next sync cookie") + // keep track of stream id's that are managed by node 2 to ensure that for each stream a down msg is + // received after node 2 was stopped + cookie := msg.GetStream().GetNextSyncCookie() + if common.BytesToAddress(cookie.NodeAddress) == node2.address { + if streamID, err := StreamIdFromBytes(cookie.GetStreamId()); err == nil { + nextCanBeDownMsg.Store(streamID, true) + if _, loaded := node2Streams.LoadOrStore(streamID, cookie); !loaded { + node2StreamsCount.Add(1) + } + } + } + + usersMu.Lock() + for i, u := range users { + if bytes.Equal(cookie.GetStreamId(), u.GetStreamId()) { + users[i] = cookie + } + } + usersMu.Unlock() + + channelsMu.Lock() + for i, cc := range channels { + if bytes.Equal(cookie.GetStreamId(), cc.GetStreamId()) { + channels[i] = cookie + } + } + channelsMu.Unlock() + + for _, e := range msg.GetStream().GetEvents() { + var payload protocol.StreamEvent + err = proto.Unmarshal(e.Event, &payload) + req.NoError(err) + switch p := payload.Payload.(type) { + case *protocol.StreamEvent_ChannelPayload: + switch p.ChannelPayload.Content.(type) { + case *protocol.ChannelPayload_Message: + messages <- p.ChannelPayload.GetMessage().GetCiphertext() + } + } + } + + case protocol.SyncOp_SYNC_DOWN: + streamID, err := StreamIdFromBytes(msg.GetStreamId()) + req.NoError(err, "stream id in sync op down msg") + + prev, loaded := nextCanBeDownMsg.Swap(streamID, false) + if loaded { + req.True(prev.(bool), "can't receive 2 down message after each other for a stream") + } + down <- streamID + + case protocol.SyncOp_SYNC_CLOSE, protocol.SyncOp_SYNC_UNSPECIFIED, protocol.SyncOp_SYNC_PONG: + continue + + default: + t.Errorf("unexpected sync operation %s", msg.GetSyncOp()) + return + } + } + }() + + // users join channels + channelsMu.Lock() + channelsCount := len(channels) + channelsMu.Unlock() + for i, wallet := range wallets[1:] { + for c := range channelsCount { + mu.Lock() + channelsMu.Lock() + channel := channels[c] + channelsMu.Unlock() + mu.Unlock() + + usersMu.Lock() + miniBlockHashResp, err := client1.GetLastMiniblockHash( + ctx, + connect.NewRequest(&protocol.GetLastMiniblockHashRequest{StreamId: users[i+1].StreamId})) + usersMu.Unlock() + + req.NoError(err, "get last miniblock hash") + + channelId, _ := StreamIdFromBytes(channel.GetStreamId()) + usersMu.Lock() + userJoin, err := events.MakeEnvelopeWithPayload( + wallet, + events.Make_UserPayload_Membership(protocol.MembershipOp_SO_JOIN, channelId, nil, spaceID[:]), + miniBlockHashResp.Msg.GetHash(), //users[i+1].GetPrevMiniblockHash(), + ) + usersMu.Unlock() + req.NoError(err) + + usersMu.Lock() + resp, err := client1.AddEvent( + ctx, + connect.NewRequest( + &protocol.AddEventRequest{ + StreamId: users[i+1].StreamId, + Event: userJoin, + }, + ), + ) + usersMu.Unlock() + + req.NoError(err) + req.Nil(resp.Msg.GetError()) + } + } + + // send a bunch of messages and ensure that all of them are received + sendMessagesAndReceive(100, wallets, &channelsMu, channels, req, client0, ctx, messages, func(StreamId) bool { return false }) + + t.Logf("first messages batch received") + + if node2StreamsCount.Load() == 0 { + t.Skip("no streams managed by node that is about to be stopped") + } + + // stop node 2 and ensure that for each stream managed by node 2 a down msg is received + t.Logf("stop node %s", node2.address) + services.stopSingle(2) + + for range node2StreamsCount.Load() { + streamID := <-down + t.Logf("stream %s down", streamID) + _, loaded := node2Streams.Load(streamID) + req.Truef(loaded, "stream %s unexpected reported as down", streamID) + } + + t.Logf("received for all expected %d streams a down message", node2StreamsCount.Load()) + + // wait a bit and ensure that no down messages arrive for other streams + <-time.After(3 * time.Second) + req.Equal(0, len(down), "unexpected stream down messages") + + // send again a bunch of messages to streams managed by nodes that are still up and ensure that all are received. + sendMessagesAndReceive(100, wallets, &channelsMu, channels, req, client0, ctx, messages, func(streamID StreamId) bool { + _, found := node2Streams.Load(streamID) + return found + }) + t.Logf("received second batch of messages") + + // wait a bit and ensure that no down messages arrive for other streams + <-time.After(3 * time.Second) + req.Equal(0, len(down), "unexpected stream down messages") + + t.Logf("received no unexpected down messages") +} + +func sendMessagesAndReceive( + N int, + wallets []*crypto.Wallet, + muChannels *sync.Mutex, + channels []*protocol.SyncCookie, + require *require.Assertions, + client protocolconnect.StreamServiceClient, + ctx context.Context, + messages chan string, + skip func(streamID StreamId) bool, +) { + sendMsgCount := 0 + // send a bunch of messages to random channels + for range N { + wallet := wallets[rand.Int()%len(wallets)] + muChannels.Lock() + channel := channels[rand.Int()%len(channels)] + muChannels.Unlock() + streamID, _ := StreamIdFromBytes(channel.GetStreamId()) + if skip(streamID) { + continue + } + + message, err := events.MakeEnvelopeWithPayload( + wallet, + events.Make_ChannelPayload_Message(fmt.Sprintf("msg #%d", sendMsgCount)), + channel.GetPrevMiniblockHash(), + ) + require.NoError(err) + + _, err = client.AddEvent( + ctx, + connect.NewRequest( + &protocol.AddEventRequest{ + StreamId: channel.GetStreamId(), + Event: message, + }, + ), + ) + + require.NoError(err) + + sendMsgCount++ + } + + // make sure we received all messages + received := make(map[string]struct{}) + for range sendMsgCount { + received[<-messages] = struct{}{} + } + for i := range sendMsgCount { + require.Contains(received, fmt.Sprintf("msg #%d", i)) + } +} + +func TestCancelSync(t *testing.T) { + var ( + req = require.New(t) + services = newServiceTester(t, serviceTesterOpts{numNodes: 3, start: true}) + client0 = services.testClient(0) + client1 = services.testClient(1) + ctx = services.ctx + mu sync.Mutex + wallets []*crypto.Wallet + users []*protocol.SyncCookie + channelsMu sync.Mutex + channels []*protocol.SyncCookie + ) + + // create users that will join and add messages to channels. + for range 5 { + // Create user streams + wallet, err := crypto.NewWallet(ctx) + req.NoError(err, "new wallet") + syncCookie, _, err := createUser(ctx, wallet, client0, nil) + req.NoError(err, "create user") + + _, _, err = createUserDeviceKeyStream(ctx, wallet, client0, nil) + req.NoError(err) + + wallets = append(wallets, wallet) + users = append(users, syncCookie) + } + + // create a space and several channels in it + spaceID := testutils.FakeStreamId(STREAM_SPACE_BIN) + resspace, _, err := createSpace(ctx, wallets[0], client0, spaceID, nil) + req.NoError(err) + req.NotNil(resspace, "create space sync cookie") + + for range 10 { + channelId := testutils.FakeStreamId(STREAM_CHANNEL_BIN) + channel, _, err := createChannel(ctx, wallets[0], client0, spaceID, channelId, nil) + req.NoError(err) + req.NotNil(channel, "nil create channel sync cookie") + + channelsMu.Lock() + channels = append(channels, channel) + channelsMu.Unlock() + } + + // subscribe to channel updates + syncRes, err := client1.SyncStreams(context.Background(), connect.NewRequest(&protocol.SyncStreamsRequest{SyncPos: channels})) + req.NoError(err, "sync streams") + + syncRes.Receive() + syncID := syncRes.Msg().SyncId + t.Logf("subscription %s created on node: %s", syncID, services.nodes[1].address) + + // collect sync cookie updates for channels + var ( + messages = make(chan string, 512) + down = make(chan StreamId, 512) + nextCanBeDownMsg sync.Map + gotSyncClosedOp = make(chan *protocol.SyncStreamsResponse) + syncStopped = make(chan struct{}) + ) + + go func() { + for syncRes.Receive() { + msg := syncRes.Msg() + + switch msg.GetSyncOp() { + case protocol.SyncOp_SYNC_NEW: + syncID := msg.GetSyncId() + t.Logf("start stream sync %s ", syncID) + case protocol.SyncOp_SYNC_UPDATE: + for _, e := range msg.GetStream().GetEvents() { + var payload protocol.StreamEvent + err = proto.Unmarshal(e.Event, &payload) + req.NoError(err) + switch p := payload.Payload.(type) { + case *protocol.StreamEvent_ChannelPayload: + switch p.ChannelPayload.Content.(type) { + case *protocol.ChannelPayload_Message: + messages <- p.ChannelPayload.GetMessage().GetCiphertext() + } + } + } + + cookie := msg.GetStream().GetNextSyncCookie() + channelsMu.Lock() + for i, cc := range channels { + if bytes.Equal(cookie.GetStreamId(), cc.GetStreamId()) { + channels[i] = cookie + } + } + channelsMu.Unlock() + + case protocol.SyncOp_SYNC_DOWN: + streamID, _ := StreamIdFromBytes(msg.GetStreamId()) + prev, loaded := nextCanBeDownMsg.Swap(streamID, false) + if loaded { + req.True(prev.(bool), "can't receive 2 down message after each other for a stream") + } + down <- streamID + + case protocol.SyncOp_SYNC_CLOSE: + gotSyncClosedOp <- msg + + case protocol.SyncOp_SYNC_UNSPECIFIED, protocol.SyncOp_SYNC_PONG: + t.Errorf("unexpected sync operation %s", msg.GetSyncOp()) + } + } + + close(syncStopped) + }() + + // users join channels + channelsMu.Lock() + channelsCount := len(channels) + channelsMu.Unlock() + for i, wallet := range wallets[1:] { + for c := range channelsCount { + mu.Lock() + channelsMu.Lock() + channel := channels[c] + channelsMu.Unlock() + mu.Unlock() + + channelId, _ := StreamIdFromBytes(channel.GetStreamId()) + userJoin, err := events.MakeEnvelopeWithPayload( + wallet, + events.Make_UserPayload_Membership(protocol.MembershipOp_SO_JOIN, channelId, nil, spaceID[:]), + users[i+1].GetPrevMiniblockHash(), + ) + req.NoError(err) + + resp, err := client1.AddEvent( + ctx, + connect.NewRequest( + &protocol.AddEventRequest{ + StreamId: users[i+1].StreamId, + Event: userJoin, + }, + ), + ) + req.NoError(err) + req.Nil(resp.Msg.GetError()) + } + } + + sendMessagesAndReceive(10, wallets, &channelsMu, channels, req, client0, ctx, messages, func(StreamId) bool { + return false + }) + + r := connect.NewRequest(&protocol.CancelSyncRequest{SyncId: syncID}) + _, err = client1.CancelSync(ctx, r) + req.NoError(err, "cancel sync") + + // ensure that there are no stream down messages + select { + case <-down: + t.Fatal("received unexpected stream down msg") + case <-time.After(5 * time.Second): + break + } + + // ensure that we received the sync closed msg and that the sync stream is closed + select { + case closedOpMsg := <-gotSyncClosedOp: + req.Equal(syncID, closedOpMsg.GetSyncId(), "unexpected sync id") + case <-time.After(10 * time.Second): + t.Fatal("didn't receive sync close op msg within reasonable time") + } + + select { + case <-syncStopped: + break + case <-time.After(10 * time.Second): + t.Fatal("sync stream wasn't closed within reasonable time") + } +} diff --git a/core/node/rpc/sync/client/local.go b/core/node/rpc/sync/client/local.go new file mode 100644 index 0000000000..f6a27aeee1 --- /dev/null +++ b/core/node/rpc/sync/client/local.go @@ -0,0 +1,136 @@ +package client + +import ( + "context" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/river-build/river/core/node/events" + . "github.com/river-build/river/core/node/protocol" + "github.com/river-build/river/core/node/shared" +) + +type localSyncer struct { + syncStreamCtx context.Context + syncStreamCancel context.CancelFunc + + streamCache events.StreamCache + cookies []*SyncCookie + messages chan<- *SyncStreamsResponse + localAddr common.Address + + activeStreamsMu sync.Mutex + activeStreams map[shared.StreamId]events.SyncStream +} + +func newLocalSyncer( + ctx context.Context, + localAddr common.Address, + streamCache events.StreamCache, + cookies []*SyncCookie, + messages chan<- *SyncStreamsResponse, +) (*localSyncer, error) { + syncStreamCtx, syncStreamCancel := context.WithCancel(ctx) + return &localSyncer{ + syncStreamCtx: syncStreamCtx, + syncStreamCancel: syncStreamCancel, + streamCache: streamCache, + localAddr: localAddr, + cookies: cookies, + messages: messages, + activeStreams: make(map[shared.StreamId]events.SyncStream), + }, nil +} + +func (s *localSyncer) Run() { + for _, cookie := range s.cookies { + streamID, _ := shared.StreamIdFromBytes(cookie.GetStreamId()) + _ = s.addStream(s.syncStreamCtx, streamID, cookie) + } + + <-s.syncStreamCtx.Done() + + s.activeStreamsMu.Lock() + defer s.activeStreamsMu.Unlock() + + for streamID, syncStream := range s.activeStreams { + syncStream.Unsub(s) + delete(s.activeStreams, streamID) + } +} + +func (s *localSyncer) Address() common.Address { + return s.localAddr +} + +func (s *localSyncer) AddStream(ctx context.Context, cookie *SyncCookie) error { + streamID, err := shared.StreamIdFromBytes(cookie.GetStreamId()) + if err != nil { + return err + } + return s.addStream(ctx, streamID, cookie) +} + +func (s *localSyncer) RemoveStream(_ context.Context, streamID shared.StreamId) (bool, error) { + s.activeStreamsMu.Lock() + defer s.activeStreamsMu.Unlock() + + syncStream, found := s.activeStreams[streamID] + if found { + syncStream.Unsub(s) + delete(s.activeStreams, streamID) + } + + return len(s.activeStreams) == 0, nil +} + +// OnUpdate is called each time a new cookie is available for a stream +func (s *localSyncer) OnUpdate(r *StreamAndCookie) { + s.messages <- &SyncStreamsResponse{ + SyncOp: SyncOp_SYNC_UPDATE, + Stream: r, + } +} + +// OnSyncError is called when a sync subscription failed unrecoverable +func (s *localSyncer) OnSyncError(error) { + s.activeStreamsMu.Lock() + defer s.activeStreamsMu.Unlock() + + for streamID, syncStream := range s.activeStreams { + syncStream.Unsub(s) + delete(s.activeStreams, streamID) + s.OnStreamSyncDown(streamID) + } +} + +// OnStreamSyncDown is called when updates for a stream could not be given. +func (s *localSyncer) OnStreamSyncDown(streamID shared.StreamId) { + s.messages <- &SyncStreamsResponse{ + SyncOp: SyncOp_SYNC_DOWN, + StreamId: streamID[:], + } +} + +func (s *localSyncer) addStream(ctx context.Context, streamID shared.StreamId, cookie *SyncCookie) error { + s.activeStreamsMu.Lock() + defer s.activeStreamsMu.Unlock() + + // prevent subscribing multiple times on the same stream + if _, found := s.activeStreams[streamID]; found { + return nil + } + + syncStream, err := s.streamCache.GetSyncStream(ctx, streamID) + if err != nil { + return err + } + + if err := syncStream.Sub(ctx, cookie, s); err != nil { + return err + } + + s.activeStreams[streamID] = syncStream + + return nil +} diff --git a/core/node/rpc/sync/client/remote.go b/core/node/rpc/sync/client/remote.go new file mode 100644 index 0000000000..206f329ddf --- /dev/null +++ b/core/node/rpc/sync/client/remote.go @@ -0,0 +1,230 @@ +package client + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "time" + + "connectrpc.com/connect" + "github.com/ethereum/go-ethereum/common" + "github.com/river-build/river/core/node/base" + "github.com/river-build/river/core/node/dlog" + . "github.com/river-build/river/core/node/protocol" + "github.com/river-build/river/core/node/protocol/protocolconnect" + "github.com/river-build/river/core/node/shared" +) + +type remoteSyncer struct { + syncStreamCtx context.Context + syncStreamCancel context.CancelFunc + syncID atomic.Value + forwarderSyncID string + remoteAddr common.Address + client protocolconnect.StreamServiceClient + cookies []*SyncCookie + messages chan<- *SyncStreamsResponse + streams sync.Map + responseStream *connect.ServerStreamForClient[SyncStreamsResponse] +} + +func newRemoteSyncer( + ctx context.Context, + forwarderSyncID string, + remoteAddr common.Address, + client protocolconnect.StreamServiceClient, + cookies []*SyncCookie, + messages chan<- *SyncStreamsResponse, +) (*remoteSyncer, error) { + syncStreamCtx, syncStreamCancel := context.WithCancel(ctx) + responseStream, err := client.SyncStreams(syncStreamCtx, connect.NewRequest(&SyncStreamsRequest{SyncPos: cookies})) + if err != nil { + for _, cookie := range cookies { + messages <- &SyncStreamsResponse{ + SyncOp: SyncOp_SYNC_DOWN, + StreamId: cookie.GetStreamId(), + } + } + syncStreamCancel() + return nil, err + } + + if !responseStream.Receive() { + syncStreamCancel() + return nil, responseStream.Err() + } + + log := dlog.FromCtx(ctx) + + if responseStream.Msg().SyncOp != SyncOp_SYNC_NEW || responseStream.Msg().SyncId == "" { + log.Error("Received unexpected sync stream message", + "syncOp", responseStream.Msg().SyncOp, + "syncId", responseStream.Msg().SyncId) + syncStreamCancel() + return nil, err + } + + s := &remoteSyncer{ + forwarderSyncID: forwarderSyncID, + syncStreamCtx: syncStreamCtx, + syncStreamCancel: syncStreamCancel, + client: client, + cookies: cookies, + messages: messages, + responseStream: responseStream, + remoteAddr: remoteAddr, + } + + s.syncID.Store(responseStream.Msg().SyncId) + + for _, cookie := range s.cookies { + streamID, _ := shared.StreamIdFromBytes(cookie.GetStreamId()) + s.streams.Store(streamID, struct{}{}) + } + + return s, nil +} + +func (s *remoteSyncer) Run() { + defer s.responseStream.Close() + + var latestMsgReceived atomic.Value + + latestMsgReceived.Store(time.Now()) + + go func() { + var ( + // check every pingTicker if it's time to send a ping req to remote + pingTicker = time.NewTicker(3 * time.Second) + // don't send a ping req if there was activity within recentActivityInterval + recentActivityInterval = 15 * time.Second + // if no message was receiving within recentActivityDeadline assume stream is dead + recentActivityDeadline = 30 * time.Second + ) + defer pingTicker.Stop() + + for { + select { + case <-pingTicker.C: + now := time.Now() + lastMsgRecv := latestMsgReceived.Load().(time.Time) + if lastMsgRecv.Add(recentActivityDeadline).Before(now) { // no recent activity -> conn dead + s.syncStreamCancel() + return + } + + if lastMsgRecv.Add(recentActivityInterval).After(now) { // seen recent activity + continue + } + + // send ping to remote to generate activity to check if remote is still alive + if syncID := s.syncID.Load().(string); syncID != "" { + if _, err := s.client.PingSync(s.syncStreamCtx, connect.NewRequest(&PingSyncRequest{ + SyncId: syncID, + Nonce: fmt.Sprintf("%d", now.Unix()), + })); err != nil { + s.syncStreamCancel() + return + } + return + } + case <-s.syncStreamCtx.Done(): + return + case <-s.syncStreamCtx.Done(): + return + } + } + }() + + for s.responseStream.Receive() { + if s.syncStreamCtx.Err() != nil { + break + } + + latestMsgReceived.Store(time.Now()) + + res := s.responseStream.Msg() + + if res.GetSyncOp() == SyncOp_SYNC_UPDATE { + s.messages <- res + } else if res.GetSyncOp() == SyncOp_SYNC_DOWN { + if streamID, err := shared.StreamIdFromBytes(res.GetStreamId()); err == nil { + s.messages <- res + s.streams.Delete(streamID) + } + } + } + + // stream interrupted while client didn't cancel sync -> remote is unavailable + if s.syncStreamCtx.Err() == nil { + log := dlog.FromCtx(s.syncStreamCtx) + log.Info("remote node disconnected", "remote", s.remoteAddr) + + s.streams.Range(func(key, value any) bool { + streamID := key.(shared.StreamId) + + log.Debug("stream down", "syncId", s.forwarderSyncID, "remote", s.remoteAddr, "stream", streamID) + s.messages <- &SyncStreamsResponse{ + SyncOp: SyncOp_SYNC_DOWN, + StreamId: streamID[:], + } + return true + }) + } +} + +func (s *remoteSyncer) Address() common.Address { + return s.remoteAddr +} + +func (s *remoteSyncer) AddStream(ctx context.Context, cookie *SyncCookie) error { + syncID := s.syncID.Load().(string) + if syncID == "" { + return base.RiverError(Err_UNAVAILABLE, "sync not started") + } + + streamID, err := shared.StreamIdFromBytes(cookie.GetStreamId()) + if err != nil { + return err + } + + _, err = s.client.AddStreamToSync(ctx, connect.NewRequest(&AddStreamToSyncRequest{ + SyncId: syncID, + SyncPos: cookie, + })) + + if err == nil { + s.streams.Store(streamID, struct{}{}) + } + + return err +} + +func (s *remoteSyncer) RemoveStream(ctx context.Context, streamID shared.StreamId) (bool, error) { + syncID := s.syncID.Load().(string) + if syncID == "" { + return false, base.RiverError(Err_UNAVAILABLE, "sync not started") + } + + _, err := s.client.RemoveStreamFromSync(ctx, connect.NewRequest(&RemoveStreamFromSyncRequest{ + SyncId: syncID, + StreamId: streamID[:], + })) + + if err == nil { + s.streams.Delete(streamID) + } + + noMoreStreams := true + s.streams.Range(func(key, value any) bool { + noMoreStreams = false + return false + }) + + if noMoreStreams { + s.syncStreamCancel() + } + + return noMoreStreams, err +} diff --git a/core/node/rpc/sync/client/syncer_set.go b/core/node/rpc/sync/client/syncer_set.go new file mode 100644 index 0000000000..9828d7a368 --- /dev/null +++ b/core/node/rpc/sync/client/syncer_set.go @@ -0,0 +1,234 @@ +package client + +import ( + "context" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/river-build/river/core/node/base" + "github.com/river-build/river/core/node/events" + "github.com/river-build/river/core/node/nodes" + . "github.com/river-build/river/core/node/protocol" + . "github.com/river-build/river/core/node/shared" +) + +type ( + StreamsSyncer interface { + Run() + Address() common.Address + AddStream(ctx context.Context, cookie *SyncCookie) error + RemoveStream(ctx context.Context, streamID StreamId) (bool, error) + } + + // SyncerSet is the set of StreamsSyncers that are used for a sync operation. + SyncerSet struct { + // ctx is the root context for all syncers in this set and used to cancel them + ctx context.Context + // syncID is the sync id as used between the client and this node + syncID string + // localNodeAddress is the node address for this stream node instance + localNodeAddress common.Address + // messages is the channel to which StreamsSyncers write updates that must be sent to the client + messages chan *SyncStreamsResponse + // streamCache is used to subscribe to streams managed by this node instance + streamCache events.StreamCache + // nodeRegistry keeps a mapping from node address to node meta-data + nodeRegistry nodes.NodeRegistry + // syncerTasks is a wait group for running background StreamsSyncers that is used to ensure all syncers stopped + syncerTasks sync.WaitGroup + // muSyncers guards syncers and streamID2Syncer + muSyncers sync.Mutex + // syncers is the existing set of syncers, indexed by the syncer node address + syncers map[common.Address]StreamsSyncer + // streamID2Syncer maps from a stream to its syncer + streamID2Syncer map[StreamId]StreamsSyncer + } + + // SyncCookieSet maps from a stream id to a sync cookie + SyncCookieSet map[StreamId]*SyncCookie + // StreamCookieSetGroupedByNodeAddress is a mapping from a node address to a SyncCookieSet + StreamCookieSetGroupedByNodeAddress map[common.Address]SyncCookieSet +) + +func (cs SyncCookieSet) AsSlice() []*SyncCookie { + cookies := make([]*SyncCookie, 0, len(cs)) + for _, cookie := range cs { + cookies = append(cookies, cookie) + } + return cookies +} + +// NewSyncers creates the required syncer set that subscribe on all given cookies. +// A syncer can either be local or remote and writes received events to an internal messages channel from which events +// are streamed to the client. +func NewSyncers( + ctx context.Context, + syncID string, + streamCache events.StreamCache, + nodeRegistry nodes.NodeRegistry, + localNodeAddress common.Address, + cookies StreamCookieSetGroupedByNodeAddress, +) (*SyncerSet, <-chan *SyncStreamsResponse, error) { + var ( + syncers = make(map[common.Address]StreamsSyncer) + streamID2Syncer = make(map[StreamId]StreamsSyncer) + messages = make(chan *SyncStreamsResponse, 128) + ) + + // instantiate background syncers for sync operation + for nodeAddress, cookieSet := range cookies { + if nodeAddress == localNodeAddress { // stream managing node it this node + syncer, err := newLocalSyncer(ctx, localNodeAddress, streamCache, cookieSet.AsSlice(), messages) + if err != nil { + return nil, nil, err + } + syncers[nodeAddress] = syncer + } else { + client, err := nodeRegistry.GetStreamServiceClientForAddress(nodeAddress) + if err != nil { + return nil, nil, err + } + + syncer, err := newRemoteSyncer(ctx, syncID, nodeAddress, client, cookieSet.AsSlice(), messages) + if err != nil { + return nil, nil, err + } + + syncers[nodeAddress] = syncer + } + + // associate syncer with streamId to remove stream from sync operation + syncer := syncers[nodeAddress] + for streamID := range cookieSet { + streamID2Syncer[streamID] = syncer + } + } + + return &SyncerSet{ + ctx: ctx, + syncID: syncID, + streamCache: streamCache, + nodeRegistry: nodeRegistry, + localNodeAddress: localNodeAddress, + syncers: syncers, + streamID2Syncer: streamID2Syncer, + messages: messages, + }, messages, nil +} + +func (ss *SyncerSet) Run() { + ss.muSyncers.Lock() + ss.syncerTasks.Add(len(ss.syncers)) + for syncerAddr, syncer := range ss.syncers { + go func() { + syncer.Run() + ss.muSyncers.Lock() + delete(ss.syncers, syncerAddr) + ss.muSyncers.Unlock() + ss.syncerTasks.Done() + }() + } + ss.muSyncers.Unlock() + + <-ss.ctx.Done() // wait till sync is cancelled + ss.syncerTasks.Wait() // wait till background syncers finished + close(ss.messages) +} + +func (ss *SyncerSet) AddStream(ctx context.Context, nodeAddress common.Address, streamID StreamId, cookie *SyncCookie) error { + ss.muSyncers.Lock() + defer ss.muSyncers.Unlock() + + if _, found := ss.streamID2Syncer[streamID]; found { + return nil // stream is already part of sync operation + } + + // check if there is already a syncer that can sync the given stream -> add stream to the syncer + if syncer, found := ss.syncers[nodeAddress]; found { + if err := syncer.AddStream(ctx, cookie); err != nil { + return err + } + ss.streamID2Syncer[streamID] = syncer + return nil + } + + // first stream to sync with remote -> create a new syncer instance + var ( + syncer StreamsSyncer + err error + ) + if nodeAddress == ss.localNodeAddress { + if syncer, err = newLocalSyncer(ss.ctx, ss.localNodeAddress, ss.streamCache, []*SyncCookie{cookie}, ss.messages); err != nil { + return err + } + } else { + client, err := ss.nodeRegistry.GetStreamServiceClientForAddress(nodeAddress) + if err != nil { + return err + } + if syncer, err = newRemoteSyncer(ss.ctx, ss.syncID, nodeAddress, client, []*SyncCookie{cookie}, ss.messages); err != nil { + return err + } + } + + ss.syncers[nodeAddress] = syncer + ss.streamID2Syncer[streamID] = syncer + + ss.syncerTasks.Add(1) + go func() { + ss.syncers[nodeAddress].Run() + ss.muSyncers.Lock() + delete(ss.syncers, nodeAddress) + ss.muSyncers.Unlock() + ss.syncerTasks.Done() + }() + + return nil +} + +func (ss *SyncerSet) RemoveStream(ctx context.Context, streamID StreamId) error { + ss.muSyncers.Lock() + defer ss.muSyncers.Unlock() + + // get the syncer that is responsible for the stream. + // (if not it indicates state corruption between ss.streamID2Syncer and ss.syncers) + syncer, found := ss.streamID2Syncer[streamID] + if !found { + return base.RiverError(Err_NOT_FOUND, "Stream not part of sync operation"). + Tags("syncId", ss.syncID, "streamId", streamID) + } + + syncerStopped, err := syncer.RemoveStream(ctx, streamID) + if err != nil { + return err + } + + delete(ss.streamID2Syncer, streamID) + if syncerStopped { + delete(ss.syncers, syncer.Address()) + } + + return nil +} + +// ValidateAndGroupSyncCookies validates the given syncCookies and groups them by node address/streamID. +func ValidateAndGroupSyncCookies(syncCookies []*SyncCookie) (StreamCookieSetGroupedByNodeAddress, error) { + cookies := make(StreamCookieSetGroupedByNodeAddress) + for _, cookie := range syncCookies { + if err := events.SyncCookieValidate(cookie); err != nil { + return nil, err + } + + streamID, err := StreamIdFromBytes(cookie.GetStreamId()) + if err != nil { + return nil, err + } + + nodeAddr := common.BytesToAddress(cookie.NodeAddress) + if cookies[nodeAddr] == nil { + cookies[nodeAddr] = make(map[StreamId]*SyncCookie) + } + cookies[nodeAddr][streamID] = cookie + } + return cookies, nil +} diff --git a/core/node/rpc/sync/handler.go b/core/node/rpc/sync/handler.go new file mode 100644 index 0000000000..f53f7663e2 --- /dev/null +++ b/core/node/rpc/sync/handler.go @@ -0,0 +1,139 @@ +package sync + +import ( + "context" + "sync" + + "connectrpc.com/connect" + "github.com/ethereum/go-ethereum/common" + . "github.com/river-build/river/core/node/base" + "github.com/river-build/river/core/node/events" + "github.com/river-build/river/core/node/nodes" + . "github.com/river-build/river/core/node/protocol" +) + +type ( + // Handler defines the external grpc interface that clients can call. + Handler interface { + SyncStreams( + ctx context.Context, + req *connect.Request[SyncStreamsRequest], + res *connect.ServerStream[SyncStreamsResponse], + ) error + + AddStreamToSync( + ctx context.Context, + req *connect.Request[AddStreamToSyncRequest], + ) (*connect.Response[AddStreamToSyncResponse], error) + + RemoveStreamFromSync( + ctx context.Context, + req *connect.Request[RemoveStreamFromSyncRequest], + ) (*connect.Response[RemoveStreamFromSyncResponse], error) + + CancelSync( + ctx context.Context, + req *connect.Request[CancelSyncRequest], + ) (*connect.Response[CancelSyncResponse], error) + + PingSync( + ctx context.Context, + req *connect.Request[PingSyncRequest], + ) (*connect.Response[PingSyncResponse], error) + } + + handlerImpl struct { + // nodeAddr is used to determine if a stream is local or remote + nodeAddr common.Address + // streamCache is used to subscribe on local streams + streamCache events.StreamCache + // nodeRegistry is used to find a node endpoint to subscribe on remote streams + nodeRegistry nodes.NodeRegistry + // subscriptions keeps a mapping from SyncID -> *StreamSyncOperation + subscriptions sync.Map + } +) + +// NewHandler returns a structure that implements the Handler interface. +// It keeps internally a map of in progress stream sync operations and forwards add stream, remove sream, cancel sync +// requests to the associated stream sync operation. +func NewHandler( + nodeAddr common.Address, + cache events.StreamCache, + nodeRegistry nodes.NodeRegistry, +) *handlerImpl { + return &handlerImpl{ + nodeAddr: nodeAddr, + streamCache: cache, + nodeRegistry: nodeRegistry, + } +} + +func (h *handlerImpl) SyncStreams( + ctx context.Context, + req *connect.Request[SyncStreamsRequest], + res *connect.ServerStream[SyncStreamsResponse], +) error { + ctx, log := ctxAndLogForRequest(ctx, req) + + sub, err := NewStreamsSyncOperation(ctx, h.nodeAddr, h.streamCache, h.nodeRegistry) + if err != nil { + log.Error("Unable to create streams sync subscription", "error", err) + return err + } + + h.subscriptions.Store(sub.SyncID, sub) + defer h.subscriptions.Delete(sub.SyncID) + + // send SyncID to client + if err := res.Send(&SyncStreamsResponse{ + SyncId: sub.SyncID, + SyncOp: SyncOp_SYNC_NEW, + }); err != nil { + err := AsRiverError(err).Func("SyncStreams") + return err + } + + // run until sub.ctx expires or until the client calls CancelSync + return sub.Run(req, res) +} + +func (h *handlerImpl) AddStreamToSync( + ctx context.Context, + req *connect.Request[AddStreamToSyncRequest], +) (*connect.Response[AddStreamToSyncResponse], error) { + if sub, ok := h.subscriptions.Load(req.Msg.GetSyncId()); ok { + return sub.(*StreamSyncOperation).AddStreamToSync(ctx, req) + } + return nil, RiverError(Err_NOT_FOUND, "unknown sync operation").Tag("syncId", req.Msg.GetSyncId()) +} + +func (h *handlerImpl) RemoveStreamFromSync( + ctx context.Context, + req *connect.Request[RemoveStreamFromSyncRequest], +) (*connect.Response[RemoveStreamFromSyncResponse], error) { + if sub, ok := h.subscriptions.Load(req.Msg.GetSyncId()); ok { + return sub.(*StreamSyncOperation).RemoveStreamFromSync(ctx, req) + } + return nil, RiverError(Err_NOT_FOUND, "unknown sync operation").Tag("syncId", req.Msg.GetSyncId()) +} + +func (h *handlerImpl) CancelSync( + ctx context.Context, + req *connect.Request[CancelSyncRequest], +) (*connect.Response[CancelSyncResponse], error) { + if sub, ok := h.subscriptions.Load(req.Msg.GetSyncId()); ok { + return sub.(*StreamSyncOperation).CancelSync(ctx, req) // causes StreamSyncOperation.run to return in SyncStream + } + return nil, RiverError(Err_NOT_FOUND, "unknown sync operation").Tag("syncId", req.Msg.GetSyncId()) +} + +func (h *handlerImpl) PingSync( + ctx context.Context, + req *connect.Request[PingSyncRequest], +) (*connect.Response[PingSyncResponse], error) { + if sub, ok := h.subscriptions.Load(req.Msg.GetSyncId()); ok { + return sub.(*StreamSyncOperation).PingSync(ctx, req) + } + return nil, RiverError(Err_NOT_FOUND, "unknown sync operation").Tag("syncId", req.Msg.GetSyncId()) +} diff --git a/core/node/rpc/sync/operation.go b/core/node/rpc/sync/operation.go new file mode 100644 index 0000000000..5a196061eb --- /dev/null +++ b/core/node/rpc/sync/operation.go @@ -0,0 +1,246 @@ +package sync + +import ( + "connectrpc.com/connect" + "context" + + "github.com/ethereum/go-ethereum/common" + . "github.com/river-build/river/core/node/base" + "github.com/river-build/river/core/node/events" + "github.com/river-build/river/core/node/nodes" + . "github.com/river-build/river/core/node/protocol" + "github.com/river-build/river/core/node/rpc/sync/client" + "github.com/river-build/river/core/node/shared" +) + +type ( + // StreamSyncOperation represents a stream sync operation that is currently in progress. + StreamSyncOperation struct { + // SyncID is the identifier as used with the external client to identify the streams sync operation. + SyncID string + // ctx is the root context for this subscription, when expires the subscription and all background syncers are + // cancelled + ctx context.Context + // cancel sync operation + cancel context.CancelFunc + // commands holds incoming requests from the client to add/remove/cancel commands + commands chan *subCommand + // thisNodeAddress keeps the address of this stream thisNodeAddress instance + thisNodeAddress common.Address + // streamCache gives access to streams managed by this thisNodeAddress + streamCache events.StreamCache + // nodeRegistry is used to get the remote remoteNode endpoint from a thisNodeAddress address + nodeRegistry nodes.NodeRegistry + } + + // subCommand represents a request to add or remove a stream and ping sync operation + subCommand struct { + Ctx context.Context + RmStreamReq *connect.Request[RemoveStreamFromSyncRequest] + AddStreamReq *connect.Request[AddStreamToSyncRequest] + PingReq *connect.Request[PingSyncRequest] + ReplyErr chan error + } +) + +// NewStreamsSyncOperation initialises a new sync stream operation. It groups the given syncCookies per stream node +// by its address and subscribes on the internal stream streamCache for local streams. +// +// Use the Run method to start syncing. +func NewStreamsSyncOperation( + ctx context.Context, + node common.Address, + streamCache events.StreamCache, + nodeRegistry nodes.NodeRegistry, +) (*StreamSyncOperation, error) { + // make the sync operation cancellable for CancelSync + ctx, cancel := context.WithCancel(ctx) + + return &StreamSyncOperation{ + ctx: ctx, + cancel: cancel, + SyncID: GenNanoid(), + thisNodeAddress: node, + commands: make(chan *subCommand, 10), + streamCache: streamCache, + nodeRegistry: nodeRegistry, + }, nil +} + +// Run the stream sync until either sub.Cancel is called or until sub.ctx expired +func (syncOp *StreamSyncOperation) Run( + req *connect.Request[SyncStreamsRequest], + res *connect.ServerStream[SyncStreamsResponse], +) error { + cookies, err := client.ValidateAndGroupSyncCookies(req.Msg.GetSyncPos()) + if err != nil { + return err + } + + syncers, messages, err := client.NewSyncers( + syncOp.ctx, syncOp.SyncID, syncOp.streamCache, syncOp.nodeRegistry, syncOp.thisNodeAddress, cookies) + if err != nil { + return err + } + + go syncers.Run() + + for { + select { + case msg, ok := <-messages: + if !ok { // messages is closed in syncers when syncOp.ctx is cancelled + _ = res.Send(&SyncStreamsResponse{ + SyncId: syncOp.SyncID, + SyncOp: SyncOp_SYNC_CLOSE, + }) + return nil + } + + // use the syncID as used between client and subscription node + msg.SyncId = syncOp.SyncID + if err := res.Send(msg); err != nil { + syncOp.cancel() + return err + } + + case cmd := <-syncOp.commands: + if cmd.AddStreamReq != nil { + nodeAddress := common.BytesToAddress(cmd.AddStreamReq.Msg.GetSyncPos().GetNodeAddress()) + streamID, err := shared.StreamIdFromBytes(cmd.AddStreamReq.Msg.GetSyncPos().GetStreamId()) + if err != nil { + cmd.ReplyErr <- err + close(cmd.ReplyErr) + continue + } + cmd.ReplyErr <- syncers.AddStream(cmd.Ctx, nodeAddress, streamID, cmd.AddStreamReq.Msg.GetSyncPos()) + close(cmd.ReplyErr) + } else if cmd.RmStreamReq != nil { + streamID, err := shared.StreamIdFromBytes(cmd.RmStreamReq.Msg.GetStreamId()) + if err != nil { + cmd.ReplyErr <- err + close(cmd.ReplyErr) + continue + } + cmd.ReplyErr <- syncers.RemoveStream(cmd.Ctx, streamID) + close(cmd.ReplyErr) + } else if cmd.PingReq != nil { + _ = res.Send(&SyncStreamsResponse{ + SyncOp: SyncOp_SYNC_PONG, + PongNonce: cmd.PingReq.Msg.GetNonce(), + }) + close(cmd.ReplyErr) + } + } + } +} + +func (syncOp *StreamSyncOperation) AddStreamToSync( + ctx context.Context, + req *connect.Request[AddStreamToSyncRequest], +) (*connect.Response[AddStreamToSyncResponse], error) { + if err := events.SyncCookieValidate(req.Msg.GetSyncPos()); err != nil { + return nil, err + } + + op := &subCommand{ + Ctx: ctx, + AddStreamReq: req, + ReplyErr: make(chan error, 1), + } + + if syncOp.sendMsg(op) { + select { + case err := <-op.ReplyErr: + if err == nil { + return &connect.Response[AddStreamToSyncResponse]{}, nil + } + return nil, err + case <-syncOp.ctx.Done(): + return nil, RiverError(Err_CANCELED, "sync operation cancelled").Tags("syncId", syncOp.SyncID) + } + } + + return nil, RiverError(Err_CANCELED, "sync operation cancelled").Tags("syncId", syncOp.SyncID) +} + +func (syncOp *StreamSyncOperation) RemoveStreamFromSync( + ctx context.Context, + req *connect.Request[RemoveStreamFromSyncRequest], +) (*connect.Response[RemoveStreamFromSyncResponse], error) { + if req.Msg.GetSyncId() != syncOp.SyncID { + return nil, RiverError(Err_INVALID_ARGUMENT, "invalid syncId").Tag("syncId", req.Msg.GetSyncId()) + } + + op := &subCommand{ + Ctx: ctx, + RmStreamReq: req, + ReplyErr: make(chan error, 1), + } + + if syncOp.sendMsg(op) { + select { + case err := <-op.ReplyErr: + if err == nil { + return &connect.Response[RemoveStreamFromSyncResponse]{}, nil + } + return nil, err + case <-syncOp.ctx.Done(): + return nil, RiverError(Err_CANCELED, "sync operation cancelled").Tags("syncId", syncOp.SyncID) + } + } + + return nil, RiverError(Err_CANCELED, "sync operation cancelled").Tags("syncId", syncOp.SyncID) +} + +func (syncOp *StreamSyncOperation) CancelSync( + _ context.Context, + req *connect.Request[CancelSyncRequest], +) (*connect.Response[CancelSyncResponse], error) { + if req.Msg.GetSyncId() != syncOp.SyncID { + return nil, RiverError(Err_INVALID_ARGUMENT, "invalid syncId").Tag("syncId", req.Msg.GetSyncId()) + } + + syncOp.cancel() + + return &connect.Response[CancelSyncResponse]{ + Msg: &CancelSyncResponse{}, + }, nil +} + +func (syncOp *StreamSyncOperation) PingSync( + ctx context.Context, + req *connect.Request[PingSyncRequest], +) (*connect.Response[PingSyncResponse], error) { + if req.Msg.GetSyncId() != syncOp.SyncID { + return nil, RiverError(Err_INVALID_ARGUMENT, "invalid syncId").Tag("syncId", req.Msg.GetSyncId()) + } + + op := &subCommand{ + Ctx: ctx, + PingReq: req, + ReplyErr: make(chan error, 1), + } + + if syncOp.sendMsg(op) { + select { + case err := <-op.ReplyErr: + if err == nil { + return &connect.Response[PingSyncResponse]{}, nil + } + return nil, err + case <-syncOp.ctx.Done(): + return nil, RiverError(Err_CANCELED, "sync operation cancelled").Tags("syncId", syncOp.SyncID) + } + } + + return nil, RiverError(Err_CANCELED, "sync operation cancelled").Tags("syncId", syncOp.SyncID) +} + +func (syncOp *StreamSyncOperation) sendMsg(cmd *subCommand) bool { + select { + case syncOp.commands <- cmd: + return true + case <-syncOp.ctx.Done(): + return false + } +} diff --git a/core/node/rpc/sync/util.go b/core/node/rpc/sync/util.go new file mode 100644 index 0000000000..9480342439 --- /dev/null +++ b/core/node/rpc/sync/util.go @@ -0,0 +1,28 @@ +package sync + +import ( + "context" + "log/slog" + + "connectrpc.com/connect" + "github.com/river-build/river/core/node/dlog" +) + +type RequestWithStreamId interface { + GetStreamId() string +} + +func ctxAndLogForRequest[T any](ctx context.Context, req *connect.Request[T]) (context.Context, *slog.Logger) { + log := dlog.FromCtx(ctx) + + // Add streamId to log context if present in request + if reqMsg, ok := any(req.Msg).(RequestWithStreamId); ok { + streamId := reqMsg.GetStreamId() + if streamId != "" { + log = log.With("streamId", streamId) + return dlog.CtxWithLog(ctx, log), log + } + } + + return ctx, log +} diff --git a/core/node/rpc/sync_receiver.go b/core/node/rpc/sync_receiver.go deleted file mode 100644 index 9a918b904d..0000000000 --- a/core/node/rpc/sync_receiver.go +++ /dev/null @@ -1,84 +0,0 @@ -package rpc - -import ( - "context" - "sync" - - . "github.com/river-build/river/core/node/base" - "github.com/river-build/river/core/node/dlog" - . "github.com/river-build/river/core/node/events" - . "github.com/river-build/river/core/node/protocol" -) - -type syncReceiver struct { - ctx context.Context - cancel context.CancelFunc - channel chan *StreamAndCookie - - mu sync.Mutex - firstError error -} - -var _ SyncResultReceiver = (*syncReceiver)(nil) - -func (s *syncReceiver) OnUpdate(r *StreamAndCookie) { - if s.ctx.Err() != nil { - return - } - - select { - case s.channel <- r: - return - default: - err := RiverError( - Err_BUFFER_FULL, - "channel full, dropping update and canceling", - "streamId", - r.NextSyncCookie.StreamId, - ). - Func("OnUpdate"). - LogWarn(dlog.FromCtx(s.ctx)) - s.setErrorAndCancel(err) - return - } -} - -func (s *syncReceiver) OnSyncError(err error) { - if s.ctx.Err() != nil { - return - } - s.setErrorAndCancel(err) - dlog.FromCtx(s.ctx).Warn("OnSyncError: cancelling sync", "error", err) -} - -func (s *syncReceiver) setErrorAndCancel(err error) { - s.mu.Lock() - if s.firstError == nil { - s.firstError = err - } - s.mu.Unlock() - - s.cancel() -} - -func (s *syncReceiver) Dispatch(sender syncStream) { - log := dlog.FromCtx(s.ctx) - - for { - select { - case <-s.ctx.Done(): - err := s.ctx.Err() - s.setErrorAndCancel(err) - log.Debug("SyncStreams: context done", "err", err) - return - case data := <-s.channel: - log.Debug("SyncStreams: received update in forward loop", "data", data) - resp := SyncStreamsResponseFromStreamAndCookie(data) - if err := sender.Send(resp); err != nil { - s.setErrorAndCancel(err) - log.Debug("SyncStreams: failed to send update", "resp", data, "err", err) - return - } - } - } -} diff --git a/core/node/rpc/sync_streams.go b/core/node/rpc/sync_streams.go deleted file mode 100644 index d9a3e5187b..0000000000 --- a/core/node/rpc/sync_streams.go +++ /dev/null @@ -1,843 +0,0 @@ -package rpc - -import ( - "bytes" - "context" - "errors" - "sync" - - "connectrpc.com/connect" - "github.com/ethereum/go-ethereum/common" - - . "github.com/river-build/river/core/node/base" - "github.com/river-build/river/core/node/crypto" - "github.com/river-build/river/core/node/dlog" - "github.com/river-build/river/core/node/events" - "github.com/river-build/river/core/node/nodes" - . "github.com/river-build/river/core/node/protocol" - "github.com/river-build/river/core/node/protocol/protocolconnect" - . "github.com/river-build/river/core/node/shared" -) - -// TODO: wire metrics. -// var ( -// syncStreamsRequests = infra.NewSuccessMetrics("sync_streams_requests", serviceRequests) -// syncStreamsResultSize = infra.NewCounter("sync_streams_result_size", "The total number of events returned by sync streams") -// ) - -// func addUpdatesToCounter(updates []*StreamAndCookie) { -// for _, stream := range updates { -// syncStreamsResultSize.Add(float64(len(stream.Events))) -// } -// } - -func NewSyncHandler( - wallet *crypto.Wallet, - cache events.StreamCache, - nodeRegistry nodes.NodeRegistry, - streamRegistry nodes.StreamRegistry, -) SyncHandler { - return &syncHandlerImpl{ - wallet: wallet, - cache: cache, - nodeRegistry: nodeRegistry, - streamRegistry: streamRegistry, - mu: sync.Mutex{}, - syncIdToSubscription: make(map[string]*syncSubscriptionImpl), - } -} - -type SyncHandler interface { - SyncStreams( - ctx context.Context, - req *connect.Request[SyncStreamsRequest], - res *connect.ServerStream[SyncStreamsResponse], - ) error - AddStreamToSync( - ctx context.Context, - req *connect.Request[AddStreamToSyncRequest], - ) (*connect.Response[AddStreamToSyncResponse], error) - RemoveStreamFromSync( - ctx context.Context, - req *connect.Request[RemoveStreamFromSyncRequest], - ) (*connect.Response[RemoveStreamFromSyncResponse], error) - CancelSync( - ctx context.Context, - req *connect.Request[CancelSyncRequest], - ) (*connect.Response[CancelSyncResponse], error) - PingSync( - ctx context.Context, - req *connect.Request[PingSyncRequest], - ) (*connect.Response[PingSyncResponse], error) -} - -type syncHandlerImpl struct { - wallet *crypto.Wallet - cache events.StreamCache - nodeRegistry nodes.NodeRegistry - streamRegistry nodes.StreamRegistry - mu sync.Mutex - syncIdToSubscription map[string]*syncSubscriptionImpl -} - -type syncNode struct { - address common.Address - remoteSyncId string // the syncId to the remote node's sync subscription - forwarderSyncId string // the forwarding node's sync Id - stub protocolconnect.StreamServiceClient - - mu sync.Mutex - closed bool -} - -func (s *Service) SyncStreams( - ctx context.Context, - req *connect.Request[SyncStreamsRequest], - res *connect.ServerStream[SyncStreamsResponse], -) error { - return s.syncHandler.SyncStreams(ctx, req, res) -} - -func (s *Service) AddStreamToSync( - ctx context.Context, - req *connect.Request[AddStreamToSyncRequest], -) (*connect.Response[AddStreamToSyncResponse], error) { - return s.syncHandler.AddStreamToSync(ctx, req) -} - -func (s *Service) RemoveStreamFromSync( - ctx context.Context, - req *connect.Request[RemoveStreamFromSyncRequest], -) (*connect.Response[RemoveStreamFromSyncResponse], error) { - return s.syncHandler.RemoveStreamFromSync(ctx, req) -} - -func (s *Service) CancelSync( - ctx context.Context, - req *connect.Request[CancelSyncRequest], -) (*connect.Response[CancelSyncResponse], error) { - return s.syncHandler.CancelSync(ctx, req) -} - -func (s *Service) PingSync( - ctx context.Context, - req *connect.Request[PingSyncRequest], -) (*connect.Response[PingSyncResponse], error) { - return s.syncHandler.PingSync(ctx, req) -} - -func (s *syncHandlerImpl) SyncStreams( - ctx context.Context, - req *connect.Request[SyncStreamsRequest], - res *connect.ServerStream[SyncStreamsResponse], -) error { - ctx, log := ctxAndLogForRequest(ctx, req) - - // generate a random syncId - syncId := GenNanoid() - log.Debug("SyncStreams:SyncHandlerV2.SyncStreams ENTER", "syncId", syncId, "syncPos", req.Msg.SyncPos) - - sub, err := s.addSubscription(ctx, syncId) - if err != nil { - log.Info( - "SyncStreams:SyncHandlerV2.SyncStreams LEAVE: failed to add subscription", - "syncId", - syncId, - "err", - err, - ) - return err - } - - // send syncId to client - e := res.Send(&SyncStreamsResponse{ - SyncId: syncId, - SyncOp: SyncOp_SYNC_NEW, - }) - if e != nil { - err := AsRiverError(e).Func("SyncStreams") - log.Info( - "SyncStreams:SyncHandlerV2.SyncStreams LEAVE: failed to send syncId", - "res", - res, - "err", - err, - "syncId", - syncId, - ) - return err - } - log.Debug("SyncStreams:SyncHandlerV2.SyncStreams: sent syncId", "syncId", syncId) - - e = s.handleSyncRequest(req, res, sub) - if e != nil { - err := AsRiverError(e).Func("SyncStreams") - if err.Code == Err_CANCELED { - // Context is canceled when client disconnects, so this is normal case. - log.Debug( - "SyncStreams:SyncHandlerV2.SyncStreams LEAVE: sync Dispatch() ended with expected error", - "syncId", - syncId, - ) - _ = err.LogDebug(log) - } else { - log.Info("SyncStreams:SyncHandlerV2.SyncStreams LEAVE: sync Dispatch() ended with unexpected error", "syncId", syncId) - _ = err.LogWarn(log) - } - return err.AsConnectError() - } - // no errors from handling the sync request. - log.Debug("SyncStreams:SyncHandlerV2.SyncStreams LEAVE") - return nil -} - -func (s *syncHandlerImpl) handleSyncRequest( - req *connect.Request[SyncStreamsRequest], - res *connect.ServerStream[SyncStreamsResponse], - sub *syncSubscriptionImpl, -) error { - if sub == nil { - return RiverError(Err_NOT_FOUND, "SyncId not found").Func("SyncStreams") - } - log := dlog.FromCtx(sub.ctx) - - defer s.removeSubscription(sub.ctx, sub.syncId) - - localCookies, remoteCookies := getLocalAndRemoteCookies(s.wallet.Address, req.Msg.SyncPos) - - for nodeAddr, remoteCookie := range remoteCookies { - var r *syncNode - if r = sub.getRemoteNode(nodeAddr); r == nil { - stub, err := s.nodeRegistry.GetStreamServiceClientForAddress(nodeAddr) - if err != nil { - // TODO: Handle the case when node is no longer available. HNT-4715 - log.Error( - "SyncStreams:SyncHandlerV2.SyncStreams failed to get stream service client", - "syncId", - sub.syncId, - "err", - err, - ) - return err - } - - r = &syncNode{ - address: nodeAddr, - forwarderSyncId: sub.syncId, - stub: stub, - } - } - err := sub.addSyncNode(r, remoteCookie) - if err != nil { - return err - } - } - - if len(localCookies) > 0 { - go s.syncLocalNode(sub.ctx, localCookies, sub) - } - - remotes := sub.getRemoteNodes() - for _, remote := range remotes { - cookies := remoteCookies[remote.address] - go remote.syncRemoteNode(sub.ctx, sub.syncId, cookies, sub) - } - - // start the sync loop - log.Debug("SyncStreams:SyncHandlerV2.SyncStreams: sync Dispatch() started", "syncId", sub.syncId) - sub.Dispatch(res) - log.Debug("SyncStreams:SyncHandlerV2.SyncStreams: sync Dispatch() ended", "syncId", sub.syncId) - - err := sub.getError() - if err != nil { - log.Debug( - "SyncStreams:SyncHandlerV2.SyncStreams LEAVE: sync Dispatch() ended with expected error", - "syncId", - sub.syncId, - ) - return err - } - - log.Error("SyncStreams:SyncStreams: sync always should be terminated by context cancel.") - return nil -} - -func (s *syncHandlerImpl) CancelSync( - ctx context.Context, - req *connect.Request[CancelSyncRequest], -) (*connect.Response[CancelSyncResponse], error) { - _, log := ctxAndLogForRequest(ctx, req) - log.Debug("SyncStreams:SyncHandlerV2.CancelSync ENTER", "syncId", req.Msg.SyncId) - sub := s.getSub(req.Msg.SyncId) - if sub != nil { - sub.OnClose() - } - log.Debug("SyncStreams:SyncHandlerV2.CancelSync LEAVE", "syncId", req.Msg.SyncId) - return connect.NewResponse(&CancelSyncResponse{}), nil -} - -func (s *syncHandlerImpl) PingSync( - ctx context.Context, - req *connect.Request[PingSyncRequest], -) (*connect.Response[PingSyncResponse], error) { - _, log := ctxAndLogForRequest(ctx, req) - syncId := req.Msg.SyncId - - sub := s.getSub(syncId) - if sub == nil { - log.Debug("SyncStreams: ping sync", "syncId", syncId) - return nil, RiverError(Err_NOT_FOUND, "SyncId not found").Func("PingSync") - } - - // cancel if context is done - if sub.ctx.Err() != nil { - log.Debug("SyncStreams: ping sync", "syncId", syncId, "context_error", sub.ctx.Err()) - return nil, RiverError(Err_CANCELED, "SyncId canceled").Func("PingSync") - } - - log.Debug("SyncStreams: ping sync", "syncId", syncId) - c := pingOp{ - baseSyncOp: baseSyncOp{op: SyncOp_SYNC_PONG}, - nonce: req.Msg.Nonce, - } - select { - // send the pong response to the client via the control channel - case sub.controlChannel <- &c: - return connect.NewResponse(&PingSyncResponse{}), nil - default: - return nil, RiverError(Err_BUFFER_FULL, "control channel full").Func("PingSync") - } -} - -func getLocalAndRemoteCookies( - localWalletAddr common.Address, - syncCookies []*SyncCookie, -) (localCookies []*SyncCookie, remoteCookies map[common.Address][]*SyncCookie) { - localCookies = make([]*SyncCookie, 0, 8) - remoteCookies = make(map[common.Address][]*SyncCookie) - for _, cookie := range syncCookies { - if bytes.Equal(cookie.NodeAddress, localWalletAddr[:]) { - localCookies = append(localCookies, cookie) - } else { - remoteAddr := common.BytesToAddress(cookie.NodeAddress) - if remoteCookies[remoteAddr] == nil { - remoteCookies[remoteAddr] = make([]*SyncCookie, 0, 8) - } - remoteCookies[remoteAddr] = append(remoteCookies[remoteAddr], cookie) - } - } - return -} - -func (s *syncHandlerImpl) syncLocalNode( - ctx context.Context, - syncPos []*SyncCookie, - sub *syncSubscriptionImpl, -) { - log := dlog.FromCtx(ctx) - - if ctx.Err() != nil { - log.Error("SyncStreams:SyncHandlerV2.SyncStreams: syncLocalNode not starting", "context_error", ctx.Err()) - return - } - - err := s.syncLocalStreamsImpl(ctx, syncPos, sub) - if err != nil { - log.Error("SyncStreams:SyncHandlerV2.SyncStreams: syncLocalNode failed", "err", err) - if sub != nil { - sub.OnSyncError(err) - } - } -} - -func (s *syncHandlerImpl) syncLocalStreamsImpl( - ctx context.Context, - syncPos []*SyncCookie, - sub *syncSubscriptionImpl, -) error { - if len(syncPos) <= 0 { - return nil - } - - defer func() { - if sub != nil { - sub.unsubLocalStreams() - } - }() - - for _, pos := range syncPos { - if ctx.Err() != nil { - return ctx.Err() - } - - err := s.addLocalStreamToSync(ctx, pos, sub) - if err != nil { - return err - } - } - - // Wait for context to be done before unsubbing. - <-ctx.Done() - return nil -} - -func (s *syncHandlerImpl) addLocalStreamToSync( - ctx context.Context, - cookie *SyncCookie, - subs *syncSubscriptionImpl, -) error { - log := dlog.FromCtx(ctx) - log.Debug("SyncStreams:SyncHandlerV2.addLocalStreamToSync ENTER", "syncId", subs.syncId, "syncPos", cookie) - - if ctx.Err() != nil { - log.Error("SyncStreams:SyncHandlerV2.addLocalStreamToSync: context error", "err", ctx.Err()) - return ctx.Err() - } - if subs == nil { - return RiverError(Err_NOT_FOUND, "SyncId not found").Func("SyncStreams") - } - - err := events.SyncCookieValidate(cookie) - if err != nil { - log.Debug("SyncStreams:SyncHandlerV2.addLocalStreamToSync: invalid cookie", "err", err) - return nil - } - - cookieStreamId, err := StreamIdFromBytes(cookie.StreamId) - if err != nil { - return err - } - - if s := subs.getLocalStream(cookieStreamId); s != nil { - // stream is already subscribed. no need to re-subscribe. - log.Debug( - "SyncStreams:SyncHandlerV2.addLocalStreamToSync: stream already subscribed", - "streamId", - cookieStreamId, - ) - return nil - } - - streamSub, err := s.cache.GetSyncStream(ctx, cookieStreamId) - if err != nil { - log.Info( - "SyncStreams:SyncHandlerV2.addLocalStreamToSync: failed to get stream", - "streamId", - cookieStreamId, - "err", - err, - ) - return err - } - - err = subs.addLocalStream(ctx, cookie, &streamSub) - if err != nil { - log.Info( - "SyncStreams:SyncHandlerV2.addLocalStreamToSync: error subscribing to stream", - "streamId", - cookie.StreamId, - "err", - err, - ) - return err - } - - log.Debug( - "SyncStreams:SyncHandlerV2.addLocalStreamToSync LEAVE", - "syncId", - subs.syncId, - "streamId", - cookie.StreamId, - ) - return nil -} - -func (s *syncHandlerImpl) AddStreamToSync( - ctx context.Context, - req *connect.Request[AddStreamToSyncRequest], -) (*connect.Response[AddStreamToSyncResponse], error) { - ctx, log := ctxAndLogForRequest(ctx, req) - log.Debug("SyncStreams:SyncHandlerV2.AddStreamToSync ENTER", "syncId", req.Msg.SyncId, "syncPos", req.Msg.SyncPos) - - syncId := req.Msg.SyncId - cookie := req.Msg.SyncPos - - log.Debug("SyncStreams:SyncHandlerV2.AddStreamToSync: getting sub", "syncId", syncId) - sub := s.getSub(syncId) - if sub == nil { - log.Info("SyncStreams:SyncHandlerV2.AddStreamToSync LEAVE: SyncId not found", "syncId", syncId) - return nil, RiverError(Err_NOT_FOUND, "SyncId not found").Func("AddStreamToSync") - } - log.Debug("SyncStreams:SyncHandlerV2.AddStreamToSync: got sub", "syncId", syncId) - - // Two cases to handle. Either local cookie or remote cookie. - if bytes.Equal(cookie.NodeAddress[:], s.wallet.Address[:]) { - // Case 1: local cookie - if err := s.addLocalStreamToSync(ctx, cookie, sub); err != nil { - log.Info( - "SyncStreams:SyncHandlerV2.AddStreamToSync LEAVE: failed to add local streams", - "syncId", - syncId, - "err", - err, - ) - return nil, err - } - // done. - log.Debug("SyncStreams:SyncHandlerV2.AddStreamToSync: LEAVE", "syncId", syncId) - return connect.NewResponse(&AddStreamToSyncResponse{}), nil - } - - // Case 2: remote cookie - log.Debug("SyncStreams:SyncHandlerV2.AddStreamToSync: adding remote streams", "syncId", syncId) - nodeAddress := common.BytesToAddress(cookie.NodeAddress[:]) - remoteNode := sub.getRemoteNode(nodeAddress) - isNewRemoteNode := remoteNode == nil - log.Debug( - "SyncStreams:SyncHandlerV2.AddStreamToSync: remote node", - "syncId", - syncId, - "isNewRemoteNode", - isNewRemoteNode, - ) - if isNewRemoteNode { - // the remote node does not exist in the subscription. add it. - stub, err := s.nodeRegistry.GetStreamServiceClientForAddress(nodeAddress) - if err != nil { - log.Info( - "SyncStreams:SyncHandlerV2.AddStreamToSync: failed to get stream service client", - "syncId", - req.Msg.SyncId, - "err", - err, - ) - // TODO: Handle the case when node is no longer available. - return nil, err - } - if stub == nil { - panic("stub always should set for the remote node") - } - - remoteNode = &syncNode{ - address: nodeAddress, - forwarderSyncId: sub.syncId, - stub: stub, - } - sub.addRemoteNode(nodeAddress, remoteNode) - log.Info("SyncStreams:SyncHandlerV2.AddStreamToSync: added remote node", "syncId", req.Msg.SyncId) - } - err := sub.addRemoteStream(cookie) - if err != nil { - log.Info( - "SyncStreams:SyncHandlerV2.AddStreamToSync LEAVE: failed to add remote streams", - "syncId", - req.Msg.SyncId, - "err", - err, - ) - return nil, err - } - log.Info("SyncStreams:SyncHandlerV2.AddStreamToSync: added remote stream", "syncId", req.Msg.SyncId) - - if isNewRemoteNode { - // tell the new remote node to sync - syncPos := make([]*SyncCookie, 0, 1) - syncPos = append(syncPos, cookie) - log.Info("SyncStreams:SyncHandlerV2.AddStreamToSync: syncing new remote node", "syncId", req.Msg.SyncId) - go remoteNode.syncRemoteNode(sub.ctx, sub.syncId, syncPos, sub) - } else { - log.Info("SyncStreams:SyncHandlerV2.AddStreamToSync: adding stream to existing remote node", "syncId", req.Msg.SyncId) - // tell the existing remote nodes to add the streams to sync - go remoteNode.addStreamToSync(sub.ctx, cookie, sub) - } - - log.Debug("SyncStreams:SyncHandlerV2.AddStreamToSync LEAVE", "syncId", req.Msg.SyncId) - return connect.NewResponse(&AddStreamToSyncResponse{}), nil -} - -func (s *syncHandlerImpl) RemoveStreamFromSync( - ctx context.Context, - req *connect.Request[RemoveStreamFromSyncRequest], -) (*connect.Response[RemoveStreamFromSyncResponse], error) { - _, log := ctxAndLogForRequest(ctx, req) - log.Info( - "SyncStreams:SyncHandlerV2.RemoveStreamFromSync ENTER", - "syncId", - req.Msg.SyncId, - "streamId", - req.Msg.StreamId, - ) - - syncId := req.Msg.SyncId - streamId, err := StreamIdFromBytes(req.Msg.StreamId) - if err != nil { - log.Info( - "SyncStreams:SyncHandlerV2.RemoveStreamFromSync LEAVE: failed to parse streamId", - "syncId", - syncId, - "err", - err, - ) - return nil, err - } - - sub := s.getSub(syncId) - if sub == nil { - log.Info("SyncStreams:SyncHandlerV2.RemoveStreamFromSync LEAVE: SyncId not found", "syncId", syncId) - return nil, RiverError(Err_NOT_FOUND, "SyncId not found").Func("RemoveStreamFromSync") - } - - // remove the streamId from the local node - sub.removeLocalStream(streamId) - - // use the streamId to find the remote node to remove - remoteNode := sub.removeRemoteStream(streamId) - if remoteNode != nil { - log.Debug( - "SyncStreams:SyncHandlerV2.RemoveStreamFromSync: removing remote stream", - "syncId", - syncId, - "streamId", - streamId, - ) - err := remoteNode.removeStreamFromSync(sub.ctx, streamId, sub) - if err != nil { - log.Info( - "SyncStreams:SyncHandlerV2.RemoveStreamFromSync: failed to remove remote stream", - "syncId", - syncId, - "streamId", - streamId, - "err", - err, - ) - return nil, err - } - // remove any remote nodes that no longer have any streams to sync - sub.purgeUnusedRemoteNodes(log) - } - - log.Info("SyncStreams:SyncHandlerV2.RemoveStreamFromSync LEAVE", "syncId", syncId) - return connect.NewResponse(&RemoveStreamFromSyncResponse{}), nil -} - -func (s *syncHandlerImpl) addSubscription( - ctx context.Context, - syncId string, -) (*syncSubscriptionImpl, error) { - log := dlog.FromCtx(ctx) - s.mu.Lock() - defer s.mu.Unlock() - - if s.syncIdToSubscription == nil { - s.syncIdToSubscription = make(map[string]*syncSubscriptionImpl) - } - if sub := s.syncIdToSubscription[syncId]; sub != nil { - return nil, errors.New("syncId subscription already exists") - } - sub := newSyncSubscription(ctx, syncId) - s.syncIdToSubscription[syncId] = sub - log.Debug("SyncStreams:addSubscription: syncId subscription added", "syncId", syncId) - return sub, nil -} - -func (s *syncHandlerImpl) removeSubscription( - ctx context.Context, - syncId string, -) { - log := dlog.FromCtx(ctx) - sub := s.getSub(syncId) - if sub != nil { - sub.deleteRemoteNodes() - } - s.mu.Lock() - if _, exists := s.syncIdToSubscription[syncId]; exists { - delete(s.syncIdToSubscription, syncId) - log.Debug("SyncStreams:removeSubscription: syncId subscription removed", "syncId", syncId) - } else { - log.Debug("SyncStreams:removeSubscription: syncId not found", "syncId", syncId) - } - s.mu.Unlock() -} - -func (s *syncHandlerImpl) getSub( - syncId string, -) *syncSubscriptionImpl { - s.mu.Lock() - defer s.mu.Unlock() - return s.syncIdToSubscription[syncId] -} - -// TODO: connect-go is not using channels for streaming (>_<), so it's a bit tricky to close all these -// streams properly. For now basic protocol is to close entire sync if there is any error. -// Which in turn means that we need to close all outstanding streams to remote nodes. -// Without control signals there is no clean way to do so, so for now both ctx is canceled and Close is called -// async hoping this will trigger Receive to abort. -func (n *syncNode) syncRemoteNode( - ctx context.Context, - forwarderSyncId string, - syncPos []*SyncCookie, - receiver events.SyncResultReceiver, -) { - log := dlog.FromCtx(ctx) - if ctx.Err() != nil || n.isClosed() { - log.Debug("SyncStreams: syncRemoteNode not started", "context_error", ctx.Err()) - return - } - if n.remoteSyncId != "" { - log.Debug( - "SyncStreams: syncRemoteNode not started because there is an existing sync", - "remoteSyncId", - n.remoteSyncId, - "forwarderSyncId", - forwarderSyncId, - ) - return - } - - defer func() { - if n != nil { - n.close() - } - }() - - responseStream, err := n.stub.SyncStreams( - ctx, - &connect.Request[SyncStreamsRequest]{ - Msg: &SyncStreamsRequest{ - SyncPos: syncPos, - }, - }, - ) - if err != nil { - log.Debug("SyncStreams: syncRemoteNode remote SyncStreams failed", "err", err) - receiver.OnSyncError(err) - return - } - defer responseStream.Close() - - if ctx.Err() != nil || n.isClosed() { - log.Debug("SyncStreams: syncRemoteNode receive canceled", "context_error", ctx.Err()) - return - } - - if !responseStream.Receive() { - receiver.OnSyncError(responseStream.Err()) - return - } - - if responseStream.Msg().SyncOp != SyncOp_SYNC_NEW || responseStream.Msg().SyncId == "" { - receiver.OnSyncError( - RiverError(Err_INTERNAL, "first sync response should be SYNC_NEW and have SyncId").Func("syncRemoteNode"), - ) - return - } - - n.remoteSyncId = responseStream.Msg().SyncId - n.forwarderSyncId = forwarderSyncId - - if ctx.Err() != nil || n.isClosed() { - log.Debug("SyncStreams: syncRemoteNode receive canceled", "context_error", ctx.Err()) - return - } - - for responseStream.Receive() { - if ctx.Err() != nil || n.isClosed() { - log.Debug("SyncStreams: syncRemoteNode receive canceled", "context_error", ctx.Err()) - return - } - - log.Debug("SyncStreams: syncRemoteNode received update", "resp", responseStream.Msg()) - - receiver.OnUpdate(responseStream.Msg().GetStream()) - } - - if ctx.Err() != nil || n.isClosed() { - return - } - - if err := responseStream.Err(); err != nil { - log.Debug("SyncStreams: syncRemoteNode receive failed", "err", err) - receiver.OnSyncError(err) - return - } -} - -func (n *syncNode) addStreamToSync( - ctx context.Context, - cookie *SyncCookie, - receiver events.SyncResultReceiver, -) { - log := dlog.FromCtx(ctx) - if ctx.Err() != nil || n.isClosed() { - log.Debug("SyncStreams:syncNode addStreamToSync not started", "context_error", ctx.Err()) - } - if n.remoteSyncId == "" { - log.Debug( - "SyncStreams:syncNode addStreamToSync not started because there is no existing sync", - "remoteSyncId", - n.remoteSyncId, - ) - } - - _, err := n.stub.AddStreamToSync( - ctx, - &connect.Request[AddStreamToSyncRequest]{ - Msg: &AddStreamToSyncRequest{ - SyncPos: cookie, - SyncId: n.remoteSyncId, - }, - }, - ) - if err != nil { - log.Debug("SyncStreams:syncNode addStreamToSync failed", "err", err) - receiver.OnSyncError(err) - } -} - -func (n *syncNode) removeStreamFromSync( - ctx context.Context, - streamId StreamId, - receiver events.SyncResultReceiver, -) error { - log := dlog.FromCtx(ctx) - if ctx.Err() != nil || n.isClosed() { - log.Debug("SyncStreams:syncNode removeStreamsFromSync not started", "context_error", ctx.Err()) - return ctx.Err() - } - if n.remoteSyncId == "" { - log.Debug( - "SyncStreams:syncNode removeStreamsFromSync not started because there is no existing sync", - "syncId", - n.remoteSyncId, - ) - return nil - } - - _, err := n.stub.RemoveStreamFromSync( - ctx, - &connect.Request[RemoveStreamFromSyncRequest]{ - Msg: &RemoveStreamFromSyncRequest{ - SyncId: n.remoteSyncId, - StreamId: streamId[:], - }, - }, - ) - if err != nil { - log.Debug("SyncStreams:syncNode removeStreamsFromSync failed", "err", err) - receiver.OnSyncError(err) - } - return err -} - -func (n *syncNode) isClosed() bool { - n.mu.Lock() - defer n.mu.Unlock() - return n.closed -} - -func (n *syncNode) close() { - n.mu.Lock() - defer n.mu.Unlock() - n.closed = true -} diff --git a/core/node/rpc/sync_subscription.go b/core/node/rpc/sync_subscription.go deleted file mode 100644 index 20562598c8..0000000000 --- a/core/node/rpc/sync_subscription.go +++ /dev/null @@ -1,423 +0,0 @@ -package rpc - -import ( - "context" - "log/slog" - "sync" - - "connectrpc.com/connect" - "github.com/ethereum/go-ethereum/common" - - . "github.com/river-build/river/core/node/base" - "github.com/river-build/river/core/node/dlog" - "github.com/river-build/river/core/node/events" - . "github.com/river-build/river/core/node/protocol" - . "github.com/river-build/river/core/node/shared" -) - -type syncOp interface { - getOp() SyncOp -} - -type baseSyncOp struct { - op SyncOp -} - -func (d *baseSyncOp) getOp() SyncOp { - return d.op -} - -type pingOp struct { - baseSyncOp - nonce string // used to match a response to a ping request -} - -type syncSubscriptionImpl struct { - ctx context.Context - syncId string - cancel context.CancelFunc - - mu sync.Mutex - firstError error - dataChannel chan *StreamAndCookie - controlChannel chan syncOp - localStreams map[StreamId]*events.SyncStream // mapping of streamId to local stream - remoteStreams map[StreamId]*syncNode // mapping of streamId to remote node - remoteNodes map[common.Address]*syncNode // mapping of node address to remote node -} - -func newSyncSubscription( - ctx context.Context, - syncId string, -) *syncSubscriptionImpl { - syncCtx, cancelSync := context.WithCancel(ctx) - return &syncSubscriptionImpl{ - ctx: syncCtx, - syncId: syncId, - cancel: cancelSync, - dataChannel: make(chan *StreamAndCookie, 256), - controlChannel: make(chan syncOp, 64), - localStreams: make(map[StreamId]*events.SyncStream), - remoteStreams: make(map[StreamId]*syncNode), - remoteNodes: make(map[common.Address]*syncNode), - } -} - -type syncStream interface { - Send(msg *SyncStreamsResponse) error -} - -func (s *syncSubscriptionImpl) addLocalStream( - ctx context.Context, - syncCookie *SyncCookie, - stream *events.SyncStream, -) error { - log := dlog.FromCtx(ctx) - log.Debug( - "SyncStreams:syncSubscriptionImpl:addLocalStream: adding local stream", - "syncId", - s.syncId, - "streamId", - syncCookie.StreamId, - ) - streamId, err := StreamIdFromBytes(syncCookie.StreamId) - if err != nil { - return err - } - - var exists bool - - s.mu.Lock() - - // only add the stream if it doesn't already exist in the subscription - if _, exists = s.localStreams[streamId]; !exists { - s.localStreams[streamId] = stream - } - s.mu.Unlock() - - if exists { - log.Debug( - "SyncStreams:syncSubscriptionImpl:addLocalStream: local stream already exists", - "syncId", - s.syncId, - "streamId", - syncCookie.StreamId, - ) - } else { - // subscribe to the stream - err := (*stream).Sub(ctx, syncCookie, s) - if err != nil { - log.Error("SyncStreams:syncSubscriptionImpl:addLocalStream: error subscribing to stream", "syncId", s.syncId, "streamId", syncCookie.StreamId, "err", err) - return err - } - log.Debug("SyncStreams:syncSubscriptionImpl:addLocalStream: added local stream", "syncId", s.syncId, "streamId", syncCookie.StreamId) - } - - return nil -} - -func (s *syncSubscriptionImpl) removeLocalStream( - streamId StreamId, -) { - var stream *events.SyncStream - - s.mu.Lock() - if st := s.localStreams[streamId]; st != nil { - stream = st - delete(s.localStreams, streamId) - } - s.mu.Unlock() - - if stream != nil { - (*stream).Unsub(s) - } -} - -func (s *syncSubscriptionImpl) unsubLocalStreams() { - s.mu.Lock() - defer s.mu.Unlock() - for key, st := range s.localStreams { - stream := *st - stream.Unsub(s) - delete(s.localStreams, key) - } -} - -func (s *syncSubscriptionImpl) addSyncNode( - node *syncNode, - cookies []*SyncCookie, -) error { - s.mu.Lock() - defer s.mu.Unlock() - - if _, exists := s.remoteNodes[node.address]; !exists { - s.remoteNodes[node.address] = node - } else { - node = s.remoteNodes[node.address] - } - for _, cookie := range cookies { - streamId, err := StreamIdFromBytes(cookie.StreamId) - if err != nil { - return err - } - s.remoteStreams[streamId] = node - } - return nil -} - -func (s *syncSubscriptionImpl) addRemoteNode( - address common.Address, - node *syncNode, -) bool { - s.mu.Lock() - defer s.mu.Unlock() - // only add the node if it doesn't already exist in the subscription - if _, exists := s.remoteNodes[address]; !exists { - s.remoteNodes[address] = node - return true // added - } - return false // not added -} - -func (s *syncSubscriptionImpl) getLocalStream( - streamId StreamId, -) *events.SyncStream { - s.mu.Lock() - defer s.mu.Unlock() - return s.localStreams[streamId] -} - -func (s *syncSubscriptionImpl) getRemoteNode( - address common.Address, -) *syncNode { - s.mu.Lock() - defer s.mu.Unlock() - return s.remoteNodes[address] -} - -func (s *syncSubscriptionImpl) getRemoteNodes() []*syncNode { - copy := make([]*syncNode, 0) - s.mu.Lock() - defer s.mu.Unlock() - for _, node := range s.remoteNodes { - copy = append(copy, node) - } - return copy -} - -func (s *syncSubscriptionImpl) addRemoteStream( - cookie *SyncCookie, -) error { - s.mu.Lock() - defer s.mu.Unlock() - nodeAddress := common.BytesToAddress(cookie.NodeAddress) - if remote := s.remoteNodes[nodeAddress]; remote != nil { - streamId, err := StreamIdFromBytes(cookie.StreamId) - if err != nil { - return err - } - s.remoteStreams[streamId] = remote - } - return nil -} - -func (s *syncSubscriptionImpl) removeRemoteStream( - streamId StreamId, -) *syncNode { - s.mu.Lock() - defer s.mu.Unlock() - if remote := s.remoteStreams[streamId]; remote != nil { - delete(s.remoteStreams, streamId) - return remote - } - return nil -} - -func (s *syncSubscriptionImpl) purgeUnusedRemoteNodes(log *slog.Logger) { - nodesToRemove := make([]*syncNode, 0) - - log.Debug( - "SyncStreams:syncSubscriptionImpl:purgeUnusedRemoteNodes: purging unused remote nodes", - "syncId", - s.syncId, - ) - - s.mu.Lock() - if len(s.remoteNodes) > 0 { - for _, remote := range s.remoteNodes { - isUsed := false - if len(s.remoteStreams) > 0 { - for _, n := range s.remoteStreams { - if n == remote { - isUsed = true - break - } - } - if !isUsed { - nodesToRemove = append(nodesToRemove, remote) - delete(s.remoteNodes, remote.address) - } - } - } - } - s.mu.Unlock() - - // now purge the nodes - for _, remote := range nodesToRemove { - if remote != nil { - remote.close() - } - } - - log.Debug("SyncStreams:syncSubscriptionImpl:purgeUnusedRemoteNodes: purged remote nodes done", "syncId", s.syncId) -} - -func (s *syncSubscriptionImpl) deleteRemoteNodes() { - s.mu.Lock() - defer s.mu.Unlock() - for key := range s.remoteNodes { - delete(s.remoteNodes, key) - } - for key := range s.remoteStreams { - delete(s.remoteStreams, key) - } -} - -func (s *syncSubscriptionImpl) setErrorAndCancel(err error) { - s.mu.Lock() - if s.firstError == nil { - s.firstError = err - } - s.mu.Unlock() - - s.cancel() -} - -func (s *syncSubscriptionImpl) OnSyncError(err error) { - if s.ctx.Err() != nil { - return - } - log := dlog.FromCtx(s.ctx) - log.Info("SyncStreams:syncSubscriptionImpl:OnSyncError: received error", "error", err) - s.setErrorAndCancel(err) - log.Warn("SyncStreams:syncSubscriptionImpl:OnSyncError: cancelling sync", "error", err) -} - -func (s *syncSubscriptionImpl) OnUpdate(r *StreamAndCookie) { - // cancel if context is done - if s.ctx.Err() != nil { - return - } - - select { - case s.dataChannel <- r: - return - default: - // end the update stream if the channel is full - err := RiverError( - Err_BUFFER_FULL, - "channel full, dropping update and canceling", - "streamId", - r.NextSyncCookie.StreamId, - ). - Func("OnUpdate"). - LogWarn(dlog.FromCtx(s.ctx)) - s.setErrorAndCancel(err) - return - } -} - -func (s *syncSubscriptionImpl) OnClose() { - // cancel if context is done - if s.ctx.Err() != nil { - return - } - - log := dlog.FromCtx(s.ctx) - log.Debug("SyncStreams:OnClose: closing stream", "syncId", s.syncId) - c := baseSyncOp{ - op: SyncOp_SYNC_CLOSE, - } - select { - case s.controlChannel <- &c: - return - default: - log.Info("SyncStreams:OnClose: control channel full") - return - } -} - -func (s *syncSubscriptionImpl) Dispatch(res *connect.ServerStream[SyncStreamsResponse]) { - log := dlog.FromCtx(s.ctx) - - for { - select { - case <-s.ctx.Done(): - err := s.ctx.Err() - s.setErrorAndCancel(err) - log.Debug("SyncStreams: context done", "err", err) - return - case data, ok := <-s.dataChannel: - log.Debug( - "SyncStreams: Dispatch received response in dispatch loop", - "syncId", - s.syncId, - "data", - data, - ) - if ok { - // gather the response metadata + content, and send it - resp := events.SyncStreamsResponseFromStreamAndCookie(data) - resp.SyncId = s.syncId - resp.SyncOp = SyncOp_SYNC_UPDATE - if err := res.Send(resp); err != nil { - log.Info("SyncStreams: Dispatch error sending response", "syncId", s.syncId, "err", err) - s.setErrorAndCancel(err) - return - } - } else { - log.Debug("SyncStreams: Dispatch data channel closed", "syncId", s.syncId) - } - case control := <-s.controlChannel: - log.Debug("SyncStreams: Dispatch received control message", "syncId", s.syncId, "control", control) - if control.getOp() == SyncOp_SYNC_CLOSE { - err := res.Send(&SyncStreamsResponse{ - SyncId: s.syncId, - SyncOp: SyncOp_SYNC_CLOSE, - }) - if err != nil { - log.Warn( - "SyncStreams: Dispatch error sending close response", - "syncId", - s.syncId, - "err", - err, - ) - log.Warn("SyncStreams: error closing stream", "err", err) - } - s.cancel() - log.Debug("SyncStreams: closed stream", "syncId", s.syncId) - } else if control.getOp() == SyncOp_SYNC_PONG { - log.Debug("SyncStreams: send pong to client", "syncId", s.syncId) - data := control.(*pingOp) - err := res.Send(&SyncStreamsResponse{ - SyncId: s.syncId, - SyncOp: SyncOp_SYNC_PONG, - PongNonce: data.nonce, - }) - if err != nil { - log.Warn("SyncStreams: cancel stream because of error sending pong response", "syncId", s.syncId, "err", err) - s.cancel() - } - } else { - log.Warn("SyncStreams: Dispatch received unknown control message", "syncId", s.syncId, "control", control) - } - } - } -} - -func (s *syncSubscriptionImpl) getError() error { - s.mu.Lock() - defer s.mu.Unlock() - return s.firstError -} diff --git a/core/node/rpc/tester_test.go b/core/node/rpc/tester_test.go index 2deb294e66..b6e362fe7c 100644 --- a/core/node/rpc/tester_test.go +++ b/core/node/rpc/tester_test.go @@ -34,6 +34,7 @@ type testNodeRecord struct { url string service *Service address common.Address + cancel context.CancelFunc } func (n *testNodeRecord) Close(ctx context.Context, dbUrl string) { @@ -249,15 +250,20 @@ func (st *serviceTester) startSingle(i int, opts ...startOpts) error { listener = options.listeners[i] } - bc := st.btc.GetBlockchain(st.ctx, i) - service, err := StartServer(st.ctx, cfg, bc, listener) + nodeCtx, nodeCancel := context.WithCancel(st.ctx) + + bc := st.btc.GetBlockchain(nodeCtx, i) + service, err := StartServer(nodeCtx, cfg, bc, listener) if err != nil { if service != nil { // Sanity check panic("service should be nil") } + nodeCancel() return err } + + st.nodes[i].cancel = nodeCancel st.nodes[i].service = service st.nodes[i].address = bc.Wallet.Address @@ -268,6 +274,10 @@ func (st *serviceTester) startSingle(i int, opts ...startOpts) error { return nil } +func (st *serviceTester) stopSingle(i int) { + st.nodes[i].cancel() +} + func (st *serviceTester) testClient(i int) protocolconnect.StreamServiceClient { return testClient(st.nodes[i].url) } diff --git a/packages/proto/src/gen/protocol_pb.ts b/packages/proto/src/gen/protocol_pb.ts index f5769beab9..46b464c71f 100644 --- a/packages/proto/src/gen/protocol_pb.ts +++ b/packages/proto/src/gen/protocol_pb.ts @@ -42,6 +42,13 @@ export enum SyncOp { * @generated from enum value: SYNC_PONG = 4; */ SYNC_PONG = 4, + + /** + * indication that stream updates could (temporarily) not be provided + * + * @generated from enum value: SYNC_DOWN = 5; + */ + SYNC_DOWN = 5, } // Retrieve enum metadata with: proto3.getEnumType(SyncOp) proto3.util.setEnumType(SyncOp, "river.SyncOp", [ @@ -50,6 +57,7 @@ proto3.util.setEnumType(SyncOp, "river.SyncOp", [ { no: 2, name: "SYNC_CLOSE" }, { no: 3, name: "SYNC_UPDATE" }, { no: 4, name: "SYNC_PONG" }, + { no: 5, name: "SYNC_DOWN" }, ]); /** @@ -4677,6 +4685,11 @@ export class SyncStreamsResponse extends Message { */ pongNonce = ""; + /** + * @generated from field: bytes stream_id = 5; + */ + streamId = new Uint8Array(0); + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -4689,6 +4702,7 @@ export class SyncStreamsResponse extends Message { { no: 2, name: "sync_op", kind: "enum", T: proto3.getEnumType(SyncOp) }, { no: 3, name: "stream", kind: "message", T: StreamAndCookie }, { no: 4, name: "pong_nonce", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 5, name: "stream_id", kind: "scalar", T: 12 /* ScalarType.BYTES */ }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): SyncStreamsResponse { diff --git a/protocol/protocol.proto b/protocol/protocol.proto index e7a35b7a5f..be6009945b 100644 --- a/protocol/protocol.proto +++ b/protocol/protocol.proto @@ -694,6 +694,7 @@ message SyncStreamsResponse { SyncOp sync_op = 2; StreamAndCookie stream = 3; string pong_nonce = 4; + bytes stream_id = 5; } message AddStreamToSyncRequest { @@ -754,6 +755,7 @@ enum SyncOp { SYNC_CLOSE = 2; // close the sync SYNC_UPDATE = 3; // update from server SYNC_PONG = 4; // respond to the ping message from the client. + SYNC_DOWN = 5; // indication that stream updates could (temporarily) not be provided } enum MembershipOp {