From b3f45bc2c6283e3c0a7df4d21b7778eb4ed149af Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 5 Jun 2024 16:00:18 -0700 Subject: [PATCH 01/34] wip --- cli/start.go | 1 - client/db.go | 2 +- events/db_update.go => client/events.go | 29 ++---- client/mocks/db.go | 14 +-- events/bus.go | 103 +++++++++++++++++++ events/errors.go | 19 ---- events/events.go | 52 ---------- events/publisher.go | 68 ------------ events/publisher_test.go | 81 --------------- events/simple.go | 131 ------------------------ events/simple_test.go | 110 -------------------- events/subscription.go | 18 ++++ http/client.go | 2 +- http/handler_ccip_test.go | 2 +- internal/db/collection.go | 26 ++--- internal/db/collection_delete.go | 23 ++--- internal/db/config.go | 11 -- internal/db/config_test.go | 6 -- internal/db/db.go | 8 +- internal/db/subscriptions.go | 26 ++--- net/client.go | 4 +- net/client_test.go | 7 +- net/node_test.go | 20 ++-- net/peer.go | 33 +++--- net/peer_test.go | 33 +++--- tests/clients/cli/wrapper.go | 2 +- tests/clients/http/wrapper.go | 2 +- tests/integration/db.go | 3 - tests/integration/events/utils.go | 9 +- 29 files changed, 228 insertions(+), 617 deletions(-) rename events/db_update.go => client/events.go (50%) create mode 100644 events/bus.go delete mode 100644 events/errors.go delete mode 100644 events/events.go delete mode 100644 events/publisher.go delete mode 100644 events/publisher_test.go delete mode 100644 events/simple.go delete mode 100644 events/simple_test.go create mode 100644 events/subscription.go diff --git a/cli/start.go b/cli/start.go index 9505fd7fff..20f7bc1200 100644 --- a/cli/start.go +++ b/cli/start.go @@ -115,7 +115,6 @@ func MakeStartCommand() *cobra.Command { node.WithACPType(node.LocalACPType), node.WithPeers(peers...), // db options - db.WithUpdateEvents(), db.WithMaxRetries(cfg.GetInt("datastore.MaxTxnRetries")), // net node options net.WithListenAddresses(cfg.GetStringSlice("net.p2pAddresses")...), diff --git a/client/db.go b/client/db.go index 6ab945a815..d83a0fc2ac 100644 --- a/client/db.go +++ b/client/db.go @@ -75,7 +75,7 @@ type DB interface { // // It may be used to monitor database events - a new event will be yielded for each mutation. // Note: it does not copy the queue, just the reference to it. - Events() events.Events + Events() *events.Bus // MaxTxnRetries returns the number of retries that this DefraDB instance has been configured to // make in the event of a transaction conflict in certain scenarios. diff --git a/events/db_update.go b/client/events.go similarity index 50% rename from events/db_update.go rename to client/events.go index 2b93752573..f3266a87a5 100644 --- a/events/db_update.go +++ b/client/events.go @@ -1,32 +1,19 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package events +package client -import ( - "github.com/ipfs/go-cid" +import "github.com/ipfs/go-cid" - "github.com/sourcenetwork/immutable" +const ( + // UpdateEventName is the name of the database update event. + UpdateEventName = "db:update" + // ResultsEventName is the name of the database results event. + ResultsEventName = "db:results" ) -// UpdateChannel is the bus onto which updates are published. -type UpdateChannel = immutable.Option[Channel[Update]] - -// EmptyUpdateChannel is an empty UpdateChannel. -var EmptyUpdateChannel = immutable.None[Channel[Update]]() - // UpdateEvent represents a new DAG node added to the append-only composite MerkleCRDT Clock graph // of a document. // // It must only contain public elements not protected by ACP. -type Update struct { +type UpdateEvent struct { // DocID is the unique immutable identifier of the document that was updated. DocID string diff --git a/client/mocks/db.go b/client/mocks/db.go index 4f0320f0c4..5652361118 100644 --- a/client/mocks/db.go +++ b/client/mocks/db.go @@ -360,14 +360,16 @@ func (_c *DB_Close_Call) RunAndReturn(run func()) *DB_Close_Call { } // Events provides a mock function with given fields: -func (_m *DB) Events() events.Events { +func (_m *DB) Events() *events.Bus { ret := _m.Called() - var r0 events.Events - if rf, ok := ret.Get(0).(func() events.Events); ok { + var r0 *events.Bus + if rf, ok := ret.Get(0).(func() *events.Bus); ok { r0 = rf() } else { - r0 = ret.Get(0).(events.Events) + if ret.Get(0) != nil { + r0 = ret.Get(0).(*events.Bus) + } } return r0 @@ -390,12 +392,12 @@ func (_c *DB_Events_Call) Run(run func()) *DB_Events_Call { return _c } -func (_c *DB_Events_Call) Return(_a0 events.Events) *DB_Events_Call { +func (_c *DB_Events_Call) Return(_a0 *events.Bus) *DB_Events_Call { _c.Call.Return(_a0) return _c } -func (_c *DB_Events_Call) RunAndReturn(run func() events.Events) *DB_Events_Call { +func (_c *DB_Events_Call) RunAndReturn(run func() *events.Bus) *DB_Events_Call { _c.Call.Return(run) return _c } diff --git a/events/bus.go b/events/bus.go new file mode 100644 index 0000000000..8fe226f806 --- /dev/null +++ b/events/bus.go @@ -0,0 +1,103 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package events + +import ( + "sync" +) + +// Bus is an event bus used to publish and subscribe to internal +// subsystem messages. +type Bus struct { + subId int + subs map[int]*Subscription + events map[string]map[int]any + mutex sync.RWMutex +} + +// NewBus returns a new event bus. +func NewBus() *Bus { + return &Bus{ + subs: make(map[int]*Subscription), + events: make(map[string]map[int]any), + } +} + +// Publish publishes the given event to all subscribers. +// +// This method will never block if the subscribers buffer is full. +func (b *Bus) Publish(event string, value any) { + b.mutex.RLock() + defer b.mutex.RUnlock() + + for id := range b.events[event] { + select { + case b.subs[id].value <- value: + // published event + default: + // channel full + } + } +} + +// Subscribe returns a new channel that will receive all of the subscribed events. +// +// The size of the buffer should be appropriate for the consumer or events will be dropped. +func (b *Bus) Subscribe(size int, events ...string) *Subscription { + b.mutex.Lock() + defer b.mutex.Unlock() + + sub := &Subscription{ + id: b.subId, + value: make(chan any, size), + events: events, + } + + b.subId++ + b.subs[sub.id] = sub + + // add sub to all events + for _, event := range events { + b.events[event][sub.id] = struct{}{} + } + return sub +} + +// Unsubscribe unsubscribes from all events and closes the event channel of the given subscription. +func (b *Bus) Unsubscribe(sub *Subscription) { + b.mutex.Lock() + defer b.mutex.Unlock() + + // delete sub from all events + for _, event := range sub.events { + delete(b.events[event], sub.id) + } + + delete(b.subs, sub.id) + close(sub.value) +} + +// Close closes the event bus by unsubscribing all subscribers. +func (b *Bus) Close() { + var subs []*Subscription + + // get list of all subs + b.mutex.RLock() + for _, sub := range b.subs { + subs = append(subs, sub) + } + b.mutex.RUnlock() + + // unsubscribe all subs + for _, sub := range subs { + b.Unsubscribe(sub) + } +} diff --git a/events/errors.go b/events/errors.go deleted file mode 100644 index bc0a06808d..0000000000 --- a/events/errors.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package events - -import ( - "github.com/sourcenetwork/defradb/errors" -) - -var ( - ErrSubscribedToClosedChan = errors.New("cannot subscribe to a closed channel") -) diff --git a/events/events.go b/events/events.go deleted file mode 100644 index 4f910ab454..0000000000 --- a/events/events.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -/* -Package events provides the internal event system. -*/ -package events - -type Subscription[T any] chan T - -// Channel represents a subscribable type that will expose inputted items to subscribers. -type Channel[T any] interface { - // Subscribe subscribes to the Channel, returning a channel by which events can - // be read from, or an error should one occur (e.g. if this object is closed). - // - // This function is non-blocking unless the subscription-buffer is full. - Subscribe() (Subscription[T], error) - - // Unsubscribe unsubscribes from the Channel, closing the provided channel. - // - // Will do nothing if this object is already closed. - Unsubscribe(Subscription[T]) - - // Publish pushes the given item into this channel. Non-blocking. - Publish(item T) - - // Close closes this Channel, and any owned or subscribing channels. - Close() -} - -var _ Channel[int] = (*simpleChannel[int])(nil) - -// New creates and returns a new Channel instance. -// -// At the moment this will always return a new simpleChannel, however that may change in -// the future as this feature gets fleshed out. -func New[T any](commandBufferSize int, eventBufferSize int) Channel[T] { - return NewSimpleChannel[T](commandBufferSize, eventBufferSize) -} - -// Events hold the supported event types -type Events struct { - // Updates publishes an `Update` for each document written to in the database. - Updates UpdateChannel -} diff --git a/events/publisher.go b/events/publisher.go deleted file mode 100644 index 2d2d93db60..0000000000 --- a/events/publisher.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package events - -import "time" - -// time limit we set for the client to read after publishing. -var clientTimeout = 60 * time.Second - -// Publisher hold a referance to the event channel, -// the associated subscription channel and the stream channel that -// returns data to the subscribed client -type Publisher[T any] struct { - ch Channel[T] - event Subscription[T] - stream chan any -} - -// NewPublisher creates a new Publisher with the given event Channel, subscribes to the -// event Channel and opens a new channel for the stream. -func NewPublisher[T any](ch Channel[T], streamBufferSize int) (*Publisher[T], error) { - evtCh, err := ch.Subscribe() - if err != nil { - return nil, err - } - - return &Publisher[T]{ - ch: ch, - event: evtCh, - stream: make(chan any, streamBufferSize), - }, nil -} - -// Event returns the subscription channel -func (p *Publisher[T]) Event() Subscription[T] { - return p.event -} - -// Stream returns the streaming channel -func (p *Publisher[T]) Stream() chan any { - return p.stream -} - -// Publish sends data to the streaming channel and unsubscribes if -// the client hangs for too long. -func (p *Publisher[T]) Publish(data any) { - select { - case p.stream <- data: - case <-time.After(clientTimeout): - // if sending to the client times out, we assume an inactive or problematic client and - // unsubscribe them from the event stream - p.Unsubscribe() - } -} - -// Unsubscribe unsubscribes the client for the event channel and closes the stream. -func (p *Publisher[T]) Unsubscribe() { - p.ch.Unsubscribe(p.event) - close(p.stream) -} diff --git a/events/publisher_test.go b/events/publisher_test.go deleted file mode 100644 index 97ff7b6255..0000000000 --- a/events/publisher_test.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package events - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestNewPublisher(t *testing.T) { - ch := startEventChanel() - - pub, err := NewPublisher(ch, 0) - if err != nil { - t.Fatal(err) - } - assert.NotNil(t, pub) -} - -func TestNewPublisherWithError(t *testing.T) { - ch := startEventChanel() - ch.Close() - _, err := NewPublisher(ch, 0) - assert.Error(t, err) -} - -func TestPublisherToStream(t *testing.T) { - ch := startEventChanel() - - pub, err := NewPublisher(ch, 1) - if err != nil { - t.Fatal(err) - } - assert.NotNil(t, pub) - - ch.Publish(10) - evt := <-pub.Event() - assert.Equal(t, 10, evt) - - pub.Publish(evt) - assert.Equal(t, 10, <-pub.Stream()) - - pub.Unsubscribe() - - _, open := <-pub.Stream() - assert.Equal(t, false, open) -} - -func TestPublisherToStreamWithTimeout(t *testing.T) { - clientTimeout = 1 * time.Second - ch := startEventChanel() - - pub, err := NewPublisher(ch, 0) - if err != nil { - t.Fatal(err) - } - assert.NotNil(t, pub) - - ch.Publish(10) - evt := <-pub.Event() - assert.Equal(t, 10, evt) - - pub.Publish(evt) - - _, open := <-pub.Stream() - assert.Equal(t, false, open) -} - -func startEventChanel() Channel[int] { - return New[int](0, 0) -} diff --git a/events/simple.go b/events/simple.go deleted file mode 100644 index bf247a7a16..0000000000 --- a/events/simple.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package events - -type simpleChannel[T any] struct { - subscribers []chan T - // commandChannel manages all commands sent to this simpleChannel. - // - // It is important that all stuff gets sent through this single channel to ensure - // that the order of operations is preserved. - // - // WARNING: This does mean that non-event commands can block the database if the buffer - // size is breached (e.g. if many subscribe commands occupy the buffer). - commandChannel chan any - eventBufferSize int - hasClosedChan chan struct{} - isClosed bool -} - -type subscribeCommand[T any] Subscription[T] - -type unsubscribeCommand[T any] Subscription[T] - -type publishCommand[T any] struct { - item T -} - -type closeCommand struct{} - -// NewSimpleChannel creates a new simpleChannel with the given commandBufferSize and -// eventBufferSize. -// -// Should the buffers be filled subsequent calls to functions on this object may start to block. -func NewSimpleChannel[T any](commandBufferSize int, eventBufferSize int) Channel[T] { - c := simpleChannel[T]{ - commandChannel: make(chan any, commandBufferSize), - hasClosedChan: make(chan struct{}), - eventBufferSize: eventBufferSize, - } - - go c.handleChannel() - - return &c -} - -func (c *simpleChannel[T]) Subscribe() (Subscription[T], error) { - if c.isClosed { - return nil, ErrSubscribedToClosedChan - } - - // It is important to set this buffer size too, else we may end up blocked in the handleChannel func - ch := make(chan T, c.eventBufferSize) - - c.commandChannel <- subscribeCommand[T](ch) - return ch, nil -} - -func (c *simpleChannel[T]) Unsubscribe(ch Subscription[T]) { - if c.isClosed { - return - } - c.commandChannel <- unsubscribeCommand[T](ch) -} - -func (c *simpleChannel[T]) Publish(item T) { - if c.isClosed { - return - } - c.commandChannel <- publishCommand[T]{item} -} - -func (c *simpleChannel[T]) Close() { - if c.isClosed { - return - } - c.isClosed = true - c.commandChannel <- closeCommand{} - - // Wait for the close command to be handled, in order, before returning - <-c.hasClosedChan -} - -func (c *simpleChannel[T]) handleChannel() { - for cmd := range c.commandChannel { - switch command := cmd.(type) { - case closeCommand: - for _, subscriber := range c.subscribers { - close(subscriber) - } - close(c.commandChannel) - close(c.hasClosedChan) - return - - case subscribeCommand[T]: - c.subscribers = append(c.subscribers, command) - - case unsubscribeCommand[T]: - var isFound bool - var index int - for i, subscriber := range c.subscribers { - if command == subscriber { - index = i - isFound = true - break - } - } - if !isFound { - continue - } - - // Remove channel from list of subscribers - c.subscribers[index] = c.subscribers[len(c.subscribers)-1] - c.subscribers = c.subscribers[:len(c.subscribers)-1] - - close(command) - - case publishCommand[T]: - for _, subscriber := range c.subscribers { - subscriber <- command.item - } - } - } -} diff --git a/events/simple_test.go b/events/simple_test.go deleted file mode 100644 index d4cc91047b..0000000000 --- a/events/simple_test.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package events - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestSimplePushIsNotBlockedWithoutSubscribers(t *testing.T) { - s := NewSimpleChannel[int](0, 0) - - s.Publish(1) - - // just assert that we reach this line, for the sake of having an assert - assert.True(t, true) -} - -func TestSimpleSubscribersAreNotBlockedAfterClose(t *testing.T) { - s := NewSimpleChannel[int](0, 0) - ch, err := s.Subscribe() - assert.Nil(t, err) - - s.Close() - - <-ch - - // just assert that we reach this line, for the sake of having an assert - assert.True(t, true) -} - -func TestSimpleEachSubscribersRecievesEachItem(t *testing.T) { - s := NewSimpleChannel[int](0, 0) - input1 := 1 - input2 := 2 - - ch1, err := s.Subscribe() - assert.Nil(t, err) - ch2, err := s.Subscribe() - assert.Nil(t, err) - - s.Publish(input1) - - output1Ch1 := <-ch1 - output1Ch2 := <-ch2 - - s.Publish(input2) - - output2Ch1 := <-ch1 - output2Ch2 := <-ch2 - - assert.Equal(t, input1, output1Ch1) - assert.Equal(t, input1, output1Ch2) - - assert.Equal(t, input2, output2Ch1) - assert.Equal(t, input2, output2Ch2) -} - -func TestSimpleEachSubscribersRecievesEachItemGivenBufferedEventChan(t *testing.T) { - s := NewSimpleChannel[int](0, 2) - input1 := 1 - input2 := 2 - - ch1, err := s.Subscribe() - assert.Nil(t, err) - ch2, err := s.Subscribe() - assert.Nil(t, err) - - // both inputs are added first before read, using the internal chan buffer - s.Publish(input1) - s.Publish(input2) - - output1Ch1 := <-ch1 - output1Ch2 := <-ch2 - - output2Ch1 := <-ch1 - output2Ch2 := <-ch2 - - assert.Equal(t, input1, output1Ch1) - assert.Equal(t, input1, output1Ch2) - - assert.Equal(t, input2, output2Ch1) - assert.Equal(t, input2, output2Ch2) -} - -func TestSimpleSubscribersDontRecieveItemsAfterUnsubscribing(t *testing.T) { - s := NewSimpleChannel[int](0, 0) - ch, err := s.Subscribe() - assert.Nil(t, err) - s.Unsubscribe(ch) - - s.Publish(1) - - // tiny delay to try and make sure the internal logic would have had time - // to do its thing with the pushed item. - time.Sleep(5 * time.Millisecond) - - // closing the channel will result in reads yielding the default value - assert.Equal(t, 0, <-ch) -} diff --git a/events/subscription.go b/events/subscription.go new file mode 100644 index 0000000000..97365533a3 --- /dev/null +++ b/events/subscription.go @@ -0,0 +1,18 @@ +package events + +// Subscription is a read-only event stream. +type Subscription struct { + id int + value chan any + events []string +} + +// Value returns the next event value from the subscription. +func (s *Subscription) Value() <-chan any { + return s.value +} + +// Events returns the names of all subscribed events. +func (s *Subscription) Events() []string { + return s.events +} diff --git a/http/client.go b/http/client.go index 9792208214..89f1578bb9 100644 --- a/http/client.go +++ b/http/client.go @@ -451,7 +451,7 @@ func (c *Client) Headstore() ds.Read { panic("client side database") } -func (c *Client) Events() events.Events { +func (c *Client) Events() *events.Bus { panic("client side database") } diff --git a/http/handler_ccip_test.go b/http/handler_ccip_test.go index e17d8a882a..d35a910161 100644 --- a/http/handler_ccip_test.go +++ b/http/handler_ccip_test.go @@ -193,7 +193,7 @@ func TestCCIPPost_WithInvalidBody(t *testing.T) { func setupDatabase(t *testing.T) client.DB { ctx := context.Background() - cdb, err := db.NewDB(ctx, memory.NewDatastore(ctx), acp.NoACP, nil, db.WithUpdateEvents()) + cdb, err := db.NewDB(ctx, memory.NewDatastore(ctx), acp.NoACP, nil) require.NoError(t, err) _, err = cdb.AddSchema(ctx, `type User { diff --git a/internal/db/collection.go b/internal/db/collection.go index 4b9c988288..030a5b1206 100644 --- a/internal/db/collection.go +++ b/internal/db/collection.go @@ -31,7 +31,6 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/client/request" "github.com/sourcenetwork/defradb/errors" - "github.com/sourcenetwork/defradb/events" "github.com/sourcenetwork/defradb/internal/core" coreblock "github.com/sourcenetwork/defradb/internal/core/block" "github.com/sourcenetwork/defradb/internal/db/base" @@ -1792,21 +1791,18 @@ func (c *collection) save( if err != nil { return cid.Undef, err } - if c.db.events.Updates.HasValue() { - txn.OnSuccess( - func() { - c.db.events.Updates.Value().Publish( - events.Update{ - DocID: doc.ID().String(), - Cid: link.Cid, - SchemaRoot: c.Schema().Root, - Block: headNode, - IsCreate: isCreate, - }, - ) - }, - ) + + // publish an update event when the txn succeeds + updateEvent := client.UpdateEvent{ + DocID: doc.ID().String(), + Cid: link.Cid, + SchemaRoot: c.Schema().Root, + Block: headNode, + IsCreate: isCreate, } + txn.OnSuccess(func() { + c.db.events.Publish(client.UpdateEventName, updateEvent) + }) txn.OnSuccess(func() { doc.SetHead(link.Cid) diff --git a/internal/db/collection_delete.go b/internal/db/collection_delete.go index 9612c4f42c..f7f2efdcd3 100644 --- a/internal/db/collection_delete.go +++ b/internal/db/collection_delete.go @@ -17,7 +17,6 @@ import ( "github.com/sourcenetwork/defradb/acp" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/events" "github.com/sourcenetwork/defradb/internal/core" coreblock "github.com/sourcenetwork/defradb/internal/core/block" "github.com/sourcenetwork/defradb/internal/merkle/clock" @@ -166,20 +165,16 @@ func (c *collection) applyDelete( return err } - if c.db.events.Updates.HasValue() { - txn.OnSuccess( - func() { - c.db.events.Updates.Value().Publish( - events.Update{ - DocID: primaryKey.DocID, - Cid: link.Cid, - SchemaRoot: c.Schema().Root, - Block: b, - }, - ) - }, - ) + // publish an update event if the txn succeeds + updateEvent := client.UpdateEvent{ + DocID: primaryKey.DocID, + Cid: link.Cid, + SchemaRoot: c.Schema().Root, + Block: b, } + txn.OnSuccess(func() { + c.db.events.Publish(client.UpdateEventName, updateEvent) + }) return nil } diff --git a/internal/db/config.go b/internal/db/config.go index 1364cab09b..8ce725ebd0 100644 --- a/internal/db/config.go +++ b/internal/db/config.go @@ -12,8 +12,6 @@ package db import ( "github.com/sourcenetwork/immutable" - - "github.com/sourcenetwork/defradb/events" ) const ( @@ -24,15 +22,6 @@ const ( // Option is a funtion that sets a config value on the db. type Option func(*db) -// WithUpdateEvents enables the update events channel. -func WithUpdateEvents() Option { - return func(db *db) { - db.events = events.Events{ - Updates: immutable.Some(events.New[events.Update](0, updateEventBufferSize)), - } - } -} - // WithMaxRetries sets the maximum number of retries per transaction. func WithMaxRetries(num int) Option { return func(db *db) { diff --git a/internal/db/config_test.go b/internal/db/config_test.go index f80e538b4f..405e192598 100644 --- a/internal/db/config_test.go +++ b/internal/db/config_test.go @@ -16,12 +16,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestWithUpdateEvents(t *testing.T) { - d := &db{} - WithUpdateEvents()(d) - assert.NotNil(t, d.events) -} - func TestWithMaxRetries(t *testing.T) { d := &db{} WithMaxRetries(10)(d) diff --git a/internal/db/db.go b/internal/db/db.go index 979626034c..7de64e100c 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -51,7 +51,7 @@ type db struct { rootstore datastore.RootStore multistore datastore.MultiStore - events events.Events + events *events.Bus parser core.Parser @@ -237,7 +237,7 @@ func (db *db) initialize(ctx context.Context) error { } // Events returns the events Channel. -func (db *db) Events() events.Events { +func (db *db) Events() *events.Bus { return db.events } @@ -259,9 +259,7 @@ func (db *db) PrintDump(ctx context.Context) error { // This is the place for any last minute cleanup or releasing of resources (i.e.: Badger instance). func (db *db) Close() { log.Info("Closing DefraDB process...") - if db.events.Updates.HasValue() { - db.events.Updates.Value().Close() - } + db.events.Close() err := db.rootstore.Close() if err != nil { diff --git a/internal/db/subscriptions.go b/internal/db/subscriptions.go index a1b0147df4..4d2ca64aa7 100644 --- a/internal/db/subscriptions.go +++ b/internal/db/subscriptions.go @@ -15,7 +15,6 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/client/request" - "github.com/sourcenetwork/defradb/events" "github.com/sourcenetwork/defradb/internal/planner" ) @@ -26,36 +25,33 @@ func (db *db) handleSubscription(ctx context.Context, r *request.Request) (<-cha if len(r.Subscription) == 0 || len(r.Subscription[0].Selections) == 0 { return nil, nil // This is not a subscription request and we have nothing to do here } - if !db.events.Updates.HasValue() { - return nil, ErrSubscriptionsNotAllowed - } selections := r.Subscription[0].Selections[0] subRequest, ok := selections.(*request.ObjectSubscription) if !ok { return nil, client.NewErrUnexpectedType[request.ObjectSubscription]("SubscriptionSelection", selections) } - // unsubscribing from this publisher will cause a race condition - // https://github.com/sourcenetwork/defradb/issues/2687 - pub, err := events.NewPublisher(db.events.Updates.Value(), 5) - if err != nil { - return nil, err - } - + sub := db.events.Subscribe(5, client.UpdateEventName) resCh := make(chan client.GQLResult) go func() { - defer close(resCh) + defer func() { + db.events.Unsubscribe(sub) + close(resCh) + }() // listen for events and send to the result channel for { - var evt events.Update + var evt client.UpdateEvent select { case <-ctx.Done(): return // context cancelled - case val, ok := <-pub.Event(): + case val, ok := <-sub.Value(): if !ok { return // channel closed } - evt = val + evt, ok = val.(client.UpdateEvent) + if !ok { + continue // invalid event value + } } txn, err := db.NewTxn(ctx, false) diff --git a/net/client.go b/net/client.go index d29af6f60d..25ea88b362 100644 --- a/net/client.go +++ b/net/client.go @@ -18,8 +18,8 @@ import ( "github.com/libp2p/go-libp2p/core/peer" + "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/errors" - "github.com/sourcenetwork/defradb/events" pb "github.com/sourcenetwork/defradb/net/pb" ) @@ -31,7 +31,7 @@ var ( // pushLog creates a pushLog request and sends it to another node // over libp2p grpc connection -func (s *server) pushLog(ctx context.Context, evt events.Update, pid peer.ID) error { +func (s *server) pushLog(ctx context.Context, evt client.UpdateEvent, pid peer.ID) error { body := &pb.PushLogRequest_Body{ DocID: []byte(evt.DocID), Cid: evt.Cid.Bytes(), diff --git a/net/client_test.go b/net/client_test.go index e074947213..b964e03971 100644 --- a/net/client_test.go +++ b/net/client_test.go @@ -19,7 +19,6 @@ import ( "google.golang.org/grpc" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/events" ) var def = client.CollectionDefinition{ @@ -62,7 +61,7 @@ func TestPushlogWithDialFailure(t *testing.T) { grpc.WithCredentialsBundle(nil), ) - err = n.server.pushLog(ctx, events.Update{ + err = n.server.pushLog(ctx, client.UpdateEvent{ DocID: id.String(), Cid: cid, SchemaRoot: "test", @@ -84,7 +83,7 @@ func TestPushlogWithInvalidPeerID(t *testing.T) { cid, err := createCID(doc) require.NoError(t, err) - err = n.server.pushLog(ctx, events.Update{ + err = n.server.pushLog(ctx, client.UpdateEvent{ DocID: id.String(), Cid: cid, SchemaRoot: "test", @@ -132,7 +131,7 @@ func TestPushlogW_WithValidPeerID_NoError(t *testing.T) { cid, err := createCID(doc) require.NoError(t, err) - err = n1.server.pushLog(ctx, events.Update{ + err = n1.server.pushLog(ctx, client.UpdateEvent{ DocID: doc.ID().String(), Cid: cid, SchemaRoot: col.SchemaRoot(), diff --git a/net/node_test.go b/net/node_test.go index 55b0573474..5f27d1b5e7 100644 --- a/net/node_test.go +++ b/net/node_test.go @@ -36,7 +36,7 @@ func FixtureNewMemoryDBWithBroadcaster(t *testing.T) client.DB { opts := badgerds.Options{Options: badger.DefaultOptions("").WithInMemory(true)} rootstore, err := badgerds.NewDatastore("", &opts) require.NoError(t, err) - database, err = db.NewDB(ctx, rootstore, acp.NoACP, nil, db.WithUpdateEvents()) + database, err = db.NewDB(ctx, rootstore, acp.NoACP, nil) require.NoError(t, err) return database } @@ -44,7 +44,7 @@ func FixtureNewMemoryDBWithBroadcaster(t *testing.T) client.DB { func TestNewNode_WithEnableRelay_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil) require.NoError(t, err) n, err := NewNode( context.Background(), @@ -59,7 +59,7 @@ func TestNewNode_WithDBClosed_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil) require.NoError(t, err) db.Close() @@ -73,7 +73,7 @@ func TestNewNode_WithDBClosed_NoError(t *testing.T) { func TestNewNode_NoPubSub_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil) require.NoError(t, err) n, err := NewNode( context.Background(), @@ -88,7 +88,7 @@ func TestNewNode_NoPubSub_NoError(t *testing.T) { func TestNewNode_WithEnablePubSub_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil) require.NoError(t, err) n, err := NewNode( @@ -106,7 +106,7 @@ func TestNewNode_WithEnablePubSub_NoError(t *testing.T) { func TestNodeClose_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil) require.NoError(t, err) n, err := NewNode( context.Background(), @@ -119,7 +119,7 @@ func TestNodeClose_NoError(t *testing.T) { func TestNewNode_BootstrapWithNoPeer_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil) require.NoError(t, err) n1, err := NewNode( @@ -135,7 +135,7 @@ func TestNewNode_BootstrapWithNoPeer_NoError(t *testing.T) { func TestNewNode_BootstrapWithOnePeer_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil) require.NoError(t, err) n1, err := NewNode( @@ -162,7 +162,7 @@ func TestNewNode_BootstrapWithOnePeer_NoError(t *testing.T) { func TestNewNode_BootstrapWithOneValidPeerAndManyInvalidPeers_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil) require.NoError(t, err) n1, err := NewNode( @@ -192,7 +192,7 @@ func TestNewNode_BootstrapWithOneValidPeerAndManyInvalidPeers_NoError(t *testing func TestListenAddrs_WithListenAddresses_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil) require.NoError(t, err) n, err := NewNode( context.Background(), diff --git a/net/peer.go b/net/peer.go index 3d728a1d87..fc946effb3 100644 --- a/net/peer.go +++ b/net/peer.go @@ -53,8 +53,8 @@ var ( type Peer struct { //config?? - db client.DB - updateChannel chan events.Update + db client.DB + updateSub *events.Subscription host host.Host dht routing.Routing @@ -163,16 +163,7 @@ func (p *Peer) Start() error { } if p.ps != nil { - if !p.db.Events().Updates.HasValue() { - return ErrNilUpdateChannel - } - - updateChannel, err := p.db.Events().Updates.Value().Subscribe() - if err != nil { - return err - } - p.updateChannel = updateChannel - + p.updateSub = p.db.Events().Subscribe(100, client.UpdateEventName) log.InfoContext(p.ctx, "Starting internal broadcaster for pubsub network") go p.handleBroadcastLoop() } @@ -224,8 +215,8 @@ func (p *Peer) Close() { } } - if p.db.Events().Updates.HasValue() { - p.db.Events().Updates.Value().Unsubscribe(p.updateChannel) + if p.updateSub != nil { + p.db.Events().Unsubscribe(p.updateSub) } if err := p.bserv.Close(); err != nil { @@ -243,10 +234,14 @@ func (p *Peer) Close() { // from the internal broadcaster to the external pubsub network func (p *Peer) handleBroadcastLoop() { for { - update, isOpen := <-p.updateChannel + value, isOpen := <-p.updateSub.Value() if !isOpen { return } + update, ok := value.(client.UpdateEvent) + if !ok { + continue // ignore invalid value + } var err error if update.IsCreate { @@ -335,7 +330,7 @@ func (p *Peer) pushToReplicator( continue } - evt := events.Update{ + evt := client.UpdateEvent{ DocID: docIDResult.ID.String(), Cid: c, SchemaRoot: collection.SchemaRoot(), @@ -402,7 +397,7 @@ func (p *Peer) loadP2PCollections(ctx context.Context) (map[string]struct{}, err return colMap, nil } -func (p *Peer) handleDocCreateLog(evt events.Update) error { +func (p *Peer) handleDocCreateLog(evt client.UpdateEvent) error { docID, err := client.NewDocIDFromString(evt.DocID) if err != nil { return NewErrFailedToGetDocID(err) @@ -420,7 +415,7 @@ func (p *Peer) handleDocCreateLog(evt events.Update) error { return nil } -func (p *Peer) handleDocUpdateLog(evt events.Update) error { +func (p *Peer) handleDocUpdateLog(evt client.UpdateEvent) error { docID, err := client.NewDocIDFromString(evt.DocID) if err != nil { return NewErrFailedToGetDocID(err) @@ -453,7 +448,7 @@ func (p *Peer) handleDocUpdateLog(evt events.Update) error { return nil } -func (p *Peer) pushLogToReplicators(lg events.Update) { +func (p *Peer) pushLogToReplicators(lg client.UpdateEvent) { // push to each peer (replicator) peers := make(map[string]struct{}) for _, peer := range p.ps.ListPeers(lg.DocID) { diff --git a/net/peer_test.go b/net/peer_test.go index dca864a1e3..dec53fedc6 100644 --- a/net/peer_test.go +++ b/net/peer_test.go @@ -32,7 +32,6 @@ import ( acpIdentity "github.com/sourcenetwork/defradb/acp/identity" "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/datastore/memory" - "github.com/sourcenetwork/defradb/events" coreblock "github.com/sourcenetwork/defradb/internal/core/block" "github.com/sourcenetwork/defradb/internal/core/crdt" "github.com/sourcenetwork/defradb/internal/db" @@ -75,7 +74,7 @@ func newTestNode(ctx context.Context, t *testing.T) (client.DB, *Node) { store := memory.NewDatastore(ctx) acpLocal := acp.NewLocalACP() acpLocal.Init(context.Background(), "") - db, err := db.NewDB(ctx, store, immutable.Some[acp.ACP](acpLocal), nil, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, immutable.Some[acp.ACP](acpLocal), nil) require.NoError(t, err) n, err := NewNode( @@ -91,7 +90,7 @@ func newTestNode(ctx context.Context, t *testing.T) (client.DB, *Node) { func TestNewPeer_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil) require.NoError(t, err) h, err := libp2p.New() @@ -114,7 +113,7 @@ func TestNewPeer_NoDB_NilDBError(t *testing.T) { func TestNewPeer_WithExistingTopic_TopicAlreadyExistsError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil) require.NoError(t, err) _, err = db.AddSchema(ctx, `type User { @@ -164,11 +163,11 @@ func TestStartAndClose_NoError(t *testing.T) { func TestStart_WithKnownPeer_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db1, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) + db1, err := db.NewDB(ctx, store, acp.NoACP, nil) require.NoError(t, err) store2 := memory.NewDatastore(ctx) - db2, err := db.NewDB(ctx, store2, acp.NoACP, nil, db.WithUpdateEvents()) + db2, err := db.NewDB(ctx, store2, acp.NoACP, nil) require.NoError(t, err) n1, err := NewNode( @@ -200,11 +199,11 @@ func TestStart_WithKnownPeer_NoError(t *testing.T) { func TestStart_WithOfflineKnownPeer_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db1, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) + db1, err := db.NewDB(ctx, store, acp.NoACP, nil) require.NoError(t, err) store2 := memory.NewDatastore(ctx) - db2, err := db.NewDB(ctx, store2, acp.NoACP, nil, db.WithUpdateEvents()) + db2, err := db.NewDB(ctx, store2, acp.NoACP, nil) require.NoError(t, err) n1, err := NewNode( @@ -259,7 +258,7 @@ func TestStart_WithNoUpdateChannel_NilUpdateChannelError(t *testing.T) { func TestStart_WitClosedUpdateChannel_ClosedChannelError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil) require.NoError(t, err) n, err := NewNode( @@ -269,7 +268,7 @@ func TestStart_WitClosedUpdateChannel_ClosedChannelError(t *testing.T) { ) require.NoError(t, err) - db.Events().Updates.Value().Close() + db.Events().Close() err = n.Start() require.ErrorContains(t, err, "cannot subscribe to a closed channel") @@ -928,7 +927,7 @@ func TestHandleDocCreateLog_NoError(t *testing.T) { b, err := db.Blockstore().AsIPLDStorage().Get(ctx, headCID.KeyString()) require.NoError(t, err) - err = n.handleDocCreateLog(events.Update{ + err = n.handleDocCreateLog(client.UpdateEvent{ DocID: doc.ID().String(), Cid: headCID, SchemaRoot: col.SchemaRoot(), @@ -942,7 +941,7 @@ func TestHandleDocCreateLog_WithInvalidDocID_NoError(t *testing.T) { _, n := newTestNode(ctx, t) defer n.Close() - err := n.handleDocCreateLog(events.Update{ + err := n.handleDocCreateLog(client.UpdateEvent{ DocID: "some-invalid-key", }) require.ErrorContains(t, err, "failed to get DocID from broadcast message: selected encoding not supported") @@ -971,7 +970,7 @@ func TestHandleDocCreateLog_WithExistingTopic_TopicExistsError(t *testing.T) { _, err = rpc.NewTopic(ctx, n.ps, n.host.ID(), doc.ID().String(), true) require.NoError(t, err) - err = n.handleDocCreateLog(events.Update{ + err = n.handleDocCreateLog(client.UpdateEvent{ DocID: doc.ID().String(), SchemaRoot: col.SchemaRoot(), }) @@ -1004,7 +1003,7 @@ func TestHandleDocUpdateLog_NoError(t *testing.T) { b, err := db.Blockstore().AsIPLDStorage().Get(ctx, headCID.KeyString()) require.NoError(t, err) - err = n.handleDocUpdateLog(events.Update{ + err = n.handleDocUpdateLog(client.UpdateEvent{ DocID: doc.ID().String(), Cid: headCID, SchemaRoot: col.SchemaRoot(), @@ -1018,7 +1017,7 @@ func TestHandleDoUpdateLog_WithInvalidDocID_NoError(t *testing.T) { _, n := newTestNode(ctx, t) defer n.Close() - err := n.handleDocUpdateLog(events.Update{ + err := n.handleDocUpdateLog(client.UpdateEvent{ DocID: "some-invalid-key", }) require.ErrorContains(t, err, "failed to get DocID from broadcast message: selected encoding not supported") @@ -1053,7 +1052,7 @@ func TestHandleDocUpdateLog_WithExistingDocIDTopic_TopicExistsError(t *testing.T _, err = rpc.NewTopic(ctx, n.ps, n.host.ID(), doc.ID().String(), true) require.NoError(t, err) - err = n.handleDocUpdateLog(events.Update{ + err = n.handleDocUpdateLog(client.UpdateEvent{ DocID: doc.ID().String(), Cid: headCID, SchemaRoot: col.SchemaRoot(), @@ -1091,7 +1090,7 @@ func TestHandleDocUpdateLog_WithExistingSchemaTopic_TopicExistsError(t *testing. _, err = rpc.NewTopic(ctx, n.ps, n.host.ID(), col.SchemaRoot(), true) require.NoError(t, err) - err = n.handleDocUpdateLog(events.Update{ + err = n.handleDocUpdateLog(client.UpdateEvent{ DocID: doc.ID().String(), Cid: headCID, SchemaRoot: col.SchemaRoot(), diff --git a/tests/clients/cli/wrapper.go b/tests/clients/cli/wrapper.go index 25e4c177bf..289d0435ae 100644 --- a/tests/clients/cli/wrapper.go +++ b/tests/clients/cli/wrapper.go @@ -525,7 +525,7 @@ func (w *Wrapper) Close() { w.node.Close() } -func (w *Wrapper) Events() events.Events { +func (w *Wrapper) Events() *events.Bus { return w.node.Events() } diff --git a/tests/clients/http/wrapper.go b/tests/clients/http/wrapper.go index 4727542cce..5c9e76b3ca 100644 --- a/tests/clients/http/wrapper.go +++ b/tests/clients/http/wrapper.go @@ -221,7 +221,7 @@ func (w *Wrapper) Close() { w.node.Close() } -func (w *Wrapper) Events() events.Events { +func (w *Wrapper) Events() *events.Bus { return w.node.Events() } diff --git a/tests/integration/db.go b/tests/integration/db.go index c473e4cdd0..ab15e2d5fc 100644 --- a/tests/integration/db.go +++ b/tests/integration/db.go @@ -19,7 +19,6 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/crypto" - "github.com/sourcenetwork/defradb/internal/db" "github.com/sourcenetwork/defradb/node" changeDetector "github.com/sourcenetwork/defradb/tests/change_detector" ) @@ -73,7 +72,6 @@ func init() { func NewBadgerMemoryDB(ctx context.Context) (client.DB, error) { opts := []node.Option{ node.WithInMemory(true), - db.WithUpdateEvents(), } node, err := node.NewNode(ctx, opts...) @@ -104,7 +102,6 @@ func NewBadgerFileDB(ctx context.Context, t testing.TB) (client.DB, error) { // select the datastore implementation to use. func setupDatabase(s *state) (client.DB, string, error) { opts := []node.Option{ - db.WithUpdateEvents(), node.WithLensPoolSize(lensPoolSize), // The test framework sets this up elsewhere when required so that it may be wrapped // into a [client.DB]. diff --git a/tests/integration/events/utils.go b/tests/integration/events/utils.go index eb514bce2b..b430c18635 100644 --- a/tests/integration/events/utils.go +++ b/tests/integration/events/utils.go @@ -79,14 +79,19 @@ func ExecuteRequestTestCase( testRoutineClosedChan := make(chan struct{}) closeTestRoutineChan := make(chan struct{}) - eventsChan, err := db.Events().Updates.Value().Subscribe() + eventsSub := db.Events().Subscribe(5, client.UpdateEventName) require.NoError(t, err) indexOfNextExpectedUpdate := 0 go func() { for { select { - case update := <-eventsChan: + case value := <-eventsSub.Value(): + update, ok := value.(client.UpdateEvent) + if !ok { + continue // ignore invaid value + } + if indexOfNextExpectedUpdate >= len(testCase.ExpectedUpdates) { assert.Fail(t, "More events recieved than were expected", update) testRoutineClosedChan <- struct{}{} From 10c522b6d144ef588a6c23b66de54e630479bb08 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 5 Jun 2024 16:34:48 -0700 Subject: [PATCH 02/34] move event names and types to event package --- events/bus.go | 18 ++++++++-- {client => events}/events.go | 14 +++++++- events/subscription.go | 10 ++++++ internal/db/collection.go | 5 +-- internal/db/collection_delete.go | 5 +-- internal/db/db.go | 1 + internal/db/subscriptions.go | 7 ++-- net/client.go | 4 +-- net/client_test.go | 7 ++-- net/peer.go | 12 +++---- net/peer_test.go | 55 +++++-------------------------- tests/integration/events/utils.go | 5 +-- 12 files changed, 73 insertions(+), 70 deletions(-) rename {client => events}/events.go (67%) diff --git a/events/bus.go b/events/bus.go index 8fe226f806..be1c12aaf4 100644 --- a/events/bus.go +++ b/events/bus.go @@ -38,7 +38,16 @@ func (b *Bus) Publish(event string, value any) { b.mutex.RLock() defer b.mutex.RUnlock() + subscribers := make(map[int]any) + // publish to event subscribers for id := range b.events[event] { + subscribers[id] = struct{}{} + } + // also publish to wildcard recipients + for id := range b.events[WildCardEventName] { + subscribers[id] = struct{}{} + } + for id := range subscribers { select { case b.subs[id].value <- value: // published event @@ -66,6 +75,9 @@ func (b *Bus) Subscribe(size int, events ...string) *Subscription { // add sub to all events for _, event := range events { + if _, ok := b.events[event]; !ok { + b.events[event] = make(map[int]any) + } b.events[event][sub.id] = struct{}{} } return sub @@ -80,9 +92,11 @@ func (b *Bus) Unsubscribe(sub *Subscription) { for _, event := range sub.events { delete(b.events[event], sub.id) } - + // only close channel once + if _, ok := b.subs[sub.id]; ok { + close(sub.value) + } delete(b.subs, sub.id) - close(sub.value) } // Close closes the event bus by unsubscribing all subscribers. diff --git a/client/events.go b/events/events.go similarity index 67% rename from client/events.go rename to events/events.go index f3266a87a5..12c10ddaf4 100644 --- a/client/events.go +++ b/events/events.go @@ -1,8 +1,20 @@ -package client +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package events import "github.com/ipfs/go-cid" const ( + // WildCardEventName is the alias used to subscribe to all events. + WildCardEventName = "*" // UpdateEventName is the name of the database update event. UpdateEventName = "db:update" // ResultsEventName is the name of the database results event. diff --git a/events/subscription.go b/events/subscription.go index 97365533a3..e29f25da8f 100644 --- a/events/subscription.go +++ b/events/subscription.go @@ -1,3 +1,13 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + package events // Subscription is a read-only event stream. diff --git a/internal/db/collection.go b/internal/db/collection.go index 030a5b1206..0fa10062c5 100644 --- a/internal/db/collection.go +++ b/internal/db/collection.go @@ -31,6 +31,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/client/request" "github.com/sourcenetwork/defradb/errors" + "github.com/sourcenetwork/defradb/events" "github.com/sourcenetwork/defradb/internal/core" coreblock "github.com/sourcenetwork/defradb/internal/core/block" "github.com/sourcenetwork/defradb/internal/db/base" @@ -1793,7 +1794,7 @@ func (c *collection) save( } // publish an update event when the txn succeeds - updateEvent := client.UpdateEvent{ + updateEvent := events.UpdateEvent{ DocID: doc.ID().String(), Cid: link.Cid, SchemaRoot: c.Schema().Root, @@ -1801,7 +1802,7 @@ func (c *collection) save( IsCreate: isCreate, } txn.OnSuccess(func() { - c.db.events.Publish(client.UpdateEventName, updateEvent) + c.db.events.Publish(events.UpdateEventName, updateEvent) }) txn.OnSuccess(func() { diff --git a/internal/db/collection_delete.go b/internal/db/collection_delete.go index f7f2efdcd3..41d9b9bc3e 100644 --- a/internal/db/collection_delete.go +++ b/internal/db/collection_delete.go @@ -17,6 +17,7 @@ import ( "github.com/sourcenetwork/defradb/acp" "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/events" "github.com/sourcenetwork/defradb/internal/core" coreblock "github.com/sourcenetwork/defradb/internal/core/block" "github.com/sourcenetwork/defradb/internal/merkle/clock" @@ -166,14 +167,14 @@ func (c *collection) applyDelete( } // publish an update event if the txn succeeds - updateEvent := client.UpdateEvent{ + updateEvent := events.UpdateEvent{ DocID: primaryKey.DocID, Cid: link.Cid, SchemaRoot: c.Schema().Root, Block: b, } txn.OnSuccess(func() { - c.db.events.Publish(client.UpdateEventName, updateEvent) + c.db.events.Publish(events.UpdateEventName, updateEvent) }) return nil diff --git a/internal/db/db.go b/internal/db/db.go index 7de64e100c..f90fdc79b0 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -102,6 +102,7 @@ func newDB( lensRegistry: lens, parser: parser, options: options, + events: events.NewBus(), } // apply options diff --git a/internal/db/subscriptions.go b/internal/db/subscriptions.go index 4d2ca64aa7..5fffee06a4 100644 --- a/internal/db/subscriptions.go +++ b/internal/db/subscriptions.go @@ -15,6 +15,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/client/request" + "github.com/sourcenetwork/defradb/events" "github.com/sourcenetwork/defradb/internal/planner" ) @@ -30,7 +31,7 @@ func (db *db) handleSubscription(ctx context.Context, r *request.Request) (<-cha if !ok { return nil, client.NewErrUnexpectedType[request.ObjectSubscription]("SubscriptionSelection", selections) } - sub := db.events.Subscribe(5, client.UpdateEventName) + sub := db.events.Subscribe(5, events.UpdateEventName) resCh := make(chan client.GQLResult) go func() { defer func() { @@ -40,7 +41,7 @@ func (db *db) handleSubscription(ctx context.Context, r *request.Request) (<-cha // listen for events and send to the result channel for { - var evt client.UpdateEvent + var evt events.UpdateEvent select { case <-ctx.Done(): return // context cancelled @@ -48,7 +49,7 @@ func (db *db) handleSubscription(ctx context.Context, r *request.Request) (<-cha if !ok { return // channel closed } - evt, ok = val.(client.UpdateEvent) + evt, ok = val.(events.UpdateEvent) if !ok { continue // invalid event value } diff --git a/net/client.go b/net/client.go index 25ea88b362..d774b311da 100644 --- a/net/client.go +++ b/net/client.go @@ -18,8 +18,8 @@ import ( "github.com/libp2p/go-libp2p/core/peer" - "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/errors" + "github.com/sourcenetwork/defradb/events" pb "github.com/sourcenetwork/defradb/net/pb" ) @@ -31,7 +31,7 @@ var ( // pushLog creates a pushLog request and sends it to another node // over libp2p grpc connection -func (s *server) pushLog(ctx context.Context, evt client.UpdateEvent, pid peer.ID) error { +func (s *server) pushLog(ctx context.Context, evt events.UpdateEvent, pid peer.ID) error { body := &pb.PushLogRequest_Body{ DocID: []byte(evt.DocID), Cid: evt.Cid.Bytes(), diff --git a/net/client_test.go b/net/client_test.go index b964e03971..6f68784f5d 100644 --- a/net/client_test.go +++ b/net/client_test.go @@ -19,6 +19,7 @@ import ( "google.golang.org/grpc" "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/events" ) var def = client.CollectionDefinition{ @@ -61,7 +62,7 @@ func TestPushlogWithDialFailure(t *testing.T) { grpc.WithCredentialsBundle(nil), ) - err = n.server.pushLog(ctx, client.UpdateEvent{ + err = n.server.pushLog(ctx, events.UpdateEvent{ DocID: id.String(), Cid: cid, SchemaRoot: "test", @@ -83,7 +84,7 @@ func TestPushlogWithInvalidPeerID(t *testing.T) { cid, err := createCID(doc) require.NoError(t, err) - err = n.server.pushLog(ctx, client.UpdateEvent{ + err = n.server.pushLog(ctx, events.UpdateEvent{ DocID: id.String(), Cid: cid, SchemaRoot: "test", @@ -131,7 +132,7 @@ func TestPushlogW_WithValidPeerID_NoError(t *testing.T) { cid, err := createCID(doc) require.NoError(t, err) - err = n1.server.pushLog(ctx, client.UpdateEvent{ + err = n1.server.pushLog(ctx, events.UpdateEvent{ DocID: doc.ID().String(), Cid: cid, SchemaRoot: col.SchemaRoot(), diff --git a/net/peer.go b/net/peer.go index fc946effb3..75d3551347 100644 --- a/net/peer.go +++ b/net/peer.go @@ -163,7 +163,7 @@ func (p *Peer) Start() error { } if p.ps != nil { - p.updateSub = p.db.Events().Subscribe(100, client.UpdateEventName) + p.updateSub = p.db.Events().Subscribe(100, events.UpdateEventName) log.InfoContext(p.ctx, "Starting internal broadcaster for pubsub network") go p.handleBroadcastLoop() } @@ -238,7 +238,7 @@ func (p *Peer) handleBroadcastLoop() { if !isOpen { return } - update, ok := value.(client.UpdateEvent) + update, ok := value.(events.UpdateEvent) if !ok { continue // ignore invalid value } @@ -330,7 +330,7 @@ func (p *Peer) pushToReplicator( continue } - evt := client.UpdateEvent{ + evt := events.UpdateEvent{ DocID: docIDResult.ID.String(), Cid: c, SchemaRoot: collection.SchemaRoot(), @@ -397,7 +397,7 @@ func (p *Peer) loadP2PCollections(ctx context.Context) (map[string]struct{}, err return colMap, nil } -func (p *Peer) handleDocCreateLog(evt client.UpdateEvent) error { +func (p *Peer) handleDocCreateLog(evt events.UpdateEvent) error { docID, err := client.NewDocIDFromString(evt.DocID) if err != nil { return NewErrFailedToGetDocID(err) @@ -415,7 +415,7 @@ func (p *Peer) handleDocCreateLog(evt client.UpdateEvent) error { return nil } -func (p *Peer) handleDocUpdateLog(evt client.UpdateEvent) error { +func (p *Peer) handleDocUpdateLog(evt events.UpdateEvent) error { docID, err := client.NewDocIDFromString(evt.DocID) if err != nil { return NewErrFailedToGetDocID(err) @@ -448,7 +448,7 @@ func (p *Peer) handleDocUpdateLog(evt client.UpdateEvent) error { return nil } -func (p *Peer) pushLogToReplicators(lg client.UpdateEvent) { +func (p *Peer) pushLogToReplicators(lg events.UpdateEvent) { // push to each peer (replicator) peers := make(map[string]struct{}) for _, peer := range p.ps.ListPeers(lg.DocID) { diff --git a/net/peer_test.go b/net/peer_test.go index dec53fedc6..b3bbea9f47 100644 --- a/net/peer_test.go +++ b/net/peer_test.go @@ -32,6 +32,7 @@ import ( acpIdentity "github.com/sourcenetwork/defradb/acp/identity" "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/datastore/memory" + "github.com/sourcenetwork/defradb/events" coreblock "github.com/sourcenetwork/defradb/internal/core/block" "github.com/sourcenetwork/defradb/internal/core/crdt" "github.com/sourcenetwork/defradb/internal/db" @@ -236,46 +237,6 @@ func TestStart_WithOfflineKnownPeer_NoError(t *testing.T) { db2.Close() } -func TestStart_WithNoUpdateChannel_NilUpdateChannelError(t *testing.T) { - ctx := context.Background() - store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, nil) - require.NoError(t, err) - - n, err := NewNode( - ctx, - db, - WithEnablePubSub(true), - ) - require.NoError(t, err) - - err = n.Start() - require.ErrorIs(t, err, ErrNilUpdateChannel) - - db.Close() -} - -func TestStart_WitClosedUpdateChannel_ClosedChannelError(t *testing.T) { - ctx := context.Background() - store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, nil) - require.NoError(t, err) - - n, err := NewNode( - ctx, - db, - WithEnablePubSub(true), - ) - require.NoError(t, err) - - db.Events().Close() - - err = n.Start() - require.ErrorContains(t, err, "cannot subscribe to a closed channel") - - db.Close() -} - func TestRegisterNewDocument_NoError(t *testing.T) { ctx := context.Background() db, n := newTestNode(ctx, t) @@ -927,7 +888,7 @@ func TestHandleDocCreateLog_NoError(t *testing.T) { b, err := db.Blockstore().AsIPLDStorage().Get(ctx, headCID.KeyString()) require.NoError(t, err) - err = n.handleDocCreateLog(client.UpdateEvent{ + err = n.handleDocCreateLog(events.UpdateEvent{ DocID: doc.ID().String(), Cid: headCID, SchemaRoot: col.SchemaRoot(), @@ -941,7 +902,7 @@ func TestHandleDocCreateLog_WithInvalidDocID_NoError(t *testing.T) { _, n := newTestNode(ctx, t) defer n.Close() - err := n.handleDocCreateLog(client.UpdateEvent{ + err := n.handleDocCreateLog(events.UpdateEvent{ DocID: "some-invalid-key", }) require.ErrorContains(t, err, "failed to get DocID from broadcast message: selected encoding not supported") @@ -970,7 +931,7 @@ func TestHandleDocCreateLog_WithExistingTopic_TopicExistsError(t *testing.T) { _, err = rpc.NewTopic(ctx, n.ps, n.host.ID(), doc.ID().String(), true) require.NoError(t, err) - err = n.handleDocCreateLog(client.UpdateEvent{ + err = n.handleDocCreateLog(events.UpdateEvent{ DocID: doc.ID().String(), SchemaRoot: col.SchemaRoot(), }) @@ -1003,7 +964,7 @@ func TestHandleDocUpdateLog_NoError(t *testing.T) { b, err := db.Blockstore().AsIPLDStorage().Get(ctx, headCID.KeyString()) require.NoError(t, err) - err = n.handleDocUpdateLog(client.UpdateEvent{ + err = n.handleDocUpdateLog(events.UpdateEvent{ DocID: doc.ID().String(), Cid: headCID, SchemaRoot: col.SchemaRoot(), @@ -1017,7 +978,7 @@ func TestHandleDoUpdateLog_WithInvalidDocID_NoError(t *testing.T) { _, n := newTestNode(ctx, t) defer n.Close() - err := n.handleDocUpdateLog(client.UpdateEvent{ + err := n.handleDocUpdateLog(events.UpdateEvent{ DocID: "some-invalid-key", }) require.ErrorContains(t, err, "failed to get DocID from broadcast message: selected encoding not supported") @@ -1052,7 +1013,7 @@ func TestHandleDocUpdateLog_WithExistingDocIDTopic_TopicExistsError(t *testing.T _, err = rpc.NewTopic(ctx, n.ps, n.host.ID(), doc.ID().String(), true) require.NoError(t, err) - err = n.handleDocUpdateLog(client.UpdateEvent{ + err = n.handleDocUpdateLog(events.UpdateEvent{ DocID: doc.ID().String(), Cid: headCID, SchemaRoot: col.SchemaRoot(), @@ -1090,7 +1051,7 @@ func TestHandleDocUpdateLog_WithExistingSchemaTopic_TopicExistsError(t *testing. _, err = rpc.NewTopic(ctx, n.ps, n.host.ID(), col.SchemaRoot(), true) require.NoError(t, err) - err = n.handleDocUpdateLog(client.UpdateEvent{ + err = n.handleDocUpdateLog(events.UpdateEvent{ DocID: doc.ID().String(), Cid: headCID, SchemaRoot: col.SchemaRoot(), diff --git a/tests/integration/events/utils.go b/tests/integration/events/utils.go index b430c18635..bc2972faf3 100644 --- a/tests/integration/events/utils.go +++ b/tests/integration/events/utils.go @@ -20,6 +20,7 @@ import ( "github.com/stretchr/testify/require" "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/events" testUtils "github.com/sourcenetwork/defradb/tests/integration" ) @@ -79,7 +80,7 @@ func ExecuteRequestTestCase( testRoutineClosedChan := make(chan struct{}) closeTestRoutineChan := make(chan struct{}) - eventsSub := db.Events().Subscribe(5, client.UpdateEventName) + eventsSub := db.Events().Subscribe(5, events.UpdateEventName) require.NoError(t, err) indexOfNextExpectedUpdate := 0 @@ -87,7 +88,7 @@ func ExecuteRequestTestCase( for { select { case value := <-eventsSub.Value(): - update, ok := value.(client.UpdateEvent) + update, ok := value.(events.UpdateEvent) if !ok { continue // ignore invaid value } From 8f870bd885b435e9f37c77222eaf57f3137ad2c7 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 5 Jun 2024 17:28:10 -0700 Subject: [PATCH 03/34] refactor net events --- events/events.go | 29 +++- net/dag_test.go | 22 ++- net/node.go | 213 ++++----------------------- net/node_test.go | 369 ----------------------------------------------- net/peer.go | 22 --- net/server.go | 48 ++---- 6 files changed, 89 insertions(+), 614 deletions(-) diff --git a/events/events.go b/events/events.go index 12c10ddaf4..78a2bf0de8 100644 --- a/events/events.go +++ b/events/events.go @@ -10,7 +10,11 @@ package events -import "github.com/ipfs/go-cid" +import ( + "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p/core/event" + "github.com/libp2p/go-libp2p/core/peer" +) const ( // WildCardEventName is the alias used to subscribe to all events. @@ -19,8 +23,31 @@ const ( UpdateEventName = "db:update" // ResultsEventName is the name of the database results event. ResultsEventName = "db:results" + // PushLogEventName is the name of the network pushlog event. + PushLogEventName = "net:pushlog" + // PubSubEventName is the name of the network pubsub event. + PubSubEventName = "net:pubsub" + // PeerEventName is the name of the network peer event. + PeerEventName = "net:peer" ) +// PeerEvent is an event that is published when +// a peer connection has changed status. +type PeerEvent = event.EvtPeerConnectednessChanged + +// PushLogEvent is an event that is published when +// a pushlog message has been received from a remote peer. +type PushLogEvent struct { + ByPeer peer.ID + FromPeer peer.ID +} + +// PubSubEvent is an event that is published when +// a pubsub message has been received from a remote peer. +type PubSubEvent struct { + Peer peer.ID +} + // UpdateEvent represents a new DAG node added to the append-only composite MerkleCRDT Clock graph // of a document. // diff --git a/net/dag_test.go b/net/dag_test.go index 976f43653a..9172ebf619 100644 --- a/net/dag_test.go +++ b/net/dag_test.go @@ -16,9 +16,13 @@ import ( "testing" "time" + "github.com/libp2p/go-libp2p/core/event" + "github.com/libp2p/go-libp2p/core/network" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/events" "github.com/sourcenetwork/defradb/internal/core" coreblock "github.com/sourcenetwork/defradb/internal/core/block" "github.com/sourcenetwork/defradb/internal/merkle/clock" @@ -137,14 +141,24 @@ func TestSendJobWorker_WithPeer_NoError(t *testing.T) { db1, n1 := newTestNode(ctx, t) db2, n2 := newTestNode(ctx, t) + sub1 := db1.Events().Subscribe(5, events.PeerEventName) + defer db1.Events().Unsubscribe(sub1) + + sub2 := db2.Events().Subscribe(5, events.PeerEventName) + defer db2.Events().Unsubscribe(sub2) + addrs, err := netutils.ParsePeers([]string{n1.host.Addrs()[0].String() + "/p2p/" + n1.PeerID().String()}) require.NoError(t, err) n2.Bootstrap(addrs) - err = n1.WaitForPeerConnectionEvent(n2.PeerID()) - require.NoError(t, err) - err = n2.WaitForPeerConnectionEvent(n1.PeerID()) - require.NoError(t, err) + event1 := <-sub1.Value() + assert.Equal(t, network.Connected, event1.(event.EvtPeerConnectednessChanged).Connectedness) + assert.Equal(t, n2.PeerID(), event1.(event.EvtPeerConnectednessChanged).Peer) + + event2 := <-sub2.Value() + assert.Equal(t, network.Connected, event2.(event.EvtPeerConnectednessChanged).Connectedness) + assert.Equal(t, n1.PeerID(), event2.(event.EvtPeerConnectednessChanged).Peer) + done := make(chan struct{}) go func() { n2.sendJobWorker() diff --git a/net/node.go b/net/node.go index 7683d3fb8f..be7c84cf80 100644 --- a/net/node.go +++ b/net/node.go @@ -33,7 +33,6 @@ import ( libp2pCrypto "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/event" "github.com/libp2p/go-libp2p/core/host" - "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" @@ -48,10 +47,9 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/crypto" + "github.com/sourcenetwork/defradb/events" ) -var evtWaitTimeout = 10 * time.Second - var _ client.P2P = (*Node)(nil) // Node is a networked peer instance of DefraDB. @@ -61,15 +59,6 @@ type Node struct { *Peer - // receives an event when the status of a peer connection changes. - peerEvent chan event.EvtPeerConnectednessChanged - - // receives an event when a pubsub topic is added. - pubSubEvent chan EvtPubSub - - // receives an event when a pushLog request has been processed. - pushLogEvent chan EvtReceivedPushLog - ctx context.Context cancel context.CancelFunc } @@ -171,8 +160,18 @@ func NewNode( } } - ctx, cancel := context.WithCancel(ctx) + // slice of events to subscribe to + subEvents := []any{ + &event.EvtPeerConnectednessChanged{}, + &events.PubSubEvent{}, + &events.PushLogEvent{}, + } + sub, err := h.EventBus().Subscribe(subEvents) + if err != nil { + return nil, fin.Cleanup(err) + } + ctx, cancel := context.WithCancel(ctx) peer, err := NewPeer( ctx, db, @@ -187,27 +186,28 @@ func NewNode( return nil, fin.Cleanup(err) } - n := &Node{ - // WARNING: The current usage of these channels means that consumers of them - // (the WaitForFoo funcs) can recieve events that occured before the WaitForFoo - // function call. This is tolerable at the moment as they are only used for - // test, but we should resolve this when we can (e.g. via using subscribe-like - // mechanics, potentially via use of a ring-buffer based [events.Channel] - // implementation): https://github.com/sourcenetwork/defradb/issues/1358. - pubSubEvent: make(chan EvtPubSub, 20), - pushLogEvent: make(chan EvtReceivedPushLog, 20), - peerEvent: make(chan event.EvtPeerConnectednessChanged, 20), - Peer: peer, - DB: db, - ctx: ctx, - cancel: cancel, - } + // publish subscribed events to the event bus + go func() { + for val := range sub.Out() { + switch t := val.(type) { + case event.EvtPeerConnectednessChanged: + db.Events().Publish(events.PeerEventName, events.PeerEvent(t)) - n.subscribeToPeerConnectionEvents() - n.subscribeToPubSubEvents() - n.subscribeToPushLogEvents() + case events.PubSubEvent: + db.Events().Publish(events.PubSubEventName, t) - return n, nil + case events.PushLogEvent: + db.Events().Publish(events.PushLogEventName, t) + } + } + }() + + return &Node{ + Peer: peer, + DB: db, + ctx: ctx, + cancel: cancel, + }, nil } // Bootstrap connects to the given peers. @@ -257,155 +257,6 @@ func (n *Node) PeerInfo() peer.AddrInfo { } } -// subscribeToPeerConnectionEvents subscribes the node to the event bus for a peer connection change. -func (n *Node) subscribeToPeerConnectionEvents() { - sub, err := n.host.EventBus().Subscribe(new(event.EvtPeerConnectednessChanged)) - if err != nil { - log.InfoContext( - n.ctx, - fmt.Sprintf("failed to subscribe to peer connectedness changed event: %v", err), - ) - return - } - go func() { - for e := range sub.Out() { - select { - case n.peerEvent <- e.(event.EvtPeerConnectednessChanged): - default: - <-n.peerEvent - n.peerEvent <- e.(event.EvtPeerConnectednessChanged) - } - } - }() -} - -// subscribeToPubSubEvents subscribes the node to the event bus for a pubsub. -func (n *Node) subscribeToPubSubEvents() { - sub, err := n.host.EventBus().Subscribe(new(EvtPubSub)) - if err != nil { - log.InfoContext( - n.ctx, - fmt.Sprintf("failed to subscribe to pubsub event: %v", err), - ) - return - } - go func() { - for e := range sub.Out() { - select { - case n.pubSubEvent <- e.(EvtPubSub): - default: - <-n.pubSubEvent - n.pubSubEvent <- e.(EvtPubSub) - } - } - }() -} - -// subscribeToPushLogEvents subscribes the node to the event bus for a push log request completion. -func (n *Node) subscribeToPushLogEvents() { - sub, err := n.host.EventBus().Subscribe(new(EvtReceivedPushLog)) - if err != nil { - log.InfoContext( - n.ctx, - fmt.Sprintf("failed to subscribe to push log event: %v", err), - ) - return - } - go func() { - for e := range sub.Out() { - select { - case n.pushLogEvent <- e.(EvtReceivedPushLog): - default: - <-n.pushLogEvent - n.pushLogEvent <- e.(EvtReceivedPushLog) - } - } - }() -} - -// WaitForPeerConnectionEvent listens to the event channel for a connection event from a given peer. -func (n *Node) WaitForPeerConnectionEvent(id peer.ID) error { - if n.host.Network().Connectedness(id) == network.Connected { - return nil - } - for { - select { - case evt := <-n.peerEvent: - if evt.Peer != id { - continue - } - return nil - case <-time.After(evtWaitTimeout): - return ErrPeerConnectionWaitTimout - case <-n.ctx.Done(): - return nil - } - } -} - -// WaitForPubSubEvent listens to the event channel for pub sub event from a given peer. -func (n *Node) WaitForPubSubEvent(id peer.ID) error { - for { - select { - case evt := <-n.pubSubEvent: - if evt.Peer != id { - continue - } - return nil - case <-time.After(evtWaitTimeout): - return ErrPubSubWaitTimeout - case <-n.ctx.Done(): - return nil - } - } -} - -// WaitForPushLogByPeerEvent listens to the event channel for a push log event by a given peer. -// -// By refers to the log creator. It can be different than the log sender. -// -// It will block the calling thread until an event is yielded to an internal channel. This -// event is not necessarily the next event and is dependent on the number of concurrent callers -// (each event will only notify a single caller, not all of them). -func (n *Node) WaitForPushLogByPeerEvent(id peer.ID) error { - for { - select { - case evt := <-n.pushLogEvent: - if evt.ByPeer != id { - continue - } - return nil - case <-time.After(evtWaitTimeout): - return ErrPushLogWaitTimeout - case <-n.ctx.Done(): - return nil - } - } -} - -// WaitForPushLogFromPeerEvent listens to the event channel for a push log event from a given peer. -// -// From refers to the log sender. It can be different that the log creator. -// -// It will block the calling thread until an event is yielded to an internal channel. This -// event is not necessarily the next event and is dependent on the number of concurrent callers -// (each event will only notify a single caller, not all of them). -func (n *Node) WaitForPushLogFromPeerEvent(id peer.ID) error { - for { - select { - case evt := <-n.pushLogEvent: - if evt.FromPeer != id { - continue - } - return nil - case <-time.After(evtWaitTimeout): - return ErrPushLogWaitTimeout - case <-n.ctx.Done(): - return nil - } - } -} - func newDHT(ctx context.Context, h host.Host, dsb ds.Batching) (*dualdht.DHT, error) { dhtOpts := []dualdht.Option{ dualdht.DHTOption(dht.NamespacedValidator("pk", record.PublicKeyValidator{})), diff --git a/net/node_test.go b/net/node_test.go index 5f27d1b5e7..f04e7c6bac 100644 --- a/net/node_test.go +++ b/net/node_test.go @@ -13,9 +13,7 @@ package net import ( "context" "testing" - "time" - "github.com/libp2p/go-libp2p/core/event" "github.com/libp2p/go-libp2p/core/peer" badger "github.com/sourcenetwork/badger/v4" "github.com/stretchr/testify/require" @@ -204,370 +202,3 @@ func TestListenAddrs_WithListenAddresses_NoError(t *testing.T) { require.Contains(t, n.ListenAddrs()[0].String(), "/tcp/") } - -func TestPeerConnectionEventEmitter_MultiEvent_NoError(t *testing.T) { - db := FixtureNewMemoryDBWithBroadcaster(t) - n, err := NewNode( - context.Background(), - db, - ) - require.NoError(t, err) - defer n.Close() - - emitter, err := n.host.EventBus().Emitter(new(event.EvtPeerConnectednessChanged)) - require.NoError(t, err) - - // the emitter can take 20 events in the channel. This tests what happens whe go over the 20 events. - for i := 0; i < 21; i++ { - err = emitter.Emit(event.EvtPeerConnectednessChanged{}) - require.NoError(t, err) - } -} - -func TestSubscribeToPubSubEvents_SubscriptionError(t *testing.T) { - db := FixtureNewMemoryDBWithBroadcaster(t) - n, err := NewNode( - context.Background(), - db, - ) - require.NoError(t, err) - defer n.Close() - - n.Peer.host = &mockHost{n.Peer.host} - - n.subscribeToPubSubEvents() -} - -func TestPubSubEventEmitter_MultiEvent_NoError(t *testing.T) { - db := FixtureNewMemoryDBWithBroadcaster(t) - n, err := NewNode( - context.Background(), - db, - ) - require.NoError(t, err) - defer n.Close() - - emitter, err := n.host.EventBus().Emitter(new(EvtPubSub)) - require.NoError(t, err) - - // the emitter can take 20 events in the channel. This tests what happens whe go over the 20 events. - for i := 0; i < 21; i++ { - err = emitter.Emit(EvtPubSub{}) - require.NoError(t, err) - } -} - -func TestSubscribeToPushLogEvents_SubscriptionError(t *testing.T) { - db := FixtureNewMemoryDBWithBroadcaster(t) - n, err := NewNode( - context.Background(), - db, - ) - require.NoError(t, err) - defer n.Close() - - n.Peer.host = &mockHost{n.Peer.host} - - n.subscribeToPushLogEvents() -} - -func TestPushLogEventEmitter_SingleEvent_NoError(t *testing.T) { - db := FixtureNewMemoryDBWithBroadcaster(t) - n, err := NewNode( - context.Background(), - db, - ) - require.NoError(t, err) - defer n.Close() - - emitter, err := n.host.EventBus().Emitter(new(EvtReceivedPushLog)) - require.NoError(t, err) - - err = emitter.Emit(EvtReceivedPushLog{}) - require.NoError(t, err) -} - -func TestPushLogEventEmitter_MultiEvent_NoError(t *testing.T) { - db := FixtureNewMemoryDBWithBroadcaster(t) - n, err := NewNode( - context.Background(), - db, - ) - require.NoError(t, err) - defer n.Close() - - emitter, err := n.host.EventBus().Emitter(new(EvtReceivedPushLog)) - require.NoError(t, err) - - // the emitter can take 20 events in the channel. This tests what happens whe go over the 20 events. - for i := 0; i < 21; i++ { - err = emitter.Emit(EvtReceivedPushLog{}) - require.NoError(t, err) - } -} - -func TestWaitForPeerConnectionEvent_WithSamePeer_NoError(t *testing.T) { - db := FixtureNewMemoryDBWithBroadcaster(t) - n, err := NewNode( - context.Background(), - db, - ) - require.NoError(t, err) - defer n.Close() - - emitter, err := n.host.EventBus().Emitter(new(event.EvtPeerConnectednessChanged)) - require.NoError(t, err) - - err = emitter.Emit(event.EvtPeerConnectednessChanged{ - Peer: n.PeerID(), - }) - require.NoError(t, err) - - err = n.WaitForPeerConnectionEvent(n.PeerID()) - require.NoError(t, err) -} - -func TestWaitForPeerConnectionEvent_WithDifferentPeer_TimeoutError(t *testing.T) { - evtWaitTimeout = 100 * time.Microsecond - defer func() { - evtWaitTimeout = 10 * time.Second - }() - db := FixtureNewMemoryDBWithBroadcaster(t) - n, err := NewNode( - context.Background(), - db, - ) - require.NoError(t, err) - defer n.Close() - - emitter, err := n.host.EventBus().Emitter(new(event.EvtPeerConnectednessChanged)) - require.NoError(t, err) - - err = emitter.Emit(event.EvtPeerConnectednessChanged{}) - require.NoError(t, err) - - err = n.WaitForPeerConnectionEvent(n.PeerID()) - require.ErrorIs(t, err, ErrPeerConnectionWaitTimout) -} - -func TestWaitForPeerConnectionEvent_WithDifferentPeerAndContextClosed_NoError(t *testing.T) { - db := FixtureNewMemoryDBWithBroadcaster(t) - n, err := NewNode( - context.Background(), - db, - ) - require.NoError(t, err) - defer n.Close() - - emitter, err := n.host.EventBus().Emitter(new(event.EvtPeerConnectednessChanged)) - require.NoError(t, err) - - err = emitter.Emit(event.EvtPeerConnectednessChanged{}) - require.NoError(t, err) - - n.cancel() - - err = n.WaitForPeerConnectionEvent(n.PeerID()) - require.NoError(t, err) -} - -func TestWaitForPubSubEvent_WithSamePeer_NoError(t *testing.T) { - db := FixtureNewMemoryDBWithBroadcaster(t) - n, err := NewNode( - context.Background(), - db, - ) - require.NoError(t, err) - defer n.Close() - - emitter, err := n.host.EventBus().Emitter(new(EvtPubSub)) - require.NoError(t, err) - - err = emitter.Emit(EvtPubSub{ - Peer: n.PeerID(), - }) - require.NoError(t, err) - - err = n.WaitForPubSubEvent(n.PeerID()) - require.NoError(t, err) -} - -func TestWaitForPubSubEvent_WithDifferentPeer_TimeoutError(t *testing.T) { - evtWaitTimeout = 100 * time.Microsecond - defer func() { - evtWaitTimeout = 10 * time.Second - }() - db := FixtureNewMemoryDBWithBroadcaster(t) - n, err := NewNode( - context.Background(), - db, - ) - require.NoError(t, err) - defer n.Close() - - emitter, err := n.host.EventBus().Emitter(new(EvtPubSub)) - require.NoError(t, err) - - err = emitter.Emit(EvtPubSub{}) - require.NoError(t, err) - - err = n.WaitForPubSubEvent(n.PeerID()) - require.ErrorIs(t, err, ErrPubSubWaitTimeout) -} - -func TestWaitForPubSubEvent_WithDifferentPeerAndContextClosed_NoError(t *testing.T) { - db := FixtureNewMemoryDBWithBroadcaster(t) - n, err := NewNode( - context.Background(), - db, - ) - require.NoError(t, err) - defer n.Close() - - emitter, err := n.host.EventBus().Emitter(new(EvtPubSub)) - require.NoError(t, err) - - err = emitter.Emit(EvtPubSub{}) - require.NoError(t, err) - - n.cancel() - - err = n.WaitForPubSubEvent(n.PeerID()) - require.NoError(t, err) -} - -func TestWaitForPushLogByPeerEvent_WithSamePeer_NoError(t *testing.T) { - ctx := context.Background() - db := FixtureNewMemoryDBWithBroadcaster(t) - n, err := NewNode( - ctx, - db, - ) - require.NoError(t, err) - defer n.Close() - - emitter, err := n.host.EventBus().Emitter(new(EvtReceivedPushLog)) - require.NoError(t, err) - - err = emitter.Emit(EvtReceivedPushLog{ - ByPeer: n.PeerID(), - }) - require.NoError(t, err) - - err = n.WaitForPushLogByPeerEvent(n.PeerID()) - require.NoError(t, err) -} - -func TestWaitForPushLogByPeerEvent_WithDifferentPeer_TimeoutError(t *testing.T) { - evtWaitTimeout = 100 * time.Microsecond - defer func() { - evtWaitTimeout = 10 * time.Second - }() - ctx := context.Background() - db := FixtureNewMemoryDBWithBroadcaster(t) - n, err := NewNode( - ctx, - db, - ) - require.NoError(t, err) - defer n.Close() - - emitter, err := n.host.EventBus().Emitter(new(EvtReceivedPushLog)) - require.NoError(t, err) - - err = emitter.Emit(EvtReceivedPushLog{}) - require.NoError(t, err) - - err = n.WaitForPushLogByPeerEvent(n.PeerID()) - require.ErrorIs(t, err, ErrPushLogWaitTimeout) -} - -func TestWaitForPushLogByPeerEvent_WithDifferentPeerAndContextClosed_NoError(t *testing.T) { - ctx := context.Background() - db := FixtureNewMemoryDBWithBroadcaster(t) - n, err := NewNode( - ctx, - db, - ) - require.NoError(t, err) - defer n.Close() - - emitter, err := n.host.EventBus().Emitter(new(EvtReceivedPushLog)) - require.NoError(t, err) - - err = emitter.Emit(EvtReceivedPushLog{}) - require.NoError(t, err) - - n.cancel() - - err = n.WaitForPushLogByPeerEvent(n.PeerID()) - require.NoError(t, err) -} - -func TestWaitForPushLogFromPeerEvent_WithSamePeer_NoError(t *testing.T) { - ctx := context.Background() - db := FixtureNewMemoryDBWithBroadcaster(t) - n, err := NewNode( - ctx, - db, - ) - require.NoError(t, err) - defer n.Close() - - emitter, err := n.host.EventBus().Emitter(new(EvtReceivedPushLog)) - require.NoError(t, err) - - err = emitter.Emit(EvtReceivedPushLog{ - FromPeer: n.PeerID(), - }) - require.NoError(t, err) - - err = n.WaitForPushLogFromPeerEvent(n.PeerID()) - require.NoError(t, err) -} - -func TestWaitForPushLogFromPeerEvent_WithDifferentPeer_TimeoutError(t *testing.T) { - evtWaitTimeout = 100 * time.Microsecond - defer func() { - evtWaitTimeout = 10 * time.Second - }() - ctx := context.Background() - db := FixtureNewMemoryDBWithBroadcaster(t) - n, err := NewNode( - ctx, - db, - ) - require.NoError(t, err) - defer n.Close() - - emitter, err := n.host.EventBus().Emitter(new(EvtReceivedPushLog)) - require.NoError(t, err) - - err = emitter.Emit(EvtReceivedPushLog{}) - require.NoError(t, err) - - err = n.WaitForPushLogFromPeerEvent(n.PeerID()) - require.ErrorIs(t, err, ErrPushLogWaitTimeout) -} - -func TestWaitForPushLogFromPeerEvent_WithDifferentPeerAndContextClosed_NoError(t *testing.T) { - ctx := context.Background() - db := FixtureNewMemoryDBWithBroadcaster(t) - n, err := NewNode( - ctx, - db, - ) - require.NoError(t, err) - defer n.Close() - - emitter, err := n.host.EventBus().Emitter(new(EvtReceivedPushLog)) - require.NoError(t, err) - - err = emitter.Emit(EvtReceivedPushLog{}) - require.NoError(t, err) - - n.cancel() - - err = n.WaitForPushLogFromPeerEvent(n.PeerID()) - require.NoError(t, err) -} diff --git a/net/peer.go b/net/peer.go index 75d3551347..29eccca1ad 100644 --- a/net/peer.go +++ b/net/peer.go @@ -201,19 +201,6 @@ func (p *Peer) Close() { } } stopGRPCServer(p.ctx, p.p2pRPC) - // stopGRPCServer(p.tcpRPC) - - // close event emitters - if p.server.pubSubEmitter != nil { - if err := p.server.pubSubEmitter.Close(); err != nil { - log.InfoContext(p.ctx, "Could not close pubsub event emitter", corelog.Any("Error", err.Error())) - } - } - if p.server.pushLogEmitter != nil { - if err := p.server.pushLogEmitter.Close(); err != nil { - log.InfoContext(p.ctx, "Could not close push log event emitter", corelog.Any("Error", err.Error())) - } - } if p.updateSub != nil { p.db.Events().Unsubscribe(p.updateSub) @@ -518,15 +505,6 @@ func stopGRPCServer(ctx context.Context, server *grpc.Server) { } } -type EvtReceivedPushLog struct { - ByPeer peer.ID - FromPeer peer.ID -} - -type EvtPubSub struct { - Peer peer.ID -} - // rollbackAddPubSubTopics removes the given topics from the pubsub system. func (p *Peer) rollbackAddPubSubTopics(topics []string, cause error) error { for _, topic := range topics { diff --git a/net/server.go b/net/server.go index 1cd9910856..cd7428ae4d 100644 --- a/net/server.go +++ b/net/server.go @@ -18,7 +18,6 @@ import ( "sync" "github.com/ipfs/go-cid" - "github.com/libp2p/go-libp2p/core/event" libpeer "github.com/libp2p/go-libp2p/core/peer" "github.com/sourcenetwork/corelog" rpc "github.com/sourcenetwork/go-libp2p-pubsub-rpc" @@ -31,6 +30,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/datastore/badger/v4" "github.com/sourcenetwork/defradb/errors" + "github.com/sourcenetwork/defradb/events" "github.com/sourcenetwork/defradb/internal/core" coreblock "github.com/sourcenetwork/defradb/internal/core/block" "github.com/sourcenetwork/defradb/internal/db" @@ -52,9 +52,6 @@ type server struct { conns map[libpeer.ID]*grpc.ClientConn - pubSubEmitter event.Emitter - pushLogEmitter event.Emitter - // docQueue is used to track which documents are currently being processed. // This is used to prevent multiple concurrent processing of the same document and // limit unecessary transaction conflicts. @@ -123,16 +120,6 @@ func newServer(p *Peer, db client.DB, opts ...grpc.DialOption) (*server, error) } } - var err error - s.pubSubEmitter, err = s.peer.host.EventBus().Emitter(new(EvtPubSub)) - if err != nil { - log.InfoContext(s.peer.ctx, "could not create event emitter", corelog.String("Error", err.Error())) - } - s.pushLogEmitter, err = s.peer.host.EventBus().Emitter(new(EvtReceivedPushLog)) - if err != nil { - log.InfoContext(s.peer.ctx, "could not create event emitter", corelog.String("Error", err.Error())) - } - return s, nil } @@ -207,21 +194,14 @@ func (s *server) PushLog(ctx context.Context, req *pb.PushLogRequest) (*pb.PushL s.docQueue.add(docID.String()) defer func() { s.docQueue.done(docID.String()) - if s.pushLogEmitter != nil { - byPeer, err := libpeer.Decode(req.Body.Creator) - if err != nil { - log.InfoContext(ctx, "could not decode the PeerID of the log creator", corelog.String("Error", err.Error())) - } - err = s.pushLogEmitter.Emit(EvtReceivedPushLog{ - FromPeer: pid, - ByPeer: byPeer, - }) - if err != nil { - // logging instead of returning an error because the event bus should - // not break the PushLog execution. - log.InfoContext(ctx, "could not emit push log event", corelog.String("Error", err.Error())) - } + byPeer, err := libpeer.Decode(req.Body.Creator) + if err != nil { + log.InfoContext(ctx, "could not decode the PeerID of the log creator", corelog.String("Error", err.Error())) } + s.db.Events().Publish(events.PushLogEventName, events.PushLogEvent{ + FromPeer: pid, + ByPeer: byPeer, + }) }() // make sure were not processing twice @@ -511,15 +491,9 @@ func (s *server) pubSubEventHandler(from libpeer.ID, topic string, msg []byte) { corelog.String("Topic", topic), corelog.String("Message", string(msg)), ) - - if s.pubSubEmitter != nil { - err := s.pubSubEmitter.Emit(EvtPubSub{ - Peer: from, - }) - if err != nil { - log.InfoContext(s.peer.ctx, "could not emit pubsub event", corelog.Any("Error", err.Error())) - } - } + s.db.Events().Publish(events.PubSubEventName, events.PubSubEvent{ + Peer: from, + }) } // addr implements net.Addr and holds a libp2p peer ID. From 138bb84f0769c6fd2c24a226805645e00423c20b Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Fri, 7 Jun 2024 12:00:08 -0700 Subject: [PATCH 04/34] add event messages. use separate event bus for user subscriptions --- events/bus.go | 67 ++++++++++++++++--------- events/{subscription.go => bus_test.go} | 32 +++++++----- events/events.go | 8 +-- internal/db/collection.go | 2 +- internal/db/collection_delete.go | 2 +- internal/db/db.go | 33 ++++++++++-- internal/db/subscriptions.go | 9 ++-- net/dag_test.go | 16 +++--- net/node.go | 19 +------ net/peer.go | 4 +- net/server.go | 6 ++- 11 files changed, 119 insertions(+), 79 deletions(-) rename events/{subscription.go => bus_test.go} (51%) diff --git a/events/bus.go b/events/bus.go index be1c12aaf4..b3decac9f6 100644 --- a/events/bus.go +++ b/events/bus.go @@ -14,8 +14,37 @@ import ( "sync" ) -// Bus is an event bus used to publish and subscribe to internal -// subsystem messages. +// Message contains event info. +type Message struct { + // Name is the name of the event this message was generated from. + Name string + // Data contains optional event information. + Data any +} + +// NewMessage returns a new message with the given name and optional data. +func NewMessage(name string, data any) Message { + return Message{name, data} +} + +// Subscription is a read-only event stream. +type Subscription struct { + id int + value chan Message + events []string +} + +// Message returns the next event value from the subscription. +func (s *Subscription) Message() <-chan Message { + return s.value +} + +// Events returns the names of all subscribed events. +func (s *Subscription) Events() []string { + return s.events +} + +// Bus is used to publish and subscribe to events. type Bus struct { subId int subs map[int]*Subscription @@ -32,24 +61,21 @@ func NewBus() *Bus { } // Publish publishes the given event to all subscribers. -// -// This method will never block if the subscribers buffer is full. -func (b *Bus) Publish(event string, value any) { +func (b *Bus) Publish(msg Message) { b.mutex.RLock() defer b.mutex.RUnlock() subscribers := make(map[int]any) - // publish to event subscribers - for id := range b.events[event] { + for id := range b.events[msg.Name] { subscribers[id] = struct{}{} } - // also publish to wildcard recipients for id := range b.events[WildCardEventName] { subscribers[id] = struct{}{} } + for id := range subscribers { select { - case b.subs[id].value <- value: + case b.subs[id].value <- msg: // published event default: // channel full @@ -58,28 +84,25 @@ func (b *Bus) Publish(event string, value any) { } // Subscribe returns a new channel that will receive all of the subscribed events. -// -// The size of the buffer should be appropriate for the consumer or events will be dropped. func (b *Bus) Subscribe(size int, events ...string) *Subscription { b.mutex.Lock() defer b.mutex.Unlock() sub := &Subscription{ id: b.subId, - value: make(chan any, size), + value: make(chan Message, size), events: events, } - b.subId++ - b.subs[sub.id] = sub - - // add sub to all events for _, event := range events { if _, ok := b.events[event]; !ok { b.events[event] = make(map[int]any) } b.events[event][sub.id] = struct{}{} } + + b.subId++ + b.subs[sub.id] = sub return sub } @@ -88,29 +111,27 @@ func (b *Bus) Unsubscribe(sub *Subscription) { b.mutex.Lock() defer b.mutex.Unlock() - // delete sub from all events + if _, ok := b.subs[sub.id]; !ok { + return // not subscribed + } for _, event := range sub.events { delete(b.events[event], sub.id) } - // only close channel once - if _, ok := b.subs[sub.id]; ok { - close(sub.value) - } + delete(b.subs, sub.id) + close(sub.value) } // Close closes the event bus by unsubscribing all subscribers. func (b *Bus) Close() { var subs []*Subscription - // get list of all subs b.mutex.RLock() for _, sub := range b.subs { subs = append(subs, sub) } b.mutex.RUnlock() - // unsubscribe all subs for _, sub := range subs { b.Unsubscribe(sub) } diff --git a/events/subscription.go b/events/bus_test.go similarity index 51% rename from events/subscription.go rename to events/bus_test.go index e29f25da8f..f96fc3d36f 100644 --- a/events/subscription.go +++ b/events/bus_test.go @@ -10,19 +10,25 @@ package events -// Subscription is a read-only event stream. -type Subscription struct { - id int - value chan any - events []string -} +import ( + "testing" -// Value returns the next event value from the subscription. -func (s *Subscription) Value() <-chan any { - return s.value -} + "github.com/stretchr/testify/assert" +) + +func TestBusPublish(t *testing.T) { + bus := NewBus() + defer bus.Close() + + sub1 := bus.Subscribe(1, "test") + sub2 := bus.Subscribe(1, WildCardEventName) + + msg := NewMessage("test", "hello") + bus.Publish(msg) + + event1 := <-sub1.Message() + assert.Equal(t, msg, event1) -// Events returns the names of all subscribed events. -func (s *Subscription) Events() []string { - return s.events + event2 := <-sub2.Message() + assert.Equal(t, msg, event2) } diff --git a/events/events.go b/events/events.go index 78a2bf0de8..0853fa9a26 100644 --- a/events/events.go +++ b/events/events.go @@ -27,13 +27,13 @@ const ( PushLogEventName = "net:pushlog" // PubSubEventName is the name of the network pubsub event. PubSubEventName = "net:pubsub" - // PeerEventName is the name of the network peer event. - PeerEventName = "net:peer" + // ConnectEventName is the name of the network connect event. + ConnectEventName = "net:connect" ) -// PeerEvent is an event that is published when +// ConnectEvent is an event that is published when // a peer connection has changed status. -type PeerEvent = event.EvtPeerConnectednessChanged +type ConnectEvent = event.EvtPeerConnectednessChanged // PushLogEvent is an event that is published when // a pushlog message has been received from a remote peer. diff --git a/internal/db/collection.go b/internal/db/collection.go index 0fa10062c5..a0cd538641 100644 --- a/internal/db/collection.go +++ b/internal/db/collection.go @@ -1802,7 +1802,7 @@ func (c *collection) save( IsCreate: isCreate, } txn.OnSuccess(func() { - c.db.events.Publish(events.UpdateEventName, updateEvent) + c.db.sysEventBus.Publish(events.NewMessage(events.UpdateEventName, updateEvent)) }) txn.OnSuccess(func() { diff --git a/internal/db/collection_delete.go b/internal/db/collection_delete.go index 41d9b9bc3e..eb75e29372 100644 --- a/internal/db/collection_delete.go +++ b/internal/db/collection_delete.go @@ -174,7 +174,7 @@ func (c *collection) applyDelete( Block: b, } txn.OnSuccess(func() { - c.db.events.Publish(events.UpdateEventName, updateEvent) + c.db.sysEventBus.Publish(events.NewMessage(events.UpdateEventName, updateEvent)) }) return nil diff --git a/internal/db/db.go b/internal/db/db.go index f90fdc79b0..5718b103fa 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -43,6 +43,15 @@ var ( _ client.Collection = (*collection)(nil) ) +const ( + // forwardSubBufferSize controls the size of the channel used to + // forward events from the system bus to the subscription bus + forwardSubBufferSize = 10 + // requestSubBufferSize controls the size of the channel used to + // send events to request subscriptions + requestSubBufferSize = 5 +) + // DB is the main interface for interacting with the // DefraDB storage system. type db struct { @@ -51,7 +60,11 @@ type db struct { rootstore datastore.RootStore multistore datastore.MultiStore - events *events.Bus + // sysEventBus is used to send and receive system events + sysEventBus *events.Bus + + // subEventBus is used to send and receive request subscription events + subEventBus *events.Bus parser core.Parser @@ -102,7 +115,8 @@ func newDB( lensRegistry: lens, parser: parser, options: options, - events: events.NewBus(), + sysEventBus: events.NewBus(), + subEventBus: events.NewBus(), } // apply options @@ -185,6 +199,15 @@ func (db *db) initialize(ctx context.Context) error { db.glock.Lock() defer db.glock.Unlock() + // forward system bus events to the subscriber bus + // to ensure we never block the system bus for user subscriptions + go func() { + sub := db.sysEventBus.Subscribe(forwardSubBufferSize, events.WildCardEventName) + for msg := range sub.Message() { + db.subEventBus.Publish(msg) + } + }() + ctx, txn, err := ensureContextTxn(ctx, db, false) if err != nil { return err @@ -239,7 +262,7 @@ func (db *db) initialize(ctx context.Context) error { // Events returns the events Channel. func (db *db) Events() *events.Bus { - return db.events + return db.sysEventBus } // MaxRetries returns the maximum number of retries per transaction. @@ -260,7 +283,9 @@ func (db *db) PrintDump(ctx context.Context) error { // This is the place for any last minute cleanup or releasing of resources (i.e.: Badger instance). func (db *db) Close() { log.Info("Closing DefraDB process...") - db.events.Close() + + db.subEventBus.Close() + db.sysEventBus.Close() err := db.rootstore.Close() if err != nil { diff --git a/internal/db/subscriptions.go b/internal/db/subscriptions.go index 5fffee06a4..bb49a0efa1 100644 --- a/internal/db/subscriptions.go +++ b/internal/db/subscriptions.go @@ -31,11 +31,12 @@ func (db *db) handleSubscription(ctx context.Context, r *request.Request) (<-cha if !ok { return nil, client.NewErrUnexpectedType[request.ObjectSubscription]("SubscriptionSelection", selections) } - sub := db.events.Subscribe(5, events.UpdateEventName) + // subscribe to the subscription event bus so we don't block the system bus + sub := db.subEventBus.Subscribe(requestSubBufferSize, events.UpdateEventName) resCh := make(chan client.GQLResult) go func() { defer func() { - db.events.Unsubscribe(sub) + db.sysEventBus.Unsubscribe(sub) close(resCh) }() @@ -45,11 +46,11 @@ func (db *db) handleSubscription(ctx context.Context, r *request.Request) (<-cha select { case <-ctx.Done(): return // context cancelled - case val, ok := <-sub.Value(): + case val, ok := <-sub.Message(): if !ok { return // channel closed } - evt, ok = val.(events.UpdateEvent) + evt, ok = val.Data.(events.UpdateEvent) if !ok { continue // invalid event value } diff --git a/net/dag_test.go b/net/dag_test.go index 9172ebf619..2e4bea3e0e 100644 --- a/net/dag_test.go +++ b/net/dag_test.go @@ -141,23 +141,23 @@ func TestSendJobWorker_WithPeer_NoError(t *testing.T) { db1, n1 := newTestNode(ctx, t) db2, n2 := newTestNode(ctx, t) - sub1 := db1.Events().Subscribe(5, events.PeerEventName) + sub1 := db1.Events().Subscribe(5, events.ConnectEventName) defer db1.Events().Unsubscribe(sub1) - sub2 := db2.Events().Subscribe(5, events.PeerEventName) + sub2 := db2.Events().Subscribe(5, events.ConnectEventName) defer db2.Events().Unsubscribe(sub2) addrs, err := netutils.ParsePeers([]string{n1.host.Addrs()[0].String() + "/p2p/" + n1.PeerID().String()}) require.NoError(t, err) n2.Bootstrap(addrs) - event1 := <-sub1.Value() - assert.Equal(t, network.Connected, event1.(event.EvtPeerConnectednessChanged).Connectedness) - assert.Equal(t, n2.PeerID(), event1.(event.EvtPeerConnectednessChanged).Peer) + msg1 := <-sub1.Message() + assert.Equal(t, network.Connected, msg1.Data.(event.EvtPeerConnectednessChanged).Connectedness) + assert.Equal(t, n2.PeerID(), msg1.Data.(event.EvtPeerConnectednessChanged).Peer) - event2 := <-sub2.Value() - assert.Equal(t, network.Connected, event2.(event.EvtPeerConnectednessChanged).Connectedness) - assert.Equal(t, n1.PeerID(), event2.(event.EvtPeerConnectednessChanged).Peer) + msg2 := <-sub2.Message() + assert.Equal(t, network.Connected, msg2.Data.(event.EvtPeerConnectednessChanged).Connectedness) + assert.Equal(t, n1.PeerID(), msg2.Data.(event.EvtPeerConnectednessChanged).Peer) done := make(chan struct{}) go func() { diff --git a/net/node.go b/net/node.go index be7c84cf80..a80155bb25 100644 --- a/net/node.go +++ b/net/node.go @@ -160,13 +160,7 @@ func NewNode( } } - // slice of events to subscribe to - subEvents := []any{ - &event.EvtPeerConnectednessChanged{}, - &events.PubSubEvent{}, - &events.PushLogEvent{}, - } - sub, err := h.EventBus().Subscribe(subEvents) + sub, err := h.EventBus().Subscribe(&event.EvtPeerConnectednessChanged{}) if err != nil { return nil, fin.Cleanup(err) } @@ -189,16 +183,7 @@ func NewNode( // publish subscribed events to the event bus go func() { for val := range sub.Out() { - switch t := val.(type) { - case event.EvtPeerConnectednessChanged: - db.Events().Publish(events.PeerEventName, events.PeerEvent(t)) - - case events.PubSubEvent: - db.Events().Publish(events.PubSubEventName, t) - - case events.PushLogEvent: - db.Events().Publish(events.PushLogEventName, t) - } + db.Events().Publish(events.NewMessage(events.ConnectEventName, val)) } }() diff --git a/net/peer.go b/net/peer.go index 29eccca1ad..9f60408cbc 100644 --- a/net/peer.go +++ b/net/peer.go @@ -221,11 +221,11 @@ func (p *Peer) Close() { // from the internal broadcaster to the external pubsub network func (p *Peer) handleBroadcastLoop() { for { - value, isOpen := <-p.updateSub.Value() + msg, isOpen := <-p.updateSub.Message() if !isOpen { return } - update, ok := value.(events.UpdateEvent) + update, ok := msg.Data.(events.UpdateEvent) if !ok { continue // ignore invalid value } diff --git a/net/server.go b/net/server.go index cd7428ae4d..131b6aa200 100644 --- a/net/server.go +++ b/net/server.go @@ -198,10 +198,11 @@ func (s *server) PushLog(ctx context.Context, req *pb.PushLogRequest) (*pb.PushL if err != nil { log.InfoContext(ctx, "could not decode the PeerID of the log creator", corelog.String("Error", err.Error())) } - s.db.Events().Publish(events.PushLogEventName, events.PushLogEvent{ + msg := events.NewMessage(events.PushLogEventName, events.PushLogEvent{ FromPeer: pid, ByPeer: byPeer, }) + s.db.Events().Publish(msg) }() // make sure were not processing twice @@ -491,9 +492,10 @@ func (s *server) pubSubEventHandler(from libpeer.ID, topic string, msg []byte) { corelog.String("Topic", topic), corelog.String("Message", string(msg)), ) - s.db.Events().Publish(events.PubSubEventName, events.PubSubEvent{ + evt := events.NewMessage(events.PubSubEventName, events.PubSubEvent{ Peer: from, }) + s.db.Events().Publish(evt) } // addr implements net.Addr and holds a libp2p peer ID. From 144b392d3331762b14deb5ef433073be15cfa268 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Fri, 7 Jun 2024 12:00:21 -0700 Subject: [PATCH 05/34] update tests with event bus --- tests/clients/cli/wrapper.go | 8 ------ tests/clients/clients.go | 2 -- tests/clients/http/wrapper.go | 8 ------ tests/integration/events/utils.go | 4 +-- tests/integration/net/order/utils.go | 30 +++++++-------------- tests/integration/p2p.go | 39 +++++++++++++++++++++------- tests/integration/state.go | 5 ++++ tests/integration/utils2.go | 23 ++++++++++++++-- 8 files changed, 67 insertions(+), 52 deletions(-) diff --git a/tests/clients/cli/wrapper.go b/tests/clients/cli/wrapper.go index 289d0435ae..de7d0f82cd 100644 --- a/tests/clients/cli/wrapper.go +++ b/tests/clients/cli/wrapper.go @@ -540,11 +540,3 @@ func (w *Wrapper) PrintDump(ctx context.Context) error { func (w *Wrapper) Bootstrap(addrs []peer.AddrInfo) { w.node.Bootstrap(addrs) } - -func (w *Wrapper) WaitForPushLogByPeerEvent(id peer.ID) error { - return w.node.WaitForPushLogByPeerEvent(id) -} - -func (w *Wrapper) WaitForPushLogFromPeerEvent(id peer.ID) error { - return w.node.WaitForPushLogFromPeerEvent(id) -} diff --git a/tests/clients/clients.go b/tests/clients/clients.go index 10df14212f..249b1e767f 100644 --- a/tests/clients/clients.go +++ b/tests/clients/clients.go @@ -21,6 +21,4 @@ import ( type Client interface { client.P2P Bootstrap([]peer.AddrInfo) - WaitForPushLogByPeerEvent(peer.ID) error - WaitForPushLogFromPeerEvent(peer.ID) error } diff --git a/tests/clients/http/wrapper.go b/tests/clients/http/wrapper.go index 5c9e76b3ca..81f79013fc 100644 --- a/tests/clients/http/wrapper.go +++ b/tests/clients/http/wrapper.go @@ -236,11 +236,3 @@ func (w *Wrapper) PrintDump(ctx context.Context) error { func (w *Wrapper) Bootstrap(addrs []peer.AddrInfo) { w.node.Bootstrap(addrs) } - -func (w *Wrapper) WaitForPushLogByPeerEvent(id peer.ID) error { - return w.node.WaitForPushLogByPeerEvent(id) -} - -func (w *Wrapper) WaitForPushLogFromPeerEvent(id peer.ID) error { - return w.node.WaitForPushLogFromPeerEvent(id) -} diff --git a/tests/integration/events/utils.go b/tests/integration/events/utils.go index bc2972faf3..3c5a8e940d 100644 --- a/tests/integration/events/utils.go +++ b/tests/integration/events/utils.go @@ -87,8 +87,8 @@ func ExecuteRequestTestCase( go func() { for { select { - case value := <-eventsSub.Value(): - update, ok := value.(events.UpdateEvent) + case value := <-eventsSub.Message(): + update, ok := value.Data.(events.UpdateEvent) if !ok { continue // ignore invaid value } diff --git a/tests/integration/net/order/utils.go b/tests/integration/net/order/utils.go index c7075dae22..e06ead8c26 100644 --- a/tests/integration/net/order/utils.go +++ b/tests/integration/net/order/utils.go @@ -21,6 +21,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/errors" + "github.com/sourcenetwork/defradb/events" "github.com/sourcenetwork/defradb/net" netutils "github.com/sourcenetwork/defradb/net/utils" testutils "github.com/sourcenetwork/defradb/tests/integration" @@ -210,6 +211,7 @@ func executeTestCase(t *testing.T, test P2PTestCase) { docIDs := []client.DocID{} nodes := []*net.Node{} + subs := []*events.Subscription{} for i, cfg := range test.NodeConfig { log.InfoContext(ctx, fmt.Sprintf("Setting up node %d", i)) @@ -239,27 +241,13 @@ func executeTestCase(t *testing.T, test P2PTestCase) { docIDs = d } nodes = append(nodes, n) + subs = append(subs, n.Events().Subscribe(100, events.PushLogEventName)) } ////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////// // PubSub related test logic - // wait for peers to connect to each other - if len(test.NodePeers) > 0 { - for i, n := range nodes { - for j, p := range nodes { - if i == j { - continue - } - log.InfoContext(ctx, fmt.Sprintf("Waiting for node %d to connect with peer %d", i, j)) - err := n.WaitForPubSubEvent(p.PeerID()) - require.NoError(t, err) - log.InfoContext(ctx, fmt.Sprintf("Node %d connected to peer %d", i, j)) - } - } - } - // update and sync peers for n, updateMap := range test.Updates { if n >= len(nodes) { @@ -279,13 +267,14 @@ func executeTestCase(t *testing.T, test P2PTestCase) { require.NoError(t, err) // wait for peers to sync - for n2, p := range nodes { + for n2 := range nodes { if n2 == n { continue } log.InfoContext(ctx, fmt.Sprintf("Waiting for node %d to sync with peer %d", n2, n)) - err := p.WaitForPushLogByPeerEvent(nodes[n].PeerInfo().ID) - require.NoError(t, err) + msg, ok := <-subs[n2].Message() + require.True(t, ok) + assert.Equal(t, nodes[n].PeerID(), msg.Data.(events.PushLogEvent).ByPeer) log.InfoContext(ctx, fmt.Sprintf("Node %d synced", n2)) } } @@ -348,8 +337,9 @@ func executeTestCase(t *testing.T, test P2PTestCase) { } for _, rep := range reps { log.InfoContext(ctx, fmt.Sprintf("Waiting for node %d to sync with peer %d", rep, n)) - err := nodes[rep].WaitForPushLogByPeerEvent(nodes[n].PeerID()) - require.NoError(t, err) + msg, ok := <-subs[rep].Message() + require.True(t, ok) + assert.Equal(t, nodes[n].PeerID(), msg.Data.(events.PushLogEvent).ByPeer) log.InfoContext(ctx, fmt.Sprintf("Node %d synced", rep)) for docID, results := range test.ReplicatorResult[rep] { diff --git a/tests/integration/p2p.go b/tests/integration/p2p.go index 0cace429ae..d18a72ecf6 100644 --- a/tests/integration/p2p.go +++ b/tests/integration/p2p.go @@ -14,6 +14,7 @@ import ( "time" "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/events" "github.com/sourcenetwork/defradb/net" "github.com/sourcenetwork/defradb/tests/clients" @@ -154,6 +155,9 @@ func connectPeers( sourceNode := s.nodes[cfg.SourceNodeID] targetNode := s.nodes[cfg.TargetNodeID] + sourceSub := s.eventSubs[cfg.SourceNodeID] + targetSub := s.eventSubs[cfg.TargetNodeID] + addrs := []peer.AddrInfo{targetNode.PeerInfo()} log.InfoContext(s.ctx, "Bootstrapping with peers", corelog.Any("Addresses", addrs)) sourceNode.Bootstrap(addrs) @@ -162,7 +166,7 @@ func connectPeers( // allowed to complete before documentation begins or it will not even try and sync it. So for now, we // sleep a little. time.Sleep(100 * time.Millisecond) - setupPeerWaitSync(s, 0, cfg, sourceNode, targetNode) + setupPeerWaitSync(s, 0, cfg, sourceNode, targetNode, sourceSub, targetSub) } func setupPeerWaitSync( @@ -171,6 +175,8 @@ func setupPeerWaitSync( cfg ConnectPeers, sourceNode clients.Client, targetNode clients.Client, + sourceSub *events.Subscription, + targetSub *events.Subscription, ) { sourceToTargetEvents := []int{0} targetToSourceEvents := []int{0} @@ -263,12 +269,16 @@ func setupPeerWaitSync( ready <- struct{}{} for waitIndex := 0; waitIndex < len(sourceToTargetEvents); waitIndex++ { for i := 0; i < targetToSourceEvents[waitIndex]; i++ { - err := sourceNode.WaitForPushLogByPeerEvent(targetPeerInfo.ID) - require.NoError(s.t, err) + msg, ok := <-sourceSub.Message() + if ok { + assert.Equal(s.t, targetPeerInfo.ID, msg.Data.(events.PushLogEvent).ByPeer) + } } for i := 0; i < sourceToTargetEvents[waitIndex]; i++ { - err := targetNode.WaitForPushLogByPeerEvent(sourcePeerInfo.ID) - require.NoError(s.t, err) + msg, ok := <-targetSub.Message() + if ok { + assert.Equal(s.t, sourcePeerInfo.ID, msg.Data.(events.PushLogEvent).ByPeer) + } } nodeSynced <- struct{}{} } @@ -310,6 +320,9 @@ func configureReplicator( sourceNode := s.nodes[cfg.SourceNodeID] targetNode := s.nodes[cfg.TargetNodeID] + sourceSub := s.eventSubs[cfg.SourceNodeID] + targetSub := s.eventSubs[cfg.TargetNodeID] + err := sourceNode.SetReplicator(s.ctx, client.Replicator{ Info: targetNode.PeerInfo(), }) @@ -317,7 +330,7 @@ func configureReplicator( expectedErrorRaised := AssertError(s.t, s.testCase.Description, err, cfg.ExpectedError) assertExpectedErrorRaised(s.t, s.testCase.Description, cfg.ExpectedError, expectedErrorRaised) if err == nil { - setupReplicatorWaitSync(s, 0, cfg, sourceNode, targetNode) + setupReplicatorWaitSync(s, 0, cfg, sourceNode, targetNode, sourceSub, targetSub) } } @@ -340,6 +353,8 @@ func setupReplicatorWaitSync( cfg ConfigureReplicator, sourceNode clients.Client, targetNode clients.Client, + sourceSub *events.Subscription, + targetSub *events.Subscription, ) { sourceToTargetEvents := []int{0} targetToSourceEvents := []int{0} @@ -398,12 +413,16 @@ func setupReplicatorWaitSync( ready <- struct{}{} for waitIndex := 0; waitIndex < len(sourceToTargetEvents); waitIndex++ { for i := 0; i < targetToSourceEvents[waitIndex]; i++ { - err := sourceNode.WaitForPushLogByPeerEvent(targetPeerInfo.ID) - require.NoError(s.t, err) + msg, ok := <-sourceSub.Message() + if ok { + assert.Equal(s.t, targetPeerInfo.ID, msg.Data.(events.PushLogEvent).ByPeer) + } } for i := 0; i < sourceToTargetEvents[waitIndex]; i++ { - err := targetNode.WaitForPushLogByPeerEvent(sourcePeerInfo.ID) - require.NoError(s.t, err) + msg, ok := <-targetSub.Message() + if ok { + assert.Equal(s.t, sourcePeerInfo.ID, msg.Data.(events.PushLogEvent).ByPeer) + } } nodeSynced <- struct{}{} } diff --git a/tests/integration/state.go b/tests/integration/state.go index 49030c82a6..0b5bc63ab8 100644 --- a/tests/integration/state.go +++ b/tests/integration/state.go @@ -18,6 +18,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/datastore" + "github.com/sourcenetwork/defradb/events" "github.com/sourcenetwork/defradb/net" "github.com/sourcenetwork/defradb/tests/clients" ) @@ -52,6 +53,9 @@ type state struct { // These synchronisation channels allow async actions to track their completion. syncChans []chan struct{} + // eventSubs is a list of all event subscriptions + eventSubs []*events.Subscription + // The addresses of any nodes configured. nodeAddresses []peer.AddrInfo @@ -104,6 +108,7 @@ func newState( allActionsDone: make(chan struct{}), subscriptionResultsChans: []chan func(){}, syncChans: []chan struct{}{}, + eventSubs: []*events.Subscription{}, nodeAddresses: []peer.AddrInfo{}, nodeConfigs: [][]net.NodeOpt{}, nodes: []clients.Client{}, diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 041b553548..b865a9db1f 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -32,6 +32,7 @@ import ( "github.com/sourcenetwork/defradb/datastore" badgerds "github.com/sourcenetwork/defradb/datastore/badger/v4" "github.com/sourcenetwork/defradb/errors" + "github.com/sourcenetwork/defradb/events" "github.com/sourcenetwork/defradb/internal/db" "github.com/sourcenetwork/defradb/internal/request/graphql" "github.com/sourcenetwork/defradb/net" @@ -656,6 +657,9 @@ func setStartingNodes( s.nodes = append(s.nodes, c) s.dbPaths = append(s.dbPaths, path) + + sub := c.Events().Subscribe(100, events.PushLogEventName) + s.eventSubs = append(s.eventSubs, sub) } } @@ -729,13 +733,25 @@ actionLoop: // Give the nodes a chance to connect to each other and learn about each other's subscribed topics. time.Sleep(100 * time.Millisecond) setupPeerWaitSync( - s, waitGroupStartIndex, action, s.nodes[action.SourceNodeID], s.nodes[action.TargetNodeID], + s, + waitGroupStartIndex, + action, + s.nodes[action.SourceNodeID], + s.nodes[action.TargetNodeID], + s.eventSubs[action.SourceNodeID], + s.eventSubs[action.TargetNodeID], ) case ConfigureReplicator: // Give the nodes a chance to connect to each other and learn about each other's subscribed topics. time.Sleep(100 * time.Millisecond) setupReplicatorWaitSync( - s, waitGroupStartIndex, action, s.nodes[action.SourceNodeID], s.nodes[action.TargetNodeID], + s, + waitGroupStartIndex, + action, + s.nodes[action.SourceNodeID], + s.nodes[action.TargetNodeID], + s.eventSubs[action.SourceNodeID], + s.eventSubs[action.TargetNodeID], ) } } @@ -812,6 +828,9 @@ func configureNode( s.nodes = append(s.nodes, c) s.dbPaths = append(s.dbPaths, path) + + sub := c.Events().Subscribe(100, events.PushLogEventName) + s.eventSubs = append(s.eventSubs, sub) } func refreshDocuments( From 6b8d97e896cd2d85c8d745dbc9cdb3881a8ec82c Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Fri, 7 Jun 2024 13:15:58 -0700 Subject: [PATCH 06/34] merge push log and dag merge events --- events/events.go | 27 ++++++++++---------------- internal/db/merge.go | 11 ++++------- net/process.go | 2 +- net/server.go | 38 +++++++++++++++---------------------- tests/integration/p2p.go | 8 ++++---- tests/integration/utils2.go | 4 ++-- 6 files changed, 36 insertions(+), 54 deletions(-) diff --git a/events/events.go b/events/events.go index b0d3f046da..54a43b8fe4 100644 --- a/events/events.go +++ b/events/events.go @@ -11,8 +11,6 @@ package events import ( - "sync" - "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/core/event" "github.com/libp2p/go-libp2p/core/peer" @@ -21,14 +19,14 @@ import ( const ( // WildCardEventName is the alias used to subscribe to all events. WildCardEventName = "*" - // MergeEventName is the name of the databse merge event. - MergeEventName = "db:merge" + // MergeCompleteEventName is the name of the database merge complete event. + MergeCompleteEventName = "db:merge-complete" // UpdateEventName is the name of the database update event. UpdateEventName = "db:update" // ResultsEventName is the name of the database results event. ResultsEventName = "db:results" - // PushLogEventName is the name of the network pushlog event. - PushLogEventName = "net:pushlog" + // MergeRequestEventName is the name of the net merge request event. + MergeRequestEventName = "net:merge" // PubSubEventName is the name of the network pubsub event. PubSubEventName = "net:pubsub" // ConnectEventName is the name of the network connect event. @@ -39,13 +37,6 @@ const ( // a peer connection has changed status. type ConnectEvent = event.EvtPeerConnectednessChanged -// PushLogEvent is an event that is published when -// a pushlog message has been received from a remote peer. -type PushLogEvent struct { - ByPeer peer.ID - FromPeer peer.ID -} - // PubSubEvent is an event that is published when // a pubsub message has been received from a remote peer. type PubSubEvent struct { @@ -76,13 +67,15 @@ type UpdateEvent struct { // MergeEvent is a notification that a merge can be performed up to the provided CID. type MergeEvent struct { + // ByPeer is the id of the peer that created the push log request. + ByPeer peer.ID + + // FromPeer is the id of the peer that received the push log request. + FromPeer peer.ID + // Cid is the id of the composite commit that formed this update in the DAG. Cid cid.Cid // SchemaRoot is the root identifier of the schema that defined the shape of the document that was updated. SchemaRoot string - - // Wg is a wait group that can be used to synchronize the merge, - // allowing the caller to optionnaly block until the merge is complete. - Wg *sync.WaitGroup } diff --git a/internal/db/merge.go b/internal/db/merge.go index 82a0fa26a1..15d7ecf49e 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -34,7 +34,7 @@ import ( ) func (db *db) handleMerges(ctx context.Context) { - sub := db.sysEventBus.Subscribe(mergeSubBufferSize, events.MergeEventName) + sub := db.sysEventBus.Subscribe(mergeSubBufferSize, events.MergeRequestEventName) defer db.sysEventBus.Unsubscribe(sub) for { @@ -66,12 +66,9 @@ func (db *db) handleMerges(ctx context.Context) { } func (db *db) executeMerge(ctx context.Context, dagMerge events.MergeEvent) error { - defer func() { - // Notify the caller that the merge is complete. - if dagMerge.Wg != nil { - dagMerge.Wg.Done() - } - }() + // send a complete event so we can track merges in the integration tests + defer db.sysEventBus.Publish(events.NewMessage(events.MergeCompleteEventName, dagMerge)) + ctx, txn, err := ensureContextTxn(ctx, db, false) if err != nil { return err diff --git a/net/process.go b/net/process.go index b4f85134fb..cb063d6a3a 100644 --- a/net/process.go +++ b/net/process.go @@ -64,7 +64,7 @@ func (bp *blockProcessor) processRemoteBlock( // Initiate a sync of the block's children bp.wg.Add(1) bp.handleChildBlocks(ctx, block) - + bp.wg.Wait() return nil } diff --git a/net/server.go b/net/server.go index c59e944161..0da2bfc79e 100644 --- a/net/server.go +++ b/net/server.go @@ -184,20 +184,20 @@ func (s *server) PushLog(ctx context.Context, req *pb.PushLogRequest) (*pb.PushL if err != nil { return nil, err } + byPeer, err := libpeer.Decode(req.Body.Creator) + if err != nil { + return nil, err + } s.docQueue.add(docID.String()) - defer func() { - s.docQueue.done(docID.String()) - byPeer, err := libpeer.Decode(req.Body.Creator) - if err != nil { - log.InfoContext(ctx, "could not decode the PeerID of the log creator", corelog.String("Error", err.Error())) - } - msg := events.NewMessage(events.PushLogEventName, events.PushLogEvent{ - FromPeer: pid, - ByPeer: byPeer, - }) - s.peer.db.Events().Publish(msg) - }() + defer s.docQueue.done(docID.String()) + + evt := events.MergeEvent{ + ByPeer: byPeer, + FromPeer: pid, + Cid: cid, + SchemaRoot: string(req.Body.SchemaRoot), + } // check if we already have this block exists, err := s.peer.db.Blockstore().Has(ctx, cid) @@ -205,6 +205,8 @@ func (s *server) PushLog(ctx context.Context, req *pb.PushLogRequest) (*pb.PushL return nil, NewErrCheckingForExistingBlock(err, cid.String()) } if exists { + // the integration tests expect every push log to emit a merge complete event + s.peer.db.Events().Publish(events.NewMessage(events.MergeCompleteEventName, evt)) return &pb.PushLogReply{}, nil } @@ -224,17 +226,7 @@ func (s *server) PushLog(ctx context.Context, req *pb.PushLogRequest) (*pb.PushL corelog.Any("CID", cid), ) } - bp.wg.Wait() - - wg := &sync.WaitGroup{} - wg.Add(1) - evt := events.NewMessage(events.MergeEventName, events.MergeEvent{ - Cid: cid, - SchemaRoot: string(req.Body.SchemaRoot), - Wg: wg, - }) - s.peer.db.Events().Publish(evt) - wg.Wait() + s.peer.db.Events().Publish(events.NewMessage(events.MergeRequestEventName, evt)) // Once processed, subscribe to the DocID topic on the pubsub network unless we already // suscribe to the collection. diff --git a/tests/integration/p2p.go b/tests/integration/p2p.go index d18a72ecf6..20c77e8074 100644 --- a/tests/integration/p2p.go +++ b/tests/integration/p2p.go @@ -271,13 +271,13 @@ func setupPeerWaitSync( for i := 0; i < targetToSourceEvents[waitIndex]; i++ { msg, ok := <-sourceSub.Message() if ok { - assert.Equal(s.t, targetPeerInfo.ID, msg.Data.(events.PushLogEvent).ByPeer) + assert.Equal(s.t, targetPeerInfo.ID, msg.Data.(events.MergeEvent).ByPeer) } } for i := 0; i < sourceToTargetEvents[waitIndex]; i++ { msg, ok := <-targetSub.Message() if ok { - assert.Equal(s.t, sourcePeerInfo.ID, msg.Data.(events.PushLogEvent).ByPeer) + assert.Equal(s.t, sourcePeerInfo.ID, msg.Data.(events.MergeEvent).ByPeer) } } nodeSynced <- struct{}{} @@ -415,13 +415,13 @@ func setupReplicatorWaitSync( for i := 0; i < targetToSourceEvents[waitIndex]; i++ { msg, ok := <-sourceSub.Message() if ok { - assert.Equal(s.t, targetPeerInfo.ID, msg.Data.(events.PushLogEvent).ByPeer) + assert.Equal(s.t, targetPeerInfo.ID, msg.Data.(events.MergeEvent).ByPeer) } } for i := 0; i < sourceToTargetEvents[waitIndex]; i++ { msg, ok := <-targetSub.Message() if ok { - assert.Equal(s.t, sourcePeerInfo.ID, msg.Data.(events.PushLogEvent).ByPeer) + assert.Equal(s.t, sourcePeerInfo.ID, msg.Data.(events.MergeEvent).ByPeer) } } nodeSynced <- struct{}{} diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index b865a9db1f..8e3b1744b3 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -658,7 +658,7 @@ func setStartingNodes( s.nodes = append(s.nodes, c) s.dbPaths = append(s.dbPaths, path) - sub := c.Events().Subscribe(100, events.PushLogEventName) + sub := c.Events().Subscribe(100, events.MergeCompleteEventName) s.eventSubs = append(s.eventSubs, sub) } } @@ -829,7 +829,7 @@ func configureNode( s.nodes = append(s.nodes, c) s.dbPaths = append(s.dbPaths, path) - sub := c.Events().Subscribe(100, events.PushLogEventName) + sub := c.Events().Subscribe(100, events.MergeCompleteEventName) s.eventSubs = append(s.eventSubs, sub) } From 98e77d406ad04f4e34a50af137f1761b446d42b0 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 11 Jun 2024 09:00:00 -0700 Subject: [PATCH 07/34] rename events package to event --- client/db.go | 4 ++-- client/mocks/db.go | 2 +- {events => event}/bus.go | 2 +- {events => event}/bus_test.go | 2 +- {events => event}/events.go | 2 +- http/client.go | 4 ++-- internal/db/collection.go | 6 +++--- internal/db/collection_delete.go | 6 +++--- internal/db/db.go | 14 +++++++------- internal/db/merge.go | 10 +++++----- internal/db/merge_test.go | 12 ++++++------ internal/db/subscriptions.go | 8 ++++---- net/client.go | 4 ++-- net/client_test.go | 8 ++++---- net/node.go | 4 ++-- net/peer.go | 16 ++++++++-------- net/peer_test.go | 16 ++++++++-------- net/server.go | 10 +++++----- tests/clients/cli/wrapper.go | 4 ++-- tests/clients/http/wrapper.go | 4 ++-- tests/integration/events/utils.go | 6 +++--- tests/integration/p2p.go | 18 +++++++++--------- tests/integration/state.go | 6 +++--- tests/integration/utils2.go | 6 +++--- 24 files changed, 87 insertions(+), 87 deletions(-) rename {events => event}/bus.go (99%) rename {events => event}/bus_test.go (98%) rename {events => event}/events.go (99%) diff --git a/client/db.go b/client/db.go index d83a0fc2ac..e52dfed60a 100644 --- a/client/db.go +++ b/client/db.go @@ -18,7 +18,7 @@ import ( "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/defradb/datastore" - "github.com/sourcenetwork/defradb/events" + "github.com/sourcenetwork/defradb/event" ) type CollectionName = string @@ -75,7 +75,7 @@ type DB interface { // // It may be used to monitor database events - a new event will be yielded for each mutation. // Note: it does not copy the queue, just the reference to it. - Events() *events.Bus + Events() *event.Bus // MaxTxnRetries returns the number of retries that this DefraDB instance has been configured to // make in the event of a transaction conflict in certain scenarios. diff --git a/client/mocks/db.go b/client/mocks/db.go index 5652361118..7867e90637 100644 --- a/client/mocks/db.go +++ b/client/mocks/db.go @@ -9,7 +9,7 @@ import ( datastore "github.com/sourcenetwork/defradb/datastore" - events "github.com/sourcenetwork/defradb/events" + events "github.com/sourcenetwork/defradb/event" go_datastore "github.com/ipfs/go-datastore" diff --git a/events/bus.go b/event/bus.go similarity index 99% rename from events/bus.go rename to event/bus.go index b3decac9f6..275cfac688 100644 --- a/events/bus.go +++ b/event/bus.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package events +package event import ( "sync" diff --git a/events/bus_test.go b/event/bus_test.go similarity index 98% rename from events/bus_test.go rename to event/bus_test.go index f96fc3d36f..8eed3d57e0 100644 --- a/events/bus_test.go +++ b/event/bus_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package events +package event import ( "testing" diff --git a/events/events.go b/event/events.go similarity index 99% rename from events/events.go rename to event/events.go index 54a43b8fe4..148dfeffc2 100644 --- a/events/events.go +++ b/event/events.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package events +package event import ( "github.com/ipfs/go-cid" diff --git a/http/client.go b/http/client.go index 89f1578bb9..2843ee4f2d 100644 --- a/http/client.go +++ b/http/client.go @@ -28,7 +28,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/datastore" - "github.com/sourcenetwork/defradb/events" + "github.com/sourcenetwork/defradb/event" ) var _ client.DB = (*Client)(nil) @@ -451,7 +451,7 @@ func (c *Client) Headstore() ds.Read { panic("client side database") } -func (c *Client) Events() *events.Bus { +func (c *Client) Events() *event.Bus { panic("client side database") } diff --git a/internal/db/collection.go b/internal/db/collection.go index 143d3a275b..4ea3b9a76b 100644 --- a/internal/db/collection.go +++ b/internal/db/collection.go @@ -27,7 +27,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/client/request" "github.com/sourcenetwork/defradb/errors" - "github.com/sourcenetwork/defradb/events" + "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/internal/core" coreblock "github.com/sourcenetwork/defradb/internal/core/block" "github.com/sourcenetwork/defradb/internal/db/base" @@ -677,7 +677,7 @@ func (c *collection) save( } // publish an update event when the txn succeeds - updateEvent := events.UpdateEvent{ + updateEvent := event.UpdateEvent{ DocID: doc.ID().String(), Cid: link.Cid, SchemaRoot: c.Schema().Root, @@ -685,7 +685,7 @@ func (c *collection) save( IsCreate: isCreate, } txn.OnSuccess(func() { - c.db.sysEventBus.Publish(events.NewMessage(events.UpdateEventName, updateEvent)) + c.db.sysEventBus.Publish(event.NewMessage(event.UpdateEventName, updateEvent)) }) txn.OnSuccess(func() { diff --git a/internal/db/collection_delete.go b/internal/db/collection_delete.go index eb75e29372..fc98bec5a7 100644 --- a/internal/db/collection_delete.go +++ b/internal/db/collection_delete.go @@ -17,7 +17,7 @@ import ( "github.com/sourcenetwork/defradb/acp" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/events" + "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/internal/core" coreblock "github.com/sourcenetwork/defradb/internal/core/block" "github.com/sourcenetwork/defradb/internal/merkle/clock" @@ -167,14 +167,14 @@ func (c *collection) applyDelete( } // publish an update event if the txn succeeds - updateEvent := events.UpdateEvent{ + updateEvent := event.UpdateEvent{ DocID: primaryKey.DocID, Cid: link.Cid, SchemaRoot: c.Schema().Root, Block: b, } txn.OnSuccess(func() { - c.db.sysEventBus.Publish(events.NewMessage(events.UpdateEventName, updateEvent)) + c.db.sysEventBus.Publish(event.NewMessage(event.UpdateEventName, updateEvent)) }) return nil diff --git a/internal/db/db.go b/internal/db/db.go index 32d0e01eb8..df4f6f4b74 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -29,7 +29,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/errors" - "github.com/sourcenetwork/defradb/events" + "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/internal/core" "github.com/sourcenetwork/defradb/internal/request/graphql" ) @@ -64,10 +64,10 @@ type db struct { multistore datastore.MultiStore // sysEventBus is used to send and receive system events - sysEventBus *events.Bus + sysEventBus *event.Bus // subEventBus is used to send and receive request subscription events - subEventBus *events.Bus + subEventBus *event.Bus parser core.Parser @@ -118,8 +118,8 @@ func newDB( lensRegistry: lens, parser: parser, options: options, - sysEventBus: events.NewBus(), - subEventBus: events.NewBus(), + sysEventBus: event.NewBus(), + subEventBus: event.NewBus(), } // apply options @@ -207,7 +207,7 @@ func (db *db) initialize(ctx context.Context) error { // forward system bus events to the subscriber bus // to ensure we never block the system bus for user subscriptions go func() { - sub := db.sysEventBus.Subscribe(forwardSubBufferSize, events.WildCardEventName) + sub := db.sysEventBus.Subscribe(forwardSubBufferSize, event.WildCardEventName) for msg := range sub.Message() { db.subEventBus.Publish(msg) } @@ -266,7 +266,7 @@ func (db *db) initialize(ctx context.Context) error { } // Events returns the events Channel. -func (db *db) Events() *events.Bus { +func (db *db) Events() *event.Bus { return db.sysEventBus } diff --git a/internal/db/merge.go b/internal/db/merge.go index 15d7ecf49e..eab5b7d778 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -25,7 +25,7 @@ import ( "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/datastore/badger/v4" "github.com/sourcenetwork/defradb/errors" - "github.com/sourcenetwork/defradb/events" + "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/internal/core" coreblock "github.com/sourcenetwork/defradb/internal/core/block" "github.com/sourcenetwork/defradb/internal/db/base" @@ -34,7 +34,7 @@ import ( ) func (db *db) handleMerges(ctx context.Context) { - sub := db.sysEventBus.Subscribe(mergeSubBufferSize, events.MergeRequestEventName) + sub := db.sysEventBus.Subscribe(mergeSubBufferSize, event.MergeRequestEventName) defer db.sysEventBus.Unsubscribe(sub) for { @@ -45,7 +45,7 @@ func (db *db) handleMerges(ctx context.Context) { if !ok { return } - merge, ok := msg.Data.(events.MergeEvent) + merge, ok := msg.Data.(event.MergeEvent) if !ok { continue } @@ -65,9 +65,9 @@ func (db *db) handleMerges(ctx context.Context) { } } -func (db *db) executeMerge(ctx context.Context, dagMerge events.MergeEvent) error { +func (db *db) executeMerge(ctx context.Context, dagMerge event.MergeEvent) error { // send a complete event so we can track merges in the integration tests - defer db.sysEventBus.Publish(events.NewMessage(events.MergeCompleteEventName, dagMerge)) + defer db.sysEventBus.Publish(event.NewMessage(event.MergeCompleteEventName, dagMerge)) ctx, txn, err := ensureContextTxn(ctx, db, false) if err != nil { diff --git a/internal/db/merge_test.go b/internal/db/merge_test.go index 63879cbc15..624b2d97aa 100644 --- a/internal/db/merge_test.go +++ b/internal/db/merge_test.go @@ -21,7 +21,7 @@ import ( "github.com/stretchr/testify/require" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/events" + "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/internal/core" coreblock "github.com/sourcenetwork/defradb/internal/core/block" "github.com/sourcenetwork/defradb/internal/core/crdt" @@ -58,7 +58,7 @@ func TestMerge_SingleBranch_NoError(t *testing.T) { compInfo2, err := d.generateCompositeUpdate(&lsys, map[string]any{"name": "Johny"}, compInfo) require.NoError(t, err) - err = db.executeMerge(ctx, events.MergeEvent{ + err = db.executeMerge(ctx, event.MergeEvent{ Cid: compInfo2.link.Cid, SchemaRoot: col.SchemaRoot(), }) @@ -102,7 +102,7 @@ func TestMerge_DualBranch_NoError(t *testing.T) { compInfo2, err := d.generateCompositeUpdate(&lsys, map[string]any{"name": "Johny"}, compInfo) require.NoError(t, err) - err = db.executeMerge(ctx, events.MergeEvent{ + err = db.executeMerge(ctx, event.MergeEvent{ Cid: compInfo2.link.Cid, SchemaRoot: col.SchemaRoot(), }) @@ -111,7 +111,7 @@ func TestMerge_DualBranch_NoError(t *testing.T) { compInfo3, err := d.generateCompositeUpdate(&lsys, map[string]any{"age": 30}, compInfo) require.NoError(t, err) - err = db.executeMerge(ctx, events.MergeEvent{ + err = db.executeMerge(ctx, event.MergeEvent{ Cid: compInfo3.link.Cid, SchemaRoot: col.SchemaRoot(), }) @@ -158,7 +158,7 @@ func TestMerge_DualBranchWithOneIncomplete_CouldNotFindCID(t *testing.T) { compInfo2, err := d.generateCompositeUpdate(&lsys, map[string]any{"name": "Johny"}, compInfo) require.NoError(t, err) - err = db.executeMerge(ctx, events.MergeEvent{ + err = db.executeMerge(ctx, event.MergeEvent{ Cid: compInfo2.link.Cid, SchemaRoot: col.SchemaRoot(), }) @@ -176,7 +176,7 @@ func TestMerge_DualBranchWithOneIncomplete_CouldNotFindCID(t *testing.T) { compInfo3, err := d.generateCompositeUpdate(&lsys, map[string]any{"name": "Johny"}, compInfoUnkown) require.NoError(t, err) - err = db.executeMerge(ctx, events.MergeEvent{ + err = db.executeMerge(ctx, event.MergeEvent{ Cid: compInfo3.link.Cid, SchemaRoot: col.SchemaRoot(), }) diff --git a/internal/db/subscriptions.go b/internal/db/subscriptions.go index bb49a0efa1..4aa2d0c2da 100644 --- a/internal/db/subscriptions.go +++ b/internal/db/subscriptions.go @@ -15,7 +15,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/client/request" - "github.com/sourcenetwork/defradb/events" + "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/internal/planner" ) @@ -32,7 +32,7 @@ func (db *db) handleSubscription(ctx context.Context, r *request.Request) (<-cha return nil, client.NewErrUnexpectedType[request.ObjectSubscription]("SubscriptionSelection", selections) } // subscribe to the subscription event bus so we don't block the system bus - sub := db.subEventBus.Subscribe(requestSubBufferSize, events.UpdateEventName) + sub := db.subEventBus.Subscribe(requestSubBufferSize, event.UpdateEventName) resCh := make(chan client.GQLResult) go func() { defer func() { @@ -42,7 +42,7 @@ func (db *db) handleSubscription(ctx context.Context, r *request.Request) (<-cha // listen for events and send to the result channel for { - var evt events.UpdateEvent + var evt event.UpdateEvent select { case <-ctx.Done(): return // context cancelled @@ -50,7 +50,7 @@ func (db *db) handleSubscription(ctx context.Context, r *request.Request) (<-cha if !ok { return // channel closed } - evt, ok = val.Data.(events.UpdateEvent) + evt, ok = val.Data.(event.UpdateEvent) if !ok { continue // invalid event value } diff --git a/net/client.go b/net/client.go index d774b311da..a59421c3ba 100644 --- a/net/client.go +++ b/net/client.go @@ -19,7 +19,7 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/sourcenetwork/defradb/errors" - "github.com/sourcenetwork/defradb/events" + "github.com/sourcenetwork/defradb/event" pb "github.com/sourcenetwork/defradb/net/pb" ) @@ -31,7 +31,7 @@ var ( // pushLog creates a pushLog request and sends it to another node // over libp2p grpc connection -func (s *server) pushLog(ctx context.Context, evt events.UpdateEvent, pid peer.ID) error { +func (s *server) pushLog(ctx context.Context, evt event.UpdateEvent, pid peer.ID) error { body := &pb.PushLogRequest_Body{ DocID: []byte(evt.DocID), Cid: evt.Cid.Bytes(), diff --git a/net/client_test.go b/net/client_test.go index 55a2c72a51..37bf09ddee 100644 --- a/net/client_test.go +++ b/net/client_test.go @@ -19,7 +19,7 @@ import ( "google.golang.org/grpc" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/events" + "github.com/sourcenetwork/defradb/event" ) var def = client.CollectionDefinition{ @@ -62,7 +62,7 @@ func TestPushlogWithDialFailure(t *testing.T) { grpc.WithCredentialsBundle(nil), ) - err = n.server.pushLog(ctx, events.UpdateEvent{ + err = n.server.pushLog(ctx, event.UpdateEvent{ DocID: id.String(), Cid: cid, SchemaRoot: "test", @@ -84,7 +84,7 @@ func TestPushlogWithInvalidPeerID(t *testing.T) { cid, err := createCID(doc) require.NoError(t, err) - err = n.server.pushLog(ctx, events.UpdateEvent{ + err = n.server.pushLog(ctx, event.UpdateEvent{ DocID: id.String(), Cid: cid, SchemaRoot: "test", @@ -135,7 +135,7 @@ func TestPushlogW_WithValidPeerID_NoError(t *testing.T) { b, err := n1.db.Blockstore().AsIPLDStorage().Get(ctx, headCID.KeyString()) require.NoError(t, err) - err = n1.server.pushLog(ctx, events.UpdateEvent{ + err = n1.server.pushLog(ctx, event.UpdateEvent{ DocID: doc.ID().String(), Cid: headCID, SchemaRoot: col.SchemaRoot(), diff --git a/net/node.go b/net/node.go index e127cf9aec..cb429c71d3 100644 --- a/net/node.go +++ b/net/node.go @@ -47,7 +47,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/crypto" - "github.com/sourcenetwork/defradb/events" + event1 "github.com/sourcenetwork/defradb/event" ) var _ client.P2P = (*Node)(nil) @@ -188,7 +188,7 @@ func NewNode( // publish subscribed events to the event bus go func() { for val := range sub.Out() { - db.Events().Publish(events.NewMessage(events.ConnectEventName, val)) + db.Events().Publish(event1.NewMessage(event1.ConnectEventName, val)) } }() diff --git a/net/peer.go b/net/peer.go index 7da2104107..7c656829c5 100644 --- a/net/peer.go +++ b/net/peer.go @@ -35,7 +35,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/errors" - "github.com/sourcenetwork/defradb/events" + "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/internal/core" corenet "github.com/sourcenetwork/defradb/internal/core/net" "github.com/sourcenetwork/defradb/internal/merkle/clock" @@ -48,7 +48,7 @@ type Peer struct { //config?? db client.DB - updateSub *events.Subscription + updateSub *event.Subscription host host.Host dht routing.Routing @@ -144,7 +144,7 @@ func (p *Peer) Start() error { } if p.ps != nil { - p.updateSub = p.db.Events().Subscribe(100, events.UpdateEventName) + p.updateSub = p.db.Events().Subscribe(100, event.UpdateEventName) log.InfoContext(p.ctx, "Starting internal broadcaster for pubsub network") go p.handleBroadcastLoop() } @@ -203,7 +203,7 @@ func (p *Peer) handleBroadcastLoop() { if !isOpen { return } - update, ok := msg.Data.(events.UpdateEvent) + update, ok := msg.Data.(event.UpdateEvent) if !ok { continue // ignore invalid value } @@ -295,7 +295,7 @@ func (p *Peer) pushToReplicator( continue } - evt := events.UpdateEvent{ + evt := event.UpdateEvent{ DocID: docIDResult.ID.String(), Cid: c, SchemaRoot: collection.SchemaRoot(), @@ -362,7 +362,7 @@ func (p *Peer) loadP2PCollections(ctx context.Context) (map[string]struct{}, err return colMap, nil } -func (p *Peer) handleDocCreateLog(evt events.UpdateEvent) error { +func (p *Peer) handleDocCreateLog(evt event.UpdateEvent) error { docID, err := client.NewDocIDFromString(evt.DocID) if err != nil { return NewErrFailedToGetDocID(err) @@ -380,7 +380,7 @@ func (p *Peer) handleDocCreateLog(evt events.UpdateEvent) error { return nil } -func (p *Peer) handleDocUpdateLog(evt events.UpdateEvent) error { +func (p *Peer) handleDocUpdateLog(evt event.UpdateEvent) error { docID, err := client.NewDocIDFromString(evt.DocID) if err != nil { return NewErrFailedToGetDocID(err) @@ -413,7 +413,7 @@ func (p *Peer) handleDocUpdateLog(evt events.UpdateEvent) error { return nil } -func (p *Peer) pushLogToReplicators(lg events.UpdateEvent) { +func (p *Peer) pushLogToReplicators(lg event.UpdateEvent) { // push to each peer (replicator) peers := make(map[string]struct{}) for _, peer := range p.ps.ListPeers(lg.DocID) { diff --git a/net/peer_test.go b/net/peer_test.go index b3bbea9f47..404651f9d0 100644 --- a/net/peer_test.go +++ b/net/peer_test.go @@ -32,7 +32,7 @@ import ( acpIdentity "github.com/sourcenetwork/defradb/acp/identity" "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/datastore/memory" - "github.com/sourcenetwork/defradb/events" + "github.com/sourcenetwork/defradb/event" coreblock "github.com/sourcenetwork/defradb/internal/core/block" "github.com/sourcenetwork/defradb/internal/core/crdt" "github.com/sourcenetwork/defradb/internal/db" @@ -888,7 +888,7 @@ func TestHandleDocCreateLog_NoError(t *testing.T) { b, err := db.Blockstore().AsIPLDStorage().Get(ctx, headCID.KeyString()) require.NoError(t, err) - err = n.handleDocCreateLog(events.UpdateEvent{ + err = n.handleDocCreateLog(event.UpdateEvent{ DocID: doc.ID().String(), Cid: headCID, SchemaRoot: col.SchemaRoot(), @@ -902,7 +902,7 @@ func TestHandleDocCreateLog_WithInvalidDocID_NoError(t *testing.T) { _, n := newTestNode(ctx, t) defer n.Close() - err := n.handleDocCreateLog(events.UpdateEvent{ + err := n.handleDocCreateLog(event.UpdateEvent{ DocID: "some-invalid-key", }) require.ErrorContains(t, err, "failed to get DocID from broadcast message: selected encoding not supported") @@ -931,7 +931,7 @@ func TestHandleDocCreateLog_WithExistingTopic_TopicExistsError(t *testing.T) { _, err = rpc.NewTopic(ctx, n.ps, n.host.ID(), doc.ID().String(), true) require.NoError(t, err) - err = n.handleDocCreateLog(events.UpdateEvent{ + err = n.handleDocCreateLog(event.UpdateEvent{ DocID: doc.ID().String(), SchemaRoot: col.SchemaRoot(), }) @@ -964,7 +964,7 @@ func TestHandleDocUpdateLog_NoError(t *testing.T) { b, err := db.Blockstore().AsIPLDStorage().Get(ctx, headCID.KeyString()) require.NoError(t, err) - err = n.handleDocUpdateLog(events.UpdateEvent{ + err = n.handleDocUpdateLog(event.UpdateEvent{ DocID: doc.ID().String(), Cid: headCID, SchemaRoot: col.SchemaRoot(), @@ -978,7 +978,7 @@ func TestHandleDoUpdateLog_WithInvalidDocID_NoError(t *testing.T) { _, n := newTestNode(ctx, t) defer n.Close() - err := n.handleDocUpdateLog(events.UpdateEvent{ + err := n.handleDocUpdateLog(event.UpdateEvent{ DocID: "some-invalid-key", }) require.ErrorContains(t, err, "failed to get DocID from broadcast message: selected encoding not supported") @@ -1013,7 +1013,7 @@ func TestHandleDocUpdateLog_WithExistingDocIDTopic_TopicExistsError(t *testing.T _, err = rpc.NewTopic(ctx, n.ps, n.host.ID(), doc.ID().String(), true) require.NoError(t, err) - err = n.handleDocUpdateLog(events.UpdateEvent{ + err = n.handleDocUpdateLog(event.UpdateEvent{ DocID: doc.ID().String(), Cid: headCID, SchemaRoot: col.SchemaRoot(), @@ -1051,7 +1051,7 @@ func TestHandleDocUpdateLog_WithExistingSchemaTopic_TopicExistsError(t *testing. _, err = rpc.NewTopic(ctx, n.ps, n.host.ID(), col.SchemaRoot(), true) require.NoError(t, err) - err = n.handleDocUpdateLog(events.UpdateEvent{ + err = n.handleDocUpdateLog(event.UpdateEvent{ DocID: doc.ID().String(), Cid: headCID, SchemaRoot: col.SchemaRoot(), diff --git a/net/server.go b/net/server.go index 0da2bfc79e..761d87e4c6 100644 --- a/net/server.go +++ b/net/server.go @@ -28,7 +28,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/errors" - "github.com/sourcenetwork/defradb/events" + "github.com/sourcenetwork/defradb/event" coreblock "github.com/sourcenetwork/defradb/internal/core/block" pb "github.com/sourcenetwork/defradb/net/pb" ) @@ -192,7 +192,7 @@ func (s *server) PushLog(ctx context.Context, req *pb.PushLogRequest) (*pb.PushL s.docQueue.add(docID.String()) defer s.docQueue.done(docID.String()) - evt := events.MergeEvent{ + evt := event.MergeEvent{ ByPeer: byPeer, FromPeer: pid, Cid: cid, @@ -206,7 +206,7 @@ func (s *server) PushLog(ctx context.Context, req *pb.PushLogRequest) (*pb.PushL } if exists { // the integration tests expect every push log to emit a merge complete event - s.peer.db.Events().Publish(events.NewMessage(events.MergeCompleteEventName, evt)) + s.peer.db.Events().Publish(event.NewMessage(event.MergeCompleteEventName, evt)) return &pb.PushLogReply{}, nil } @@ -226,7 +226,7 @@ func (s *server) PushLog(ctx context.Context, req *pb.PushLogRequest) (*pb.PushL corelog.Any("CID", cid), ) } - s.peer.db.Events().Publish(events.NewMessage(events.MergeRequestEventName, evt)) + s.peer.db.Events().Publish(event.NewMessage(event.MergeRequestEventName, evt)) // Once processed, subscribe to the DocID topic on the pubsub network unless we already // suscribe to the collection. @@ -373,7 +373,7 @@ func (s *server) pubSubEventHandler(from libpeer.ID, topic string, msg []byte) { corelog.String("Topic", topic), corelog.String("Message", string(msg)), ) - evt := events.NewMessage(events.PubSubEventName, events.PubSubEvent{ + evt := event.NewMessage(event.PubSubEventName, event.PubSubEvent{ Peer: from, }) s.peer.db.Events().Publish(evt) diff --git a/tests/clients/cli/wrapper.go b/tests/clients/cli/wrapper.go index de7d0f82cd..18e306c0f4 100644 --- a/tests/clients/cli/wrapper.go +++ b/tests/clients/cli/wrapper.go @@ -29,7 +29,7 @@ import ( "github.com/sourcenetwork/defradb/cli" "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/datastore" - "github.com/sourcenetwork/defradb/events" + "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/http" "github.com/sourcenetwork/defradb/net" ) @@ -525,7 +525,7 @@ func (w *Wrapper) Close() { w.node.Close() } -func (w *Wrapper) Events() *events.Bus { +func (w *Wrapper) Events() *event.Bus { return w.node.Events() } diff --git a/tests/clients/http/wrapper.go b/tests/clients/http/wrapper.go index 81f79013fc..89b5bce5e7 100644 --- a/tests/clients/http/wrapper.go +++ b/tests/clients/http/wrapper.go @@ -22,7 +22,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/datastore" - "github.com/sourcenetwork/defradb/events" + "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/http" "github.com/sourcenetwork/defradb/net" ) @@ -221,7 +221,7 @@ func (w *Wrapper) Close() { w.node.Close() } -func (w *Wrapper) Events() *events.Bus { +func (w *Wrapper) Events() *event.Bus { return w.node.Events() } diff --git a/tests/integration/events/utils.go b/tests/integration/events/utils.go index 3c5a8e940d..fd66c23e05 100644 --- a/tests/integration/events/utils.go +++ b/tests/integration/events/utils.go @@ -20,7 +20,7 @@ import ( "github.com/stretchr/testify/require" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/events" + "github.com/sourcenetwork/defradb/event" testUtils "github.com/sourcenetwork/defradb/tests/integration" ) @@ -80,7 +80,7 @@ func ExecuteRequestTestCase( testRoutineClosedChan := make(chan struct{}) closeTestRoutineChan := make(chan struct{}) - eventsSub := db.Events().Subscribe(5, events.UpdateEventName) + eventsSub := db.Events().Subscribe(5, event.UpdateEventName) require.NoError(t, err) indexOfNextExpectedUpdate := 0 @@ -88,7 +88,7 @@ func ExecuteRequestTestCase( for { select { case value := <-eventsSub.Message(): - update, ok := value.Data.(events.UpdateEvent) + update, ok := value.Data.(event.UpdateEvent) if !ok { continue // ignore invaid value } diff --git a/tests/integration/p2p.go b/tests/integration/p2p.go index 20c77e8074..25243af4cb 100644 --- a/tests/integration/p2p.go +++ b/tests/integration/p2p.go @@ -14,7 +14,7 @@ import ( "time" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/events" + "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/net" "github.com/sourcenetwork/defradb/tests/clients" @@ -175,8 +175,8 @@ func setupPeerWaitSync( cfg ConnectPeers, sourceNode clients.Client, targetNode clients.Client, - sourceSub *events.Subscription, - targetSub *events.Subscription, + sourceSub *event.Subscription, + targetSub *event.Subscription, ) { sourceToTargetEvents := []int{0} targetToSourceEvents := []int{0} @@ -271,13 +271,13 @@ func setupPeerWaitSync( for i := 0; i < targetToSourceEvents[waitIndex]; i++ { msg, ok := <-sourceSub.Message() if ok { - assert.Equal(s.t, targetPeerInfo.ID, msg.Data.(events.MergeEvent).ByPeer) + assert.Equal(s.t, targetPeerInfo.ID, msg.Data.(event.MergeEvent).ByPeer) } } for i := 0; i < sourceToTargetEvents[waitIndex]; i++ { msg, ok := <-targetSub.Message() if ok { - assert.Equal(s.t, sourcePeerInfo.ID, msg.Data.(events.MergeEvent).ByPeer) + assert.Equal(s.t, sourcePeerInfo.ID, msg.Data.(event.MergeEvent).ByPeer) } } nodeSynced <- struct{}{} @@ -353,8 +353,8 @@ func setupReplicatorWaitSync( cfg ConfigureReplicator, sourceNode clients.Client, targetNode clients.Client, - sourceSub *events.Subscription, - targetSub *events.Subscription, + sourceSub *event.Subscription, + targetSub *event.Subscription, ) { sourceToTargetEvents := []int{0} targetToSourceEvents := []int{0} @@ -415,13 +415,13 @@ func setupReplicatorWaitSync( for i := 0; i < targetToSourceEvents[waitIndex]; i++ { msg, ok := <-sourceSub.Message() if ok { - assert.Equal(s.t, targetPeerInfo.ID, msg.Data.(events.MergeEvent).ByPeer) + assert.Equal(s.t, targetPeerInfo.ID, msg.Data.(event.MergeEvent).ByPeer) } } for i := 0; i < sourceToTargetEvents[waitIndex]; i++ { msg, ok := <-targetSub.Message() if ok { - assert.Equal(s.t, sourcePeerInfo.ID, msg.Data.(events.MergeEvent).ByPeer) + assert.Equal(s.t, sourcePeerInfo.ID, msg.Data.(event.MergeEvent).ByPeer) } } nodeSynced <- struct{}{} diff --git a/tests/integration/state.go b/tests/integration/state.go index 0b5bc63ab8..c6a97dfe8e 100644 --- a/tests/integration/state.go +++ b/tests/integration/state.go @@ -18,7 +18,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/datastore" - "github.com/sourcenetwork/defradb/events" + "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/net" "github.com/sourcenetwork/defradb/tests/clients" ) @@ -54,7 +54,7 @@ type state struct { syncChans []chan struct{} // eventSubs is a list of all event subscriptions - eventSubs []*events.Subscription + eventSubs []*event.Subscription // The addresses of any nodes configured. nodeAddresses []peer.AddrInfo @@ -108,7 +108,7 @@ func newState( allActionsDone: make(chan struct{}), subscriptionResultsChans: []chan func(){}, syncChans: []chan struct{}{}, - eventSubs: []*events.Subscription{}, + eventSubs: []*event.Subscription{}, nodeAddresses: []peer.AddrInfo{}, nodeConfigs: [][]net.NodeOpt{}, nodes: []clients.Client{}, diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 8e3b1744b3..e9a1b0efe3 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -32,7 +32,7 @@ import ( "github.com/sourcenetwork/defradb/datastore" badgerds "github.com/sourcenetwork/defradb/datastore/badger/v4" "github.com/sourcenetwork/defradb/errors" - "github.com/sourcenetwork/defradb/events" + "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/internal/db" "github.com/sourcenetwork/defradb/internal/request/graphql" "github.com/sourcenetwork/defradb/net" @@ -658,7 +658,7 @@ func setStartingNodes( s.nodes = append(s.nodes, c) s.dbPaths = append(s.dbPaths, path) - sub := c.Events().Subscribe(100, events.MergeCompleteEventName) + sub := c.Events().Subscribe(100, event.MergeCompleteEventName) s.eventSubs = append(s.eventSubs, sub) } } @@ -829,7 +829,7 @@ func configureNode( s.nodes = append(s.nodes, c) s.dbPaths = append(s.dbPaths, path) - sub := c.Events().Subscribe(100, events.MergeCompleteEventName) + sub := c.Events().Subscribe(100, event.MergeCompleteEventName) s.eventSubs = append(s.eventSubs, sub) } From 2da91533f3fbec404e09be787131f43183a34423 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 11 Jun 2024 11:16:39 -0700 Subject: [PATCH 08/34] add event bus timeout --- client/mocks/db.go | 14 +++---- datastore/mocks/utils.go | 1 + event/bus.go | 43 ++++++++++++--------- event/bus_test.go | 64 ++++++++++++++++++++++++++++---- event/{events.go => event.go} | 0 internal/db/collection.go | 4 +- internal/db/collection_delete.go | 4 +- internal/db/db.go | 41 +++++++++++--------- internal/db/merge.go | 6 +-- internal/db/subscriptions.go | 4 +- 10 files changed, 123 insertions(+), 58 deletions(-) rename event/{events.go => event.go} (100%) diff --git a/client/mocks/db.go b/client/mocks/db.go index 7867e90637..c56af31167 100644 --- a/client/mocks/db.go +++ b/client/mocks/db.go @@ -9,7 +9,7 @@ import ( datastore "github.com/sourcenetwork/defradb/datastore" - events "github.com/sourcenetwork/defradb/event" + event "github.com/sourcenetwork/defradb/event" go_datastore "github.com/ipfs/go-datastore" @@ -360,15 +360,15 @@ func (_c *DB_Close_Call) RunAndReturn(run func()) *DB_Close_Call { } // Events provides a mock function with given fields: -func (_m *DB) Events() *events.Bus { +func (_m *DB) Events() *event.Bus { ret := _m.Called() - var r0 *events.Bus - if rf, ok := ret.Get(0).(func() *events.Bus); ok { + var r0 *event.Bus + if rf, ok := ret.Get(0).(func() *event.Bus); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*events.Bus) + r0 = ret.Get(0).(*event.Bus) } } @@ -392,12 +392,12 @@ func (_c *DB_Events_Call) Run(run func()) *DB_Events_Call { return _c } -func (_c *DB_Events_Call) Return(_a0 *events.Bus) *DB_Events_Call { +func (_c *DB_Events_Call) Return(_a0 *event.Bus) *DB_Events_Call { _c.Call.Return(_a0) return _c } -func (_c *DB_Events_Call) RunAndReturn(run func() *events.Bus) *DB_Events_Call { +func (_c *DB_Events_Call) RunAndReturn(run func() *event.Bus) *DB_Events_Call { _c.Call.Return(run) return _c } diff --git a/datastore/mocks/utils.go b/datastore/mocks/utils.go index af91fc6d3a..50131a8538 100644 --- a/datastore/mocks/utils.go +++ b/datastore/mocks/utils.go @@ -68,6 +68,7 @@ func prepareDAGStore(t *testing.T) *DAGStore { func NewTxnWithMultistore(t *testing.T) *MultiStoreTxn { txn := NewTxn(t) txn.EXPECT().OnSuccess(mock.Anything).Maybe() + txn.EXPECT().OnSuccessAsync(mock.Anything).Maybe() result := &MultiStoreTxn{ Txn: txn, diff --git a/event/bus.go b/event/bus.go index 275cfac688..adce978e95 100644 --- a/event/bus.go +++ b/event/bus.go @@ -12,6 +12,7 @@ package event import ( "sync" + "time" ) // Message contains event info. @@ -44,28 +45,35 @@ func (s *Subscription) Events() []string { return s.events } -// Bus is used to publish and subscribe to events. +// Bus is used to broadcast events to subscribers. type Bus struct { - subId int - subs map[int]*Subscription - events map[string]map[int]any - mutex sync.RWMutex + // subID is incremented for each subscriber and used to set subscriber ids. + subID int + // subs is a mapping of subscriber ids to subscriptions. + subs map[int]*Subscription + // events is a mapping of event names to subscriber ids. + events map[string]map[int]struct{} + // mutex is used to lock reads and writes to the subs and events maps. + mutex sync.RWMutex + // timeout is the amount of time to wait for a blocking channel before a message is discarded. + timeout time.Duration } // NewBus returns a new event bus. -func NewBus() *Bus { +func NewBus(timeout time.Duration) *Bus { return &Bus{ - subs: make(map[int]*Subscription), - events: make(map[string]map[int]any), + timeout: timeout, + subs: make(map[int]*Subscription), + events: make(map[string]map[int]struct{}), } } -// Publish publishes the given event to all subscribers. +// Publish publishes the given event message to all subscribers. func (b *Bus) Publish(msg Message) { b.mutex.RLock() defer b.mutex.RUnlock() - subscribers := make(map[int]any) + subscribers := make(map[int]struct{}) for id := range b.events[msg.Name] { subscribers[id] = struct{}{} } @@ -76,9 +84,9 @@ func (b *Bus) Publish(msg Message) { for id := range subscribers { select { case b.subs[id].value <- msg: - // published event - default: - // channel full + // published message + case <-time.After(b.timeout): + // discarded message } } } @@ -89,19 +97,19 @@ func (b *Bus) Subscribe(size int, events ...string) *Subscription { defer b.mutex.Unlock() sub := &Subscription{ - id: b.subId, + id: b.subID, value: make(chan Message, size), events: events, } for _, event := range events { if _, ok := b.events[event]; !ok { - b.events[event] = make(map[int]any) + b.events[event] = make(map[int]struct{}) } b.events[event][sub.id] = struct{}{} } - b.subId++ + b.subID++ b.subs[sub.id] = sub return sub } @@ -124,9 +132,8 @@ func (b *Bus) Unsubscribe(sub *Subscription) { // Close closes the event bus by unsubscribing all subscribers. func (b *Bus) Close() { - var subs []*Subscription - b.mutex.RLock() + subs := make([]*Subscription, 0, len(b.subs)) for _, sub := range b.subs { subs = append(subs, sub) } diff --git a/event/bus_test.go b/event/bus_test.go index 8eed3d57e0..62f178baf9 100644 --- a/event/bus_test.go +++ b/event/bus_test.go @@ -12,23 +12,73 @@ package event import ( "testing" + "time" "github.com/stretchr/testify/assert" ) -func TestBusPublish(t *testing.T) { - bus := NewBus() +func TestBusSubscribeThenPublish(t *testing.T) { + bus := NewBus(100 * time.Millisecond) defer bus.Close() sub1 := bus.Subscribe(1, "test") - sub2 := bus.Subscribe(1, WildCardEventName) + sub2 := bus.Subscribe(1, WildCardEventName, "test") + + assert.ElementsMatch(t, sub1.Events(), []string{"test"}) + assert.ElementsMatch(t, sub2.Events(), []string{WildCardEventName, "test"}) + + msg := NewMessage("test", "hello") + go bus.Publish(msg) + + event := <-sub1.Message() + assert.Equal(t, msg, event) + + event = <-sub2.Message() + assert.Equal(t, msg, event) + + select { + case <-sub2.Message(): + t.Fatalf("subscriber should not recieve duplicate message") + case <-time.After(150 * time.Millisecond): + // wait for publish timeout + skew + } +} + +func TestBusPublishThenSubscribe(t *testing.T) { + bus := NewBus(100 * time.Millisecond) + defer bus.Close() + + msg := NewMessage("test", "hello") + bus.Publish(msg) + + sub := bus.Subscribe(1, "test") + select { + case <-sub.Message(): + t.Fatalf("subscriber should not recieve message") + case <-time.After(150 * time.Millisecond): + // wait for publish timeout + skew + } +} + +func TestBusSubscribeThenUnsubscribeThenPublish(t *testing.T) { + bus := NewBus(100 * time.Millisecond) + defer bus.Close() + + sub := bus.Subscribe(1, "test") + bus.Unsubscribe(sub) msg := NewMessage("test", "hello") bus.Publish(msg) - event1 := <-sub1.Message() - assert.Equal(t, msg, event1) + _, ok := <-sub.Message() + assert.False(t, ok, "channel should be closed") +} + +func TestBusUnsubscribeTwice(t *testing.T) { + bus := NewBus(100 * time.Millisecond) + defer bus.Close() - event2 := <-sub2.Message() - assert.Equal(t, msg, event2) + sub := bus.Subscribe(1, "test") + bus.Unsubscribe(sub) + bus.Unsubscribe(sub) } diff --git a/event/events.go b/event/event.go similarity index 100% rename from event/events.go rename to event/event.go diff --git a/internal/db/collection.go b/internal/db/collection.go index 4ea3b9a76b..38b59e1db9 100644 --- a/internal/db/collection.go +++ b/internal/db/collection.go @@ -684,8 +684,8 @@ func (c *collection) save( Block: headNode, IsCreate: isCreate, } - txn.OnSuccess(func() { - c.db.sysEventBus.Publish(event.NewMessage(event.UpdateEventName, updateEvent)) + txn.OnSuccessAsync(func() { + c.db.sysBus.Publish(event.NewMessage(event.UpdateEventName, updateEvent)) }) txn.OnSuccess(func() { diff --git a/internal/db/collection_delete.go b/internal/db/collection_delete.go index fc98bec5a7..0a6c870257 100644 --- a/internal/db/collection_delete.go +++ b/internal/db/collection_delete.go @@ -173,8 +173,8 @@ func (c *collection) applyDelete( SchemaRoot: c.Schema().Root, Block: b, } - txn.OnSuccess(func() { - c.db.sysEventBus.Publish(event.NewMessage(event.UpdateEventName, updateEvent)) + txn.OnSuccessAsync(func() { + c.db.sysBus.Publish(event.NewMessage(event.UpdateEventName, updateEvent)) }) return nil diff --git a/internal/db/db.go b/internal/db/db.go index df4f6f4b74..ac5c2d30f2 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -18,6 +18,7 @@ import ( "context" "sync" "sync/atomic" + "time" ds "github.com/ipfs/go-datastore" dsq "github.com/ipfs/go-datastore/query" @@ -44,15 +45,21 @@ var ( ) const ( - // forwardSubBufferSize controls the size of the channel used to + // forwardBufferSize controls the size of the channel used to // forward events from the system bus to the subscription bus - forwardSubBufferSize = 10 - // requestSubBufferSize controls the size of the channel used to + forwardBufferSize = 10 + // subscriptionBufferSize controls the size of the channel used to // send events to request subscriptions - requestSubBufferSize = 5 - // mergeSubBufferSize controls the size of the channel used to + subscriptionBufferSize = 5 + // mergeBufferSize controls the size of the channel used to // handle merge events - mergeSubBufferSize = 10 + mergeBufferSize = 10 + // sysBusTimeout is the duration to wait before discarding + // messages on the sysBus + sysBusTimeout = 5 * time.Minute + // subBusTimeout is the duration to wait before discarding + // messages on the subBus + subBusTimeout = 30 * time.Second ) // DB is the main interface for interacting with the @@ -63,11 +70,11 @@ type db struct { rootstore datastore.RootStore multistore datastore.MultiStore - // sysEventBus is used to send and receive system events - sysEventBus *event.Bus + // sysBus is used to send and receive system events + sysBus *event.Bus - // subEventBus is used to send and receive request subscription events - subEventBus *event.Bus + // subBus is used to send and receive request subscription events + subBus *event.Bus parser core.Parser @@ -118,8 +125,8 @@ func newDB( lensRegistry: lens, parser: parser, options: options, - sysEventBus: event.NewBus(), - subEventBus: event.NewBus(), + sysBus: event.NewBus(sysBusTimeout), + subBus: event.NewBus(subBusTimeout), } // apply options @@ -207,9 +214,9 @@ func (db *db) initialize(ctx context.Context) error { // forward system bus events to the subscriber bus // to ensure we never block the system bus for user subscriptions go func() { - sub := db.sysEventBus.Subscribe(forwardSubBufferSize, event.WildCardEventName) + sub := db.sysBus.Subscribe(forwardBufferSize, event.WildCardEventName) for msg := range sub.Message() { - db.subEventBus.Publish(msg) + db.subBus.Publish(msg) } }() @@ -267,7 +274,7 @@ func (db *db) initialize(ctx context.Context) error { // Events returns the events Channel. func (db *db) Events() *event.Bus { - return db.sysEventBus + return db.sysBus } // MaxRetries returns the maximum number of retries per transaction. @@ -289,8 +296,8 @@ func (db *db) PrintDump(ctx context.Context) error { func (db *db) Close() { log.Info("Closing DefraDB process...") - db.subEventBus.Close() - db.sysEventBus.Close() + db.subBus.Close() + db.sysBus.Close() err := db.rootstore.Close() if err != nil { diff --git a/internal/db/merge.go b/internal/db/merge.go index eab5b7d778..bc6fefd68f 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -34,8 +34,8 @@ import ( ) func (db *db) handleMerges(ctx context.Context) { - sub := db.sysEventBus.Subscribe(mergeSubBufferSize, event.MergeRequestEventName) - defer db.sysEventBus.Unsubscribe(sub) + sub := db.sysBus.Subscribe(mergeBufferSize, event.MergeRequestEventName) + defer db.sysBus.Unsubscribe(sub) for { select { @@ -67,7 +67,7 @@ func (db *db) handleMerges(ctx context.Context) { func (db *db) executeMerge(ctx context.Context, dagMerge event.MergeEvent) error { // send a complete event so we can track merges in the integration tests - defer db.sysEventBus.Publish(event.NewMessage(event.MergeCompleteEventName, dagMerge)) + defer db.sysBus.Publish(event.NewMessage(event.MergeCompleteEventName, dagMerge)) ctx, txn, err := ensureContextTxn(ctx, db, false) if err != nil { diff --git a/internal/db/subscriptions.go b/internal/db/subscriptions.go index 4aa2d0c2da..7547b7fffe 100644 --- a/internal/db/subscriptions.go +++ b/internal/db/subscriptions.go @@ -32,11 +32,11 @@ func (db *db) handleSubscription(ctx context.Context, r *request.Request) (<-cha return nil, client.NewErrUnexpectedType[request.ObjectSubscription]("SubscriptionSelection", selections) } // subscribe to the subscription event bus so we don't block the system bus - sub := db.subEventBus.Subscribe(requestSubBufferSize, event.UpdateEventName) + sub := db.subBus.Subscribe(subscriptionBufferSize, event.UpdateEventName) resCh := make(chan client.GQLResult) go func() { defer func() { - db.sysEventBus.Unsubscribe(sub) + db.sysBus.Unsubscribe(sub) close(resCh) }() From 3725fca6561e3864c8ac7db2b534ed3eee6f2aac Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 11 Jun 2024 14:15:48 -0700 Subject: [PATCH 09/34] add waitForMerge test util --- internal/db/db.go | 8 +-- tests/integration/events/utils.go | 2 +- tests/integration/p2p.go | 112 ++++++++++++------------------ tests/integration/utils2.go | 20 +----- 4 files changed, 52 insertions(+), 90 deletions(-) diff --git a/internal/db/db.go b/internal/db/db.go index ac5c2d30f2..4e2535f087 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -47,19 +47,19 @@ var ( const ( // forwardBufferSize controls the size of the channel used to // forward events from the system bus to the subscription bus - forwardBufferSize = 10 + forwardBufferSize = 100 // subscriptionBufferSize controls the size of the channel used to // send events to request subscriptions - subscriptionBufferSize = 5 + subscriptionBufferSize = 10 // mergeBufferSize controls the size of the channel used to // handle merge events - mergeBufferSize = 10 + mergeBufferSize = 100 // sysBusTimeout is the duration to wait before discarding // messages on the sysBus sysBusTimeout = 5 * time.Minute // subBusTimeout is the duration to wait before discarding // messages on the subBus - subBusTimeout = 30 * time.Second + subBusTimeout = 1 * time.Minute ) // DB is the main interface for interacting with the diff --git a/tests/integration/events/utils.go b/tests/integration/events/utils.go index fd66c23e05..b649612742 100644 --- a/tests/integration/events/utils.go +++ b/tests/integration/events/utils.go @@ -90,7 +90,7 @@ func ExecuteRequestTestCase( case value := <-eventsSub.Message(): update, ok := value.Data.(event.UpdateEvent) if !ok { - continue // ignore invaid value + continue // ignore invalid value } if indexOfNextExpectedUpdate >= len(testCase.ExpectedUpdates) { diff --git a/tests/integration/p2p.go b/tests/integration/p2p.go index 25243af4cb..233deb59ad 100644 --- a/tests/integration/p2p.go +++ b/tests/integration/p2p.go @@ -16,7 +16,6 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/net" - "github.com/sourcenetwork/defradb/tests/clients" "github.com/libp2p/go-libp2p/core/peer" "github.com/sourcenetwork/corelog" @@ -155,9 +154,6 @@ func connectPeers( sourceNode := s.nodes[cfg.SourceNodeID] targetNode := s.nodes[cfg.TargetNodeID] - sourceSub := s.eventSubs[cfg.SourceNodeID] - targetSub := s.eventSubs[cfg.TargetNodeID] - addrs := []peer.AddrInfo{targetNode.PeerInfo()} log.InfoContext(s.ctx, "Bootstrapping with peers", corelog.Any("Addresses", addrs)) sourceNode.Bootstrap(addrs) @@ -166,24 +162,17 @@ func connectPeers( // allowed to complete before documentation begins or it will not even try and sync it. So for now, we // sleep a little. time.Sleep(100 * time.Millisecond) - setupPeerWaitSync(s, 0, cfg, sourceNode, targetNode, sourceSub, targetSub) + setupPeerWaitSync(s, 0, cfg) } func setupPeerWaitSync( s *state, startIndex int, cfg ConnectPeers, - sourceNode clients.Client, - targetNode clients.Client, - sourceSub *event.Subscription, - targetSub *event.Subscription, ) { sourceToTargetEvents := []int{0} targetToSourceEvents := []int{0} - sourcePeerInfo := sourceNode.PeerInfo() - targetPeerInfo := targetNode.PeerInfo() - nodeCollections := map[int][]int{} waitIndex := 0 for i := startIndex; i < len(s.testCase.Actions); i++ { @@ -264,28 +253,7 @@ func setupPeerWaitSync( } nodeSynced := make(chan struct{}) - ready := make(chan struct{}) - go func(ready chan struct{}) { - ready <- struct{}{} - for waitIndex := 0; waitIndex < len(sourceToTargetEvents); waitIndex++ { - for i := 0; i < targetToSourceEvents[waitIndex]; i++ { - msg, ok := <-sourceSub.Message() - if ok { - assert.Equal(s.t, targetPeerInfo.ID, msg.Data.(event.MergeEvent).ByPeer) - } - } - for i := 0; i < sourceToTargetEvents[waitIndex]; i++ { - msg, ok := <-targetSub.Message() - if ok { - assert.Equal(s.t, sourcePeerInfo.ID, msg.Data.(event.MergeEvent).ByPeer) - } - } - nodeSynced <- struct{}{} - } - }(ready) - // Ensure that the wait routine is ready to receive events before we continue. - <-ready - + go waitForMerge(s, cfg.SourceNodeID, cfg.TargetNodeID, sourceToTargetEvents, targetToSourceEvents, nodeSynced) s.syncChans = append(s.syncChans, nodeSynced) } @@ -320,9 +288,6 @@ func configureReplicator( sourceNode := s.nodes[cfg.SourceNodeID] targetNode := s.nodes[cfg.TargetNodeID] - sourceSub := s.eventSubs[cfg.SourceNodeID] - targetSub := s.eventSubs[cfg.TargetNodeID] - err := sourceNode.SetReplicator(s.ctx, client.Replicator{ Info: targetNode.PeerInfo(), }) @@ -330,7 +295,7 @@ func configureReplicator( expectedErrorRaised := AssertError(s.t, s.testCase.Description, err, cfg.ExpectedError) assertExpectedErrorRaised(s.t, s.testCase.Description, cfg.ExpectedError, expectedErrorRaised) if err == nil { - setupReplicatorWaitSync(s, 0, cfg, sourceNode, targetNode, sourceSub, targetSub) + setupReplicatorWaitSync(s, 0, cfg) } } @@ -351,17 +316,10 @@ func setupReplicatorWaitSync( s *state, startIndex int, cfg ConfigureReplicator, - sourceNode clients.Client, - targetNode clients.Client, - sourceSub *event.Subscription, - targetSub *event.Subscription, ) { sourceToTargetEvents := []int{0} targetToSourceEvents := []int{0} - sourcePeerInfo := sourceNode.PeerInfo() - targetPeerInfo := targetNode.PeerInfo() - docIDsSyncedToSource := map[int]struct{}{} waitIndex := 0 currentDocID := 0 @@ -408,28 +366,7 @@ func setupReplicatorWaitSync( } nodeSynced := make(chan struct{}) - ready := make(chan struct{}) - go func(ready chan struct{}) { - ready <- struct{}{} - for waitIndex := 0; waitIndex < len(sourceToTargetEvents); waitIndex++ { - for i := 0; i < targetToSourceEvents[waitIndex]; i++ { - msg, ok := <-sourceSub.Message() - if ok { - assert.Equal(s.t, targetPeerInfo.ID, msg.Data.(event.MergeEvent).ByPeer) - } - } - for i := 0; i < sourceToTargetEvents[waitIndex]; i++ { - msg, ok := <-targetSub.Message() - if ok { - assert.Equal(s.t, sourcePeerInfo.ID, msg.Data.(event.MergeEvent).ByPeer) - } - } - nodeSynced <- struct{}{} - } - }(ready) - // Ensure that the wait routine is ready to receive events before we continue. - <-ready - + go waitForMerge(s, cfg.SourceNodeID, cfg.TargetNodeID, sourceToTargetEvents, targetToSourceEvents, nodeSynced) s.syncChans = append(s.syncChans, nodeSynced) } @@ -551,6 +488,47 @@ func waitForSync( } } +// waitForMerge waits for the source and target nodes to synchronize their state +// by listening to merge events sent from the network subsystem on the event bus. +// +// sourceToTargetEvents and targetToSourceEvents are slices containing the number +// of expected merge events to be received after each test action has executed. +func waitForMerge( + s *state, + sourceNodeID int, + targetNodeID int, + sourceToTargetEvents []int, + targetToSourceEvents []int, + nodeSynced chan struct{}, +) { + sourceNode := s.nodes[sourceNodeID] + targetNode := s.nodes[targetNodeID] + + sourceSub := s.eventSubs[sourceNodeID] + targetSub := s.eventSubs[targetNodeID] + + sourcePeerInfo := sourceNode.PeerInfo() + targetPeerInfo := targetNode.PeerInfo() + + for waitIndex := 0; waitIndex < len(sourceToTargetEvents); waitIndex++ { + for i := 0; i < targetToSourceEvents[waitIndex]; i++ { + // wait for message or unsubscribe + msg, ok := <-sourceSub.Message() + if ok { + require.Equal(s.t, targetPeerInfo.ID, msg.Data.(event.MergeEvent).ByPeer) + } + } + for i := 0; i < sourceToTargetEvents[waitIndex]; i++ { + // wait for message or unsubscribe + msg, ok := <-targetSub.Message() + if ok { + require.Equal(s.t, sourcePeerInfo.ID, msg.Data.(event.MergeEvent).ByPeer) + } + } + nodeSynced <- struct{}{} + } +} + func RandomNetworkingConfig() ConfigureNode { return func() []net.NodeOpt { return []net.NodeOpt{ diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index e9a1b0efe3..d6b509e8f1 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -732,27 +732,11 @@ actionLoop: case ConnectPeers: // Give the nodes a chance to connect to each other and learn about each other's subscribed topics. time.Sleep(100 * time.Millisecond) - setupPeerWaitSync( - s, - waitGroupStartIndex, - action, - s.nodes[action.SourceNodeID], - s.nodes[action.TargetNodeID], - s.eventSubs[action.SourceNodeID], - s.eventSubs[action.TargetNodeID], - ) + setupPeerWaitSync(s, waitGroupStartIndex, action) case ConfigureReplicator: // Give the nodes a chance to connect to each other and learn about each other's subscribed topics. time.Sleep(100 * time.Millisecond) - setupReplicatorWaitSync( - s, - waitGroupStartIndex, - action, - s.nodes[action.SourceNodeID], - s.nodes[action.TargetNodeID], - s.eventSubs[action.SourceNodeID], - s.eventSubs[action.TargetNodeID], - ) + setupReplicatorWaitSync(s, waitGroupStartIndex, action) } } From 45f6eddedc36b5f22754318fce2c5fc29230a32d Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 11 Jun 2024 16:01:49 -0700 Subject: [PATCH 10/34] merge event bus and simple channel implementations --- event/bus.go | 168 ++++++++++++++++++------------ event/bus_test.go | 145 ++++++++++++++++++-------- event/errors.go | 19 ++++ internal/db/collection.go | 4 +- internal/db/collection_delete.go | 4 +- internal/db/db.go | 53 +++------- internal/db/merge.go | 7 +- internal/db/subscriptions.go | 7 +- net/peer.go | 6 +- tests/integration/events/utils.go | 3 +- tests/integration/utils2.go | 8 +- 11 files changed, 263 insertions(+), 161 deletions(-) create mode 100644 event/errors.go diff --git a/event/bus.go b/event/bus.go index adce978e95..30c15e9b8e 100644 --- a/event/bus.go +++ b/event/bus.go @@ -11,10 +11,17 @@ package event import ( - "sync" - "time" + "sync/atomic" ) +type subscribeCommand *Subscription + +type unsubscribeCommand *Subscription + +type publishCommand Message + +type closeCommand struct{} + // Message contains event info. type Message struct { // Name is the name of the event this message was generated from. @@ -30,7 +37,7 @@ func NewMessage(name string, data any) Message { // Subscription is a read-only event stream. type Subscription struct { - id int + id uint64 value chan Message events []string } @@ -48,98 +55,125 @@ func (s *Subscription) Events() []string { // Bus is used to broadcast events to subscribers. type Bus struct { // subID is incremented for each subscriber and used to set subscriber ids. - subID int + subID atomic.Uint64 // subs is a mapping of subscriber ids to subscriptions. - subs map[int]*Subscription + subs map[uint64]*Subscription // events is a mapping of event names to subscriber ids. - events map[string]map[int]struct{} - // mutex is used to lock reads and writes to the subs and events maps. - mutex sync.RWMutex - // timeout is the amount of time to wait for a blocking channel before a message is discarded. - timeout time.Duration + events map[string]map[uint64]struct{} + // commandChannel manages all commands sent to this simpleChannel. + // + // It is important that all stuff gets sent through this single channel to ensure + // that the order of operations is preserved. + // + // WARNING: This does mean that non-event commands can block the database if the buffer + // size is breached (e.g. if many subscribe commands occupy the buffer). + commandChannel chan any + eventBufferSize int + hasClosedChan chan struct{} + isClosed atomic.Bool } -// NewBus returns a new event bus. -func NewBus(timeout time.Duration) *Bus { - return &Bus{ - timeout: timeout, - subs: make(map[int]*Subscription), - events: make(map[string]map[int]struct{}), +// NewBus creates a new event bus with the given commandBufferSize and +// eventBufferSize. +// +// Should the buffers be filled subsequent calls to functions on this object may start to block. +func NewBus(commandBufferSize int, eventBufferSize int) *Bus { + bus := Bus{ + subs: make(map[uint64]*Subscription), + events: make(map[string]map[uint64]struct{}), + commandChannel: make(chan any, commandBufferSize), + hasClosedChan: make(chan struct{}), + eventBufferSize: eventBufferSize, } + go bus.handleChannel() + return &bus } // Publish publishes the given event message to all subscribers. func (b *Bus) Publish(msg Message) { - b.mutex.RLock() - defer b.mutex.RUnlock() - - subscribers := make(map[int]struct{}) - for id := range b.events[msg.Name] { - subscribers[id] = struct{}{} - } - for id := range b.events[WildCardEventName] { - subscribers[id] = struct{}{} - } - - for id := range subscribers { - select { - case b.subs[id].value <- msg: - // published message - case <-time.After(b.timeout): - // discarded message - } + if b.isClosed.Load() { + return } + b.commandChannel <- publishCommand(msg) } // Subscribe returns a new channel that will receive all of the subscribed events. -func (b *Bus) Subscribe(size int, events ...string) *Subscription { - b.mutex.Lock() - defer b.mutex.Unlock() +func (b *Bus) Subscribe(events ...string) (*Subscription, error) { + if b.isClosed.Load() { + return nil, ErrSubscribedToClosedChan + } sub := &Subscription{ - id: b.subID, - value: make(chan Message, size), + id: b.subID.Add(1), + value: make(chan Message, b.eventBufferSize), events: events, } - for _, event := range events { - if _, ok := b.events[event]; !ok { - b.events[event] = make(map[int]struct{}) - } - b.events[event][sub.id] = struct{}{} - } - - b.subID++ - b.subs[sub.id] = sub - return sub + b.commandChannel <- subscribeCommand(sub) + return sub, nil } // Unsubscribe unsubscribes from all events and closes the event channel of the given subscription. func (b *Bus) Unsubscribe(sub *Subscription) { - b.mutex.Lock() - defer b.mutex.Unlock() - - if _, ok := b.subs[sub.id]; !ok { - return // not subscribed + if b.isClosed.Load() { + return } - for _, event := range sub.events { - delete(b.events[event], sub.id) - } - - delete(b.subs, sub.id) - close(sub.value) + b.commandChannel <- unsubscribeCommand(sub) } // Close closes the event bus by unsubscribing all subscribers. func (b *Bus) Close() { - b.mutex.RLock() - subs := make([]*Subscription, 0, len(b.subs)) - for _, sub := range b.subs { - subs = append(subs, sub) + if b.isClosed.Load() { + return } - b.mutex.RUnlock() + b.isClosed.Store(true) + b.commandChannel <- closeCommand{} - for _, sub := range subs { - b.Unsubscribe(sub) + // Wait for the close command to be handled, in order, before returning + <-b.hasClosedChan +} + +func (b *Bus) handleChannel() { + for cmd := range b.commandChannel { + switch t := cmd.(type) { + case closeCommand: + for _, subscriber := range b.subs { + close(subscriber.value) + } + close(b.commandChannel) + close(b.hasClosedChan) + return + + case subscribeCommand: + for _, event := range t.events { + if _, ok := b.events[event]; !ok { + b.events[event] = make(map[uint64]struct{}) + } + b.events[event][t.id] = struct{}{} + } + b.subs[t.id] = t + + case unsubscribeCommand: + if _, ok := b.subs[t.id]; !ok { + continue // not subscribed + } + for _, event := range t.events { + delete(b.events[event], t.id) + } + delete(b.subs, t.id) + close(t.value) + + case publishCommand: + subscribers := make(map[uint64]struct{}) + for id := range b.events[t.Name] { + subscribers[id] = struct{}{} + } + for id := range b.events[WildCardEventName] { + subscribers[id] = struct{}{} + } + for id := range subscribers { + b.subs[id].value <- Message(t) + } + } } } diff --git a/event/bus_test.go b/event/bus_test.go index 62f178baf9..bbbbad3111 100644 --- a/event/bus_test.go +++ b/event/bus_test.go @@ -11,74 +11,137 @@ package event import ( + "sync" "testing" "time" "github.com/stretchr/testify/assert" ) -func TestBusSubscribeThenPublish(t *testing.T) { - bus := NewBus(100 * time.Millisecond) +func TestSimplePushIsNotBlockedWithoutSubscribers(t *testing.T) { + bus := NewBus(0, 0) defer bus.Close() - sub1 := bus.Subscribe(1, "test") - sub2 := bus.Subscribe(1, WildCardEventName, "test") + msg := NewMessage("test", 1) + bus.Publish(msg) + + // just assert that we reach this line, for the sake of having an assert + assert.True(t, true) +} - assert.ElementsMatch(t, sub1.Events(), []string{"test"}) - assert.ElementsMatch(t, sub2.Events(), []string{WildCardEventName, "test"}) +func TestSimpleSubscribersAreNotBlockedAfterClose(t *testing.T) { + bus := NewBus(0, 0) + defer bus.Close() - msg := NewMessage("test", "hello") - go bus.Publish(msg) + sub, err := bus.Subscribe("test") + assert.Nil(t, err) - event := <-sub1.Message() - assert.Equal(t, msg, event) + bus.Close() - event = <-sub2.Message() - assert.Equal(t, msg, event) + <-sub.Message() - select { - case <-sub2.Message(): - t.Fatalf("subscriber should not recieve duplicate message") - case <-time.After(150 * time.Millisecond): - // wait for publish timeout + skew - } + // just assert that we reach this line, for the sake of having an assert + assert.True(t, true) } -func TestBusPublishThenSubscribe(t *testing.T) { - bus := NewBus(100 * time.Millisecond) +func TestSimpleEachSubscribersRecievesEachItem(t *testing.T) { + bus := NewBus(0, 0) defer bus.Close() - msg := NewMessage("test", "hello") - bus.Publish(msg) + msg1 := NewMessage("test", 1) + msg2 := NewMessage("test", 2) + + sub1, err := bus.Subscribe("test") + assert.Nil(t, err) + + sub2, err := bus.Subscribe("test") + assert.Nil(t, err) + + // ordering of publish is not deterministic + // so capture each in a go routine + var wg sync.WaitGroup + var event1 Message + var event2 Message + + go func() { + event1 = <-sub1.Message() + wg.Done() + }() + + go func() { + event2 = <-sub2.Message() + wg.Done() + }() + + wg.Add(2) + bus.Publish(msg1) + wg.Wait() + + assert.Equal(t, msg1, event1) + assert.Equal(t, msg1, event2) + + go func() { + event1 = <-sub1.Message() + wg.Done() + }() - sub := bus.Subscribe(1, "test") - select { - case <-sub.Message(): - t.Fatalf("subscriber should not recieve message") - case <-time.After(150 * time.Millisecond): - // wait for publish timeout + skew - } + go func() { + event2 = <-sub2.Message() + wg.Done() + }() + + wg.Add(2) + bus.Publish(msg2) + wg.Wait() + + assert.Equal(t, msg2, event1) + assert.Equal(t, msg2, event2) } -func TestBusSubscribeThenUnsubscribeThenPublish(t *testing.T) { - bus := NewBus(100 * time.Millisecond) +func TestSimpleEachSubscribersRecievesEachItemGivenBufferedEventChan(t *testing.T) { + bus := NewBus(0, 2) defer bus.Close() - sub := bus.Subscribe(1, "test") - bus.Unsubscribe(sub) + msg1 := NewMessage("test", 1) + msg2 := NewMessage("test", 2) - msg := NewMessage("test", "hello") - bus.Publish(msg) + sub1, err := bus.Subscribe("test") + assert.Nil(t, err) + sub2, err := bus.Subscribe("test") + assert.Nil(t, err) + + // both inputs are added first before read, using the internal chan buffer + bus.Publish(msg1) + bus.Publish(msg2) + + output1Ch1 := <-sub1.Message() + output1Ch2 := <-sub2.Message() + + output2Ch1 := <-sub1.Message() + output2Ch2 := <-sub2.Message() - _, ok := <-sub.Message() - assert.False(t, ok, "channel should be closed") + assert.Equal(t, msg1, output1Ch1) + assert.Equal(t, msg1, output1Ch2) + + assert.Equal(t, msg2, output2Ch1) + assert.Equal(t, msg2, output2Ch2) } -func TestBusUnsubscribeTwice(t *testing.T) { - bus := NewBus(100 * time.Millisecond) +func TestSimpleSubscribersDontRecieveItemsAfterUnsubscribing(t *testing.T) { + bus := NewBus(0, 0) defer bus.Close() - sub := bus.Subscribe(1, "test") - bus.Unsubscribe(sub) + sub, err := bus.Subscribe("test") + assert.Nil(t, err) bus.Unsubscribe(sub) + + msg := NewMessage("test", 1) + bus.Publish(msg) + + // tiny delay to try and make sure the internal logic would have had time + // to do its thing with the pushed item. + time.Sleep(5 * time.Millisecond) + + // closing the channel will result in reads yielding the default value + assert.Equal(t, Message{}, <-sub.Message()) } diff --git a/event/errors.go b/event/errors.go new file mode 100644 index 0000000000..68cac62308 --- /dev/null +++ b/event/errors.go @@ -0,0 +1,19 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package event + +import ( + "github.com/sourcenetwork/defradb/errors" +) + +var ( + ErrSubscribedToClosedChan = errors.New("cannot subscribe to a closed channel") +) diff --git a/internal/db/collection.go b/internal/db/collection.go index 38b59e1db9..e71ee2d13c 100644 --- a/internal/db/collection.go +++ b/internal/db/collection.go @@ -684,8 +684,8 @@ func (c *collection) save( Block: headNode, IsCreate: isCreate, } - txn.OnSuccessAsync(func() { - c.db.sysBus.Publish(event.NewMessage(event.UpdateEventName, updateEvent)) + txn.OnSuccess(func() { + c.db.events.Publish(event.NewMessage(event.UpdateEventName, updateEvent)) }) txn.OnSuccess(func() { diff --git a/internal/db/collection_delete.go b/internal/db/collection_delete.go index 0a6c870257..2ba1ca845e 100644 --- a/internal/db/collection_delete.go +++ b/internal/db/collection_delete.go @@ -173,8 +173,8 @@ func (c *collection) applyDelete( SchemaRoot: c.Schema().Root, Block: b, } - txn.OnSuccessAsync(func() { - c.db.sysBus.Publish(event.NewMessage(event.UpdateEventName, updateEvent)) + txn.OnSuccess(func() { + c.db.events.Publish(event.NewMessage(event.UpdateEventName, updateEvent)) }) return nil diff --git a/internal/db/db.go b/internal/db/db.go index 4e2535f087..701c796447 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -18,7 +18,6 @@ import ( "context" "sync" "sync/atomic" - "time" ds "github.com/ipfs/go-datastore" dsq "github.com/ipfs/go-datastore/query" @@ -45,21 +44,10 @@ var ( ) const ( - // forwardBufferSize controls the size of the channel used to - // forward events from the system bus to the subscription bus - forwardBufferSize = 100 - // subscriptionBufferSize controls the size of the channel used to - // send events to request subscriptions - subscriptionBufferSize = 10 - // mergeBufferSize controls the size of the channel used to - // handle merge events - mergeBufferSize = 100 - // sysBusTimeout is the duration to wait before discarding - // messages on the sysBus - sysBusTimeout = 5 * time.Minute - // subBusTimeout is the duration to wait before discarding - // messages on the subBus - subBusTimeout = 1 * time.Minute + // commandBufferSize is the size of the channel buffer used to handle events. + commandBufferSize = 100 + // eventBufferSize is the size of the channel buffer used to subscribe to events. + eventBufferSize = 100 ) // DB is the main interface for interacting with the @@ -70,11 +58,7 @@ type db struct { rootstore datastore.RootStore multistore datastore.MultiStore - // sysBus is used to send and receive system events - sysBus *event.Bus - - // subBus is used to send and receive request subscription events - subBus *event.Bus + events *event.Bus parser core.Parser @@ -125,8 +109,7 @@ func newDB( lensRegistry: lens, parser: parser, options: options, - sysBus: event.NewBus(sysBusTimeout), - subBus: event.NewBus(subBusTimeout), + events: event.NewBus(commandBufferSize, eventBufferSize), } // apply options @@ -142,6 +125,13 @@ func newDB( if err != nil { return nil, err } + + sub, err := db.events.Subscribe(event.MergeRequestEventName) + if err != nil { + return nil, err + } + go db.handleMerges(ctx, sub) + return db, nil } @@ -208,18 +198,6 @@ func (db *db) initialize(ctx context.Context) error { db.glock.Lock() defer db.glock.Unlock() - // start merge process - go db.handleMerges(ctx) - - // forward system bus events to the subscriber bus - // to ensure we never block the system bus for user subscriptions - go func() { - sub := db.sysBus.Subscribe(forwardBufferSize, event.WildCardEventName) - for msg := range sub.Message() { - db.subBus.Publish(msg) - } - }() - ctx, txn, err := ensureContextTxn(ctx, db, false) if err != nil { return err @@ -274,7 +252,7 @@ func (db *db) initialize(ctx context.Context) error { // Events returns the events Channel. func (db *db) Events() *event.Bus { - return db.sysBus + return db.events } // MaxRetries returns the maximum number of retries per transaction. @@ -296,8 +274,7 @@ func (db *db) PrintDump(ctx context.Context) error { func (db *db) Close() { log.Info("Closing DefraDB process...") - db.subBus.Close() - db.sysBus.Close() + db.events.Close() err := db.rootstore.Close() if err != nil { diff --git a/internal/db/merge.go b/internal/db/merge.go index bc6fefd68f..90fffafe84 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -33,10 +33,7 @@ import ( merklecrdt "github.com/sourcenetwork/defradb/internal/merkle/crdt" ) -func (db *db) handleMerges(ctx context.Context) { - sub := db.sysBus.Subscribe(mergeBufferSize, event.MergeRequestEventName) - defer db.sysBus.Unsubscribe(sub) - +func (db *db) handleMerges(ctx context.Context, sub *event.Subscription) { for { select { case <-ctx.Done(): @@ -67,7 +64,7 @@ func (db *db) handleMerges(ctx context.Context) { func (db *db) executeMerge(ctx context.Context, dagMerge event.MergeEvent) error { // send a complete event so we can track merges in the integration tests - defer db.sysBus.Publish(event.NewMessage(event.MergeCompleteEventName, dagMerge)) + defer db.events.Publish(event.NewMessage(event.MergeCompleteEventName, dagMerge)) ctx, txn, err := ensureContextTxn(ctx, db, false) if err != nil { diff --git a/internal/db/subscriptions.go b/internal/db/subscriptions.go index 7547b7fffe..378a89f231 100644 --- a/internal/db/subscriptions.go +++ b/internal/db/subscriptions.go @@ -32,11 +32,14 @@ func (db *db) handleSubscription(ctx context.Context, r *request.Request) (<-cha return nil, client.NewErrUnexpectedType[request.ObjectSubscription]("SubscriptionSelection", selections) } // subscribe to the subscription event bus so we don't block the system bus - sub := db.subBus.Subscribe(subscriptionBufferSize, event.UpdateEventName) + sub, err := db.events.Subscribe(event.UpdateEventName) + if err != nil { + return nil, err + } resCh := make(chan client.GQLResult) go func() { defer func() { - db.sysBus.Unsubscribe(sub) + db.events.Unsubscribe(sub) close(resCh) }() diff --git a/net/peer.go b/net/peer.go index 7c656829c5..33783bcce6 100644 --- a/net/peer.go +++ b/net/peer.go @@ -144,7 +144,11 @@ func (p *Peer) Start() error { } if p.ps != nil { - p.updateSub = p.db.Events().Subscribe(100, event.UpdateEventName) + sub, err := p.db.Events().Subscribe(event.UpdateEventName) + if err != nil { + return err + } + p.updateSub = sub log.InfoContext(p.ctx, "Starting internal broadcaster for pubsub network") go p.handleBroadcastLoop() } diff --git a/tests/integration/events/utils.go b/tests/integration/events/utils.go index b649612742..751d991368 100644 --- a/tests/integration/events/utils.go +++ b/tests/integration/events/utils.go @@ -80,7 +80,8 @@ func ExecuteRequestTestCase( testRoutineClosedChan := make(chan struct{}) closeTestRoutineChan := make(chan struct{}) - eventsSub := db.Events().Subscribe(5, event.UpdateEventName) + + eventsSub, err := db.Events().Subscribe(event.UpdateEventName) require.NoError(t, err) indexOfNextExpectedUpdate := 0 diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index d6b509e8f1..9463d76250 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -658,7 +658,9 @@ func setStartingNodes( s.nodes = append(s.nodes, c) s.dbPaths = append(s.dbPaths, path) - sub := c.Events().Subscribe(100, event.MergeCompleteEventName) + // subscribe to merge complete events + sub, err := c.Events().Subscribe(event.MergeCompleteEventName) + require.NoError(s.t, err) s.eventSubs = append(s.eventSubs, sub) } } @@ -813,7 +815,9 @@ func configureNode( s.nodes = append(s.nodes, c) s.dbPaths = append(s.dbPaths, path) - sub := c.Events().Subscribe(100, event.MergeCompleteEventName) + // subscribe to merge complete events + sub, err := c.Events().Subscribe(event.MergeCompleteEventName) + require.NoError(s.t, err) s.eventSubs = append(s.eventSubs, sub) } From 938954e4f65cc8dc0b7b769e5ef238c3b4606087 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 11 Jun 2024 16:19:06 -0700 Subject: [PATCH 11/34] resubscribe to events after restart --- tests/integration/utils2.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 9463d76250..098cc3fda5 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -713,6 +713,11 @@ func restartNodes( c, err := setupClient(s, n) require.NoError(s.t, err) s.nodes[i] = c + + // subscribe to merge complete events + sub, err := c.Events().Subscribe(event.MergeCompleteEventName) + require.NoError(s.t, err) + s.eventSubs[i] = sub } // The index of the action after the last wait action before the current restart action. From 6e98699f8153ca789fb1d1bf29a35f9b781c0481 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 12 Jun 2024 09:46:50 -0700 Subject: [PATCH 12/34] update event names --- event/event.go | 33 +++++++++++++++---------------- internal/db/collection.go | 2 +- internal/db/collection_delete.go | 2 +- internal/db/db.go | 2 +- internal/db/merge.go | 4 ++-- internal/db/merge_test.go | 10 +++++----- internal/db/subscriptions.go | 4 ++-- net/client.go | 2 +- net/client_test.go | 6 +++--- net/node.go | 2 +- net/peer.go | 10 +++++----- net/peer_test.go | 14 ++++++------- net/server.go | 6 +++--- tests/integration/events/utils.go | 2 +- tests/integration/p2p.go | 4 ++-- 15 files changed, 51 insertions(+), 52 deletions(-) diff --git a/event/event.go b/event/event.go index 148dfeffc2..a2a8464288 100644 --- a/event/event.go +++ b/event/event.go @@ -19,35 +19,34 @@ import ( const ( // WildCardEventName is the alias used to subscribe to all events. WildCardEventName = "*" + // MergeEventName is the name of the net merge request event. + MergeEventName = "merge" // MergeCompleteEventName is the name of the database merge complete event. - MergeCompleteEventName = "db:merge-complete" + MergeCompleteEventName = "merge-complete" // UpdateEventName is the name of the database update event. - UpdateEventName = "db:update" - // ResultsEventName is the name of the database results event. - ResultsEventName = "db:results" - // MergeRequestEventName is the name of the net merge request event. - MergeRequestEventName = "net:merge" + UpdateEventName = "update" // PubSubEventName is the name of the network pubsub event. - PubSubEventName = "net:pubsub" - // ConnectEventName is the name of the network connect event. - ConnectEventName = "net:connect" + PubSubEventName = "pubsub" + // PeerEventName is the name of the network connect event. + PeerEventName = "peer" ) -// ConnectEvent is an event that is published when +// Peer is an event that is published when // a peer connection has changed status. -type ConnectEvent = event.EvtPeerConnectednessChanged +type Peer = event.EvtPeerConnectednessChanged -// PubSubEvent is an event that is published when +// PubSub is an event that is published when // a pubsub message has been received from a remote peer. -type PubSubEvent struct { +type PubSub struct { + // Peer is the id of the peer that published the message. Peer peer.ID } -// UpdateEvent represents a new DAG node added to the append-only composite MerkleCRDT Clock graph +// Update represents a new DAG node added to the append-only composite MerkleCRDT Clock graph // of a document. // // It must only contain public elements not protected by ACP. -type UpdateEvent struct { +type Update struct { // DocID is the unique immutable identifier of the document that was updated. DocID string @@ -65,8 +64,8 @@ type UpdateEvent struct { IsCreate bool } -// MergeEvent is a notification that a merge can be performed up to the provided CID. -type MergeEvent struct { +// Merge is a notification that a merge can be performed up to the provided CID. +type Merge struct { // ByPeer is the id of the peer that created the push log request. ByPeer peer.ID diff --git a/internal/db/collection.go b/internal/db/collection.go index e71ee2d13c..78200dc101 100644 --- a/internal/db/collection.go +++ b/internal/db/collection.go @@ -677,7 +677,7 @@ func (c *collection) save( } // publish an update event when the txn succeeds - updateEvent := event.UpdateEvent{ + updateEvent := event.Update{ DocID: doc.ID().String(), Cid: link.Cid, SchemaRoot: c.Schema().Root, diff --git a/internal/db/collection_delete.go b/internal/db/collection_delete.go index 2ba1ca845e..b621103548 100644 --- a/internal/db/collection_delete.go +++ b/internal/db/collection_delete.go @@ -167,7 +167,7 @@ func (c *collection) applyDelete( } // publish an update event if the txn succeeds - updateEvent := event.UpdateEvent{ + updateEvent := event.Update{ DocID: primaryKey.DocID, Cid: link.Cid, SchemaRoot: c.Schema().Root, diff --git a/internal/db/db.go b/internal/db/db.go index 701c796447..3f99a3141e 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -126,7 +126,7 @@ func newDB( return nil, err } - sub, err := db.events.Subscribe(event.MergeRequestEventName) + sub, err := db.events.Subscribe(event.MergeEventName) if err != nil { return nil, err } diff --git a/internal/db/merge.go b/internal/db/merge.go index 90fffafe84..39d822a946 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -42,7 +42,7 @@ func (db *db) handleMerges(ctx context.Context, sub *event.Subscription) { if !ok { return } - merge, ok := msg.Data.(event.MergeEvent) + merge, ok := msg.Data.(event.Merge) if !ok { continue } @@ -62,7 +62,7 @@ func (db *db) handleMerges(ctx context.Context, sub *event.Subscription) { } } -func (db *db) executeMerge(ctx context.Context, dagMerge event.MergeEvent) error { +func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { // send a complete event so we can track merges in the integration tests defer db.events.Publish(event.NewMessage(event.MergeCompleteEventName, dagMerge)) diff --git a/internal/db/merge_test.go b/internal/db/merge_test.go index 624b2d97aa..cac26a1129 100644 --- a/internal/db/merge_test.go +++ b/internal/db/merge_test.go @@ -58,7 +58,7 @@ func TestMerge_SingleBranch_NoError(t *testing.T) { compInfo2, err := d.generateCompositeUpdate(&lsys, map[string]any{"name": "Johny"}, compInfo) require.NoError(t, err) - err = db.executeMerge(ctx, event.MergeEvent{ + err = db.executeMerge(ctx, event.Merge{ Cid: compInfo2.link.Cid, SchemaRoot: col.SchemaRoot(), }) @@ -102,7 +102,7 @@ func TestMerge_DualBranch_NoError(t *testing.T) { compInfo2, err := d.generateCompositeUpdate(&lsys, map[string]any{"name": "Johny"}, compInfo) require.NoError(t, err) - err = db.executeMerge(ctx, event.MergeEvent{ + err = db.executeMerge(ctx, event.Merge{ Cid: compInfo2.link.Cid, SchemaRoot: col.SchemaRoot(), }) @@ -111,7 +111,7 @@ func TestMerge_DualBranch_NoError(t *testing.T) { compInfo3, err := d.generateCompositeUpdate(&lsys, map[string]any{"age": 30}, compInfo) require.NoError(t, err) - err = db.executeMerge(ctx, event.MergeEvent{ + err = db.executeMerge(ctx, event.Merge{ Cid: compInfo3.link.Cid, SchemaRoot: col.SchemaRoot(), }) @@ -158,7 +158,7 @@ func TestMerge_DualBranchWithOneIncomplete_CouldNotFindCID(t *testing.T) { compInfo2, err := d.generateCompositeUpdate(&lsys, map[string]any{"name": "Johny"}, compInfo) require.NoError(t, err) - err = db.executeMerge(ctx, event.MergeEvent{ + err = db.executeMerge(ctx, event.Merge{ Cid: compInfo2.link.Cid, SchemaRoot: col.SchemaRoot(), }) @@ -176,7 +176,7 @@ func TestMerge_DualBranchWithOneIncomplete_CouldNotFindCID(t *testing.T) { compInfo3, err := d.generateCompositeUpdate(&lsys, map[string]any{"name": "Johny"}, compInfoUnkown) require.NoError(t, err) - err = db.executeMerge(ctx, event.MergeEvent{ + err = db.executeMerge(ctx, event.Merge{ Cid: compInfo3.link.Cid, SchemaRoot: col.SchemaRoot(), }) diff --git a/internal/db/subscriptions.go b/internal/db/subscriptions.go index 378a89f231..1798e386a1 100644 --- a/internal/db/subscriptions.go +++ b/internal/db/subscriptions.go @@ -45,7 +45,7 @@ func (db *db) handleSubscription(ctx context.Context, r *request.Request) (<-cha // listen for events and send to the result channel for { - var evt event.UpdateEvent + var evt event.Update select { case <-ctx.Done(): return // context cancelled @@ -53,7 +53,7 @@ func (db *db) handleSubscription(ctx context.Context, r *request.Request) (<-cha if !ok { return // channel closed } - evt, ok = val.Data.(event.UpdateEvent) + evt, ok = val.Data.(event.Update) if !ok { continue // invalid event value } diff --git a/net/client.go b/net/client.go index a59421c3ba..9930710891 100644 --- a/net/client.go +++ b/net/client.go @@ -31,7 +31,7 @@ var ( // pushLog creates a pushLog request and sends it to another node // over libp2p grpc connection -func (s *server) pushLog(ctx context.Context, evt event.UpdateEvent, pid peer.ID) error { +func (s *server) pushLog(ctx context.Context, evt event.Update, pid peer.ID) error { body := &pb.PushLogRequest_Body{ DocID: []byte(evt.DocID), Cid: evt.Cid.Bytes(), diff --git a/net/client_test.go b/net/client_test.go index 37bf09ddee..6a43805ae8 100644 --- a/net/client_test.go +++ b/net/client_test.go @@ -62,7 +62,7 @@ func TestPushlogWithDialFailure(t *testing.T) { grpc.WithCredentialsBundle(nil), ) - err = n.server.pushLog(ctx, event.UpdateEvent{ + err = n.server.pushLog(ctx, event.Update{ DocID: id.String(), Cid: cid, SchemaRoot: "test", @@ -84,7 +84,7 @@ func TestPushlogWithInvalidPeerID(t *testing.T) { cid, err := createCID(doc) require.NoError(t, err) - err = n.server.pushLog(ctx, event.UpdateEvent{ + err = n.server.pushLog(ctx, event.Update{ DocID: id.String(), Cid: cid, SchemaRoot: "test", @@ -135,7 +135,7 @@ func TestPushlogW_WithValidPeerID_NoError(t *testing.T) { b, err := n1.db.Blockstore().AsIPLDStorage().Get(ctx, headCID.KeyString()) require.NoError(t, err) - err = n1.server.pushLog(ctx, event.UpdateEvent{ + err = n1.server.pushLog(ctx, event.Update{ DocID: doc.ID().String(), Cid: headCID, SchemaRoot: col.SchemaRoot(), diff --git a/net/node.go b/net/node.go index cb429c71d3..6f08dec00b 100644 --- a/net/node.go +++ b/net/node.go @@ -188,7 +188,7 @@ func NewNode( // publish subscribed events to the event bus go func() { for val := range sub.Out() { - db.Events().Publish(event1.NewMessage(event1.ConnectEventName, val)) + db.Events().Publish(event1.NewMessage(event1.PeerEventName, val)) } }() diff --git a/net/peer.go b/net/peer.go index 33783bcce6..32255afa43 100644 --- a/net/peer.go +++ b/net/peer.go @@ -207,7 +207,7 @@ func (p *Peer) handleBroadcastLoop() { if !isOpen { return } - update, ok := msg.Data.(event.UpdateEvent) + update, ok := msg.Data.(event.Update) if !ok { continue // ignore invalid value } @@ -299,7 +299,7 @@ func (p *Peer) pushToReplicator( continue } - evt := event.UpdateEvent{ + evt := event.Update{ DocID: docIDResult.ID.String(), Cid: c, SchemaRoot: collection.SchemaRoot(), @@ -366,7 +366,7 @@ func (p *Peer) loadP2PCollections(ctx context.Context) (map[string]struct{}, err return colMap, nil } -func (p *Peer) handleDocCreateLog(evt event.UpdateEvent) error { +func (p *Peer) handleDocCreateLog(evt event.Update) error { docID, err := client.NewDocIDFromString(evt.DocID) if err != nil { return NewErrFailedToGetDocID(err) @@ -384,7 +384,7 @@ func (p *Peer) handleDocCreateLog(evt event.UpdateEvent) error { return nil } -func (p *Peer) handleDocUpdateLog(evt event.UpdateEvent) error { +func (p *Peer) handleDocUpdateLog(evt event.Update) error { docID, err := client.NewDocIDFromString(evt.DocID) if err != nil { return NewErrFailedToGetDocID(err) @@ -417,7 +417,7 @@ func (p *Peer) handleDocUpdateLog(evt event.UpdateEvent) error { return nil } -func (p *Peer) pushLogToReplicators(lg event.UpdateEvent) { +func (p *Peer) pushLogToReplicators(lg event.Update) { // push to each peer (replicator) peers := make(map[string]struct{}) for _, peer := range p.ps.ListPeers(lg.DocID) { diff --git a/net/peer_test.go b/net/peer_test.go index 404651f9d0..f547894788 100644 --- a/net/peer_test.go +++ b/net/peer_test.go @@ -888,7 +888,7 @@ func TestHandleDocCreateLog_NoError(t *testing.T) { b, err := db.Blockstore().AsIPLDStorage().Get(ctx, headCID.KeyString()) require.NoError(t, err) - err = n.handleDocCreateLog(event.UpdateEvent{ + err = n.handleDocCreateLog(event.Update{ DocID: doc.ID().String(), Cid: headCID, SchemaRoot: col.SchemaRoot(), @@ -902,7 +902,7 @@ func TestHandleDocCreateLog_WithInvalidDocID_NoError(t *testing.T) { _, n := newTestNode(ctx, t) defer n.Close() - err := n.handleDocCreateLog(event.UpdateEvent{ + err := n.handleDocCreateLog(event.Update{ DocID: "some-invalid-key", }) require.ErrorContains(t, err, "failed to get DocID from broadcast message: selected encoding not supported") @@ -931,7 +931,7 @@ func TestHandleDocCreateLog_WithExistingTopic_TopicExistsError(t *testing.T) { _, err = rpc.NewTopic(ctx, n.ps, n.host.ID(), doc.ID().String(), true) require.NoError(t, err) - err = n.handleDocCreateLog(event.UpdateEvent{ + err = n.handleDocCreateLog(event.Update{ DocID: doc.ID().String(), SchemaRoot: col.SchemaRoot(), }) @@ -964,7 +964,7 @@ func TestHandleDocUpdateLog_NoError(t *testing.T) { b, err := db.Blockstore().AsIPLDStorage().Get(ctx, headCID.KeyString()) require.NoError(t, err) - err = n.handleDocUpdateLog(event.UpdateEvent{ + err = n.handleDocUpdateLog(event.Update{ DocID: doc.ID().String(), Cid: headCID, SchemaRoot: col.SchemaRoot(), @@ -978,7 +978,7 @@ func TestHandleDoUpdateLog_WithInvalidDocID_NoError(t *testing.T) { _, n := newTestNode(ctx, t) defer n.Close() - err := n.handleDocUpdateLog(event.UpdateEvent{ + err := n.handleDocUpdateLog(event.Update{ DocID: "some-invalid-key", }) require.ErrorContains(t, err, "failed to get DocID from broadcast message: selected encoding not supported") @@ -1013,7 +1013,7 @@ func TestHandleDocUpdateLog_WithExistingDocIDTopic_TopicExistsError(t *testing.T _, err = rpc.NewTopic(ctx, n.ps, n.host.ID(), doc.ID().String(), true) require.NoError(t, err) - err = n.handleDocUpdateLog(event.UpdateEvent{ + err = n.handleDocUpdateLog(event.Update{ DocID: doc.ID().String(), Cid: headCID, SchemaRoot: col.SchemaRoot(), @@ -1051,7 +1051,7 @@ func TestHandleDocUpdateLog_WithExistingSchemaTopic_TopicExistsError(t *testing. _, err = rpc.NewTopic(ctx, n.ps, n.host.ID(), col.SchemaRoot(), true) require.NoError(t, err) - err = n.handleDocUpdateLog(event.UpdateEvent{ + err = n.handleDocUpdateLog(event.Update{ DocID: doc.ID().String(), Cid: headCID, SchemaRoot: col.SchemaRoot(), diff --git a/net/server.go b/net/server.go index 761d87e4c6..a1787098f3 100644 --- a/net/server.go +++ b/net/server.go @@ -192,7 +192,7 @@ func (s *server) PushLog(ctx context.Context, req *pb.PushLogRequest) (*pb.PushL s.docQueue.add(docID.String()) defer s.docQueue.done(docID.String()) - evt := event.MergeEvent{ + evt := event.Merge{ ByPeer: byPeer, FromPeer: pid, Cid: cid, @@ -226,7 +226,7 @@ func (s *server) PushLog(ctx context.Context, req *pb.PushLogRequest) (*pb.PushL corelog.Any("CID", cid), ) } - s.peer.db.Events().Publish(event.NewMessage(event.MergeRequestEventName, evt)) + s.peer.db.Events().Publish(event.NewMessage(event.MergeEventName, evt)) // Once processed, subscribe to the DocID topic on the pubsub network unless we already // suscribe to the collection. @@ -373,7 +373,7 @@ func (s *server) pubSubEventHandler(from libpeer.ID, topic string, msg []byte) { corelog.String("Topic", topic), corelog.String("Message", string(msg)), ) - evt := event.NewMessage(event.PubSubEventName, event.PubSubEvent{ + evt := event.NewMessage(event.PubSubEventName, event.PubSub{ Peer: from, }) s.peer.db.Events().Publish(evt) diff --git a/tests/integration/events/utils.go b/tests/integration/events/utils.go index 751d991368..74ed0a7da4 100644 --- a/tests/integration/events/utils.go +++ b/tests/integration/events/utils.go @@ -89,7 +89,7 @@ func ExecuteRequestTestCase( for { select { case value := <-eventsSub.Message(): - update, ok := value.Data.(event.UpdateEvent) + update, ok := value.Data.(event.Update) if !ok { continue // ignore invalid value } diff --git a/tests/integration/p2p.go b/tests/integration/p2p.go index 233deb59ad..96e3fca492 100644 --- a/tests/integration/p2p.go +++ b/tests/integration/p2p.go @@ -515,14 +515,14 @@ func waitForMerge( // wait for message or unsubscribe msg, ok := <-sourceSub.Message() if ok { - require.Equal(s.t, targetPeerInfo.ID, msg.Data.(event.MergeEvent).ByPeer) + require.Equal(s.t, targetPeerInfo.ID, msg.Data.(event.Merge).ByPeer) } } for i := 0; i < sourceToTargetEvents[waitIndex]; i++ { // wait for message or unsubscribe msg, ok := <-targetSub.Message() if ok { - require.Equal(s.t, sourcePeerInfo.ID, msg.Data.(event.MergeEvent).ByPeer) + require.Equal(s.t, sourcePeerInfo.ID, msg.Data.(event.Merge).ByPeer) } } nodeSynced <- struct{}{} From 1a1ce8f3942c045ed558297144367b9391a5afc0 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 12 Jun 2024 09:50:17 -0700 Subject: [PATCH 13/34] update event bus publish logic --- event/bus.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/event/bus.go b/event/bus.go index 30c15e9b8e..3cc11259f2 100644 --- a/event/bus.go +++ b/event/bus.go @@ -164,14 +164,13 @@ func (b *Bus) handleChannel() { close(t.value) case publishCommand: - subscribers := make(map[uint64]struct{}) - for id := range b.events[t.Name] { - subscribers[id] = struct{}{} - } for id := range b.events[WildCardEventName] { - subscribers[id] = struct{}{} + b.subs[id].value <- Message(t) } - for id := range subscribers { + for id := range b.events[t.Name] { + if _, ok := b.events[WildCardEventName][id]; ok { + continue + } b.subs[id].value <- Message(t) } } From 9ce688f402b515f45e617f17baa9a2e32e1ee6de Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 12 Jun 2024 09:51:29 -0700 Subject: [PATCH 14/34] update command buffer size --- internal/db/db.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/db/db.go b/internal/db/db.go index 3f99a3141e..39c121960e 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -45,7 +45,7 @@ var ( const ( // commandBufferSize is the size of the channel buffer used to handle events. - commandBufferSize = 100 + commandBufferSize = 100_000 // eventBufferSize is the size of the channel buffer used to subscribe to events. eventBufferSize = 100 ) From 7e1045f79db37589e5095328ce850baa79e40606 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 12 Jun 2024 09:53:08 -0700 Subject: [PATCH 15/34] document p2p waitForMerge require --- tests/integration/p2p.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/p2p.go b/tests/integration/p2p.go index 96e3fca492..cd1dbd3b5c 100644 --- a/tests/integration/p2p.go +++ b/tests/integration/p2p.go @@ -515,6 +515,7 @@ func waitForMerge( // wait for message or unsubscribe msg, ok := <-sourceSub.Message() if ok { + // ensure the message is sent from the target node require.Equal(s.t, targetPeerInfo.ID, msg.Data.(event.Merge).ByPeer) } } @@ -522,6 +523,7 @@ func waitForMerge( // wait for message or unsubscribe msg, ok := <-targetSub.Message() if ok { + // ensure the message is sent from the source node require.Equal(s.t, sourcePeerInfo.ID, msg.Data.(event.Merge).ByPeer) } } From 573090e1e07184d5b6eddb0a16ce428e28709b27 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 12 Jun 2024 10:11:01 -0700 Subject: [PATCH 16/34] add event bus interface --- client/db.go | 2 +- event/buffered_bus.go | 141 ++++++++++++++++++ event/{bus_test.go => buffered_bus_test.go} | 10 +- event/bus.go | 149 +++----------------- http/client.go | 2 +- internal/db/db.go | 6 +- tests/clients/cli/wrapper.go | 2 +- tests/clients/http/wrapper.go | 2 +- 8 files changed, 169 insertions(+), 145 deletions(-) create mode 100644 event/buffered_bus.go rename event/{bus_test.go => buffered_bus_test.go} (95%) diff --git a/client/db.go b/client/db.go index e52dfed60a..4ec9643fff 100644 --- a/client/db.go +++ b/client/db.go @@ -75,7 +75,7 @@ type DB interface { // // It may be used to monitor database events - a new event will be yielded for each mutation. // Note: it does not copy the queue, just the reference to it. - Events() *event.Bus + Events() event.Bus // MaxTxnRetries returns the number of retries that this DefraDB instance has been configured to // make in the event of a transaction conflict in certain scenarios. diff --git a/event/buffered_bus.go b/event/buffered_bus.go new file mode 100644 index 0000000000..ed5102eb93 --- /dev/null +++ b/event/buffered_bus.go @@ -0,0 +1,141 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package event + +import ( + "sync/atomic" +) + +type subscribeCommand *Subscription + +type unsubscribeCommand *Subscription + +type publishCommand Message + +type closeCommand struct{} + +// bufferedBus is a bus that uses a buffered channel to manage subscribers and publish messages. +type bufferedBus struct { + // subID is incremented for each subscriber and used to set subscriber ids. + subID atomic.Uint64 + // subs is a mapping of subscriber ids to subscriptions. + subs map[uint64]*Subscription + // events is a mapping of event names to subscriber ids. + events map[string]map[uint64]struct{} + // commandChannel manages all commands sent to this simpleChannel. + // + // It is important that all stuff gets sent through this single channel to ensure + // that the order of operations is preserved. + // + // WARNING: This does mean that non-event commands can block the database if the buffer + // size is breached (e.g. if many subscribe commands occupy the buffer). + commandChannel chan any + eventBufferSize int + hasClosedChan chan struct{} + isClosed atomic.Bool +} + +// NewBufferedBus creates a new event bus with the given commandBufferSize and +// eventBufferSize. +// +// Should the buffers be filled, subsequent calls on this bus will block. +func NewBufferedBus(commandBufferSize int, eventBufferSize int) Bus { + bus := bufferedBus{ + subs: make(map[uint64]*Subscription), + events: make(map[string]map[uint64]struct{}), + commandChannel: make(chan any, commandBufferSize), + hasClosedChan: make(chan struct{}), + eventBufferSize: eventBufferSize, + } + go bus.handleChannel() + return &bus +} + +func (b *bufferedBus) Publish(msg Message) { + if b.isClosed.Load() { + return + } + b.commandChannel <- publishCommand(msg) +} + +func (b *bufferedBus) Subscribe(events ...string) (*Subscription, error) { + if b.isClosed.Load() { + return nil, ErrSubscribedToClosedChan + } + sub := &Subscription{ + id: b.subID.Add(1), + value: make(chan Message, b.eventBufferSize), + events: events, + } + b.commandChannel <- subscribeCommand(sub) + return sub, nil +} + +func (b *bufferedBus) Unsubscribe(sub *Subscription) { + if b.isClosed.Load() { + return + } + b.commandChannel <- unsubscribeCommand(sub) +} + +func (b *bufferedBus) Close() { + if b.isClosed.Load() { + return + } + b.isClosed.Store(true) + b.commandChannel <- closeCommand{} + // Wait for the close command to be handled, in order, before returning + <-b.hasClosedChan +} + +func (b *bufferedBus) handleChannel() { + for cmd := range b.commandChannel { + switch t := cmd.(type) { + case closeCommand: + for _, subscriber := range b.subs { + close(subscriber.value) + } + close(b.commandChannel) + close(b.hasClosedChan) + return + + case subscribeCommand: + for _, event := range t.events { + if _, ok := b.events[event]; !ok { + b.events[event] = make(map[uint64]struct{}) + } + b.events[event][t.id] = struct{}{} + } + b.subs[t.id] = t + + case unsubscribeCommand: + if _, ok := b.subs[t.id]; !ok { + continue // not subscribed + } + for _, event := range t.events { + delete(b.events[event], t.id) + } + delete(b.subs, t.id) + close(t.value) + + case publishCommand: + for id := range b.events[WildCardEventName] { + b.subs[id].value <- Message(t) + } + for id := range b.events[t.Name] { + if _, ok := b.events[WildCardEventName][id]; ok { + continue + } + b.subs[id].value <- Message(t) + } + } + } +} diff --git a/event/bus_test.go b/event/buffered_bus_test.go similarity index 95% rename from event/bus_test.go rename to event/buffered_bus_test.go index bbbbad3111..3b2a0fd1ed 100644 --- a/event/bus_test.go +++ b/event/buffered_bus_test.go @@ -19,7 +19,7 @@ import ( ) func TestSimplePushIsNotBlockedWithoutSubscribers(t *testing.T) { - bus := NewBus(0, 0) + bus := NewBufferedBus(0, 0) defer bus.Close() msg := NewMessage("test", 1) @@ -30,7 +30,7 @@ func TestSimplePushIsNotBlockedWithoutSubscribers(t *testing.T) { } func TestSimpleSubscribersAreNotBlockedAfterClose(t *testing.T) { - bus := NewBus(0, 0) + bus := NewBufferedBus(0, 0) defer bus.Close() sub, err := bus.Subscribe("test") @@ -45,7 +45,7 @@ func TestSimpleSubscribersAreNotBlockedAfterClose(t *testing.T) { } func TestSimpleEachSubscribersRecievesEachItem(t *testing.T) { - bus := NewBus(0, 0) + bus := NewBufferedBus(0, 0) defer bus.Close() msg1 := NewMessage("test", 1) @@ -99,7 +99,7 @@ func TestSimpleEachSubscribersRecievesEachItem(t *testing.T) { } func TestSimpleEachSubscribersRecievesEachItemGivenBufferedEventChan(t *testing.T) { - bus := NewBus(0, 2) + bus := NewBufferedBus(0, 2) defer bus.Close() msg1 := NewMessage("test", 1) @@ -128,7 +128,7 @@ func TestSimpleEachSubscribersRecievesEachItemGivenBufferedEventChan(t *testing. } func TestSimpleSubscribersDontRecieveItemsAfterUnsubscribing(t *testing.T) { - bus := NewBus(0, 0) + bus := NewBufferedBus(0, 0) defer bus.Close() sub, err := bus.Subscribe("test") diff --git a/event/bus.go b/event/bus.go index 3cc11259f2..d43b237db7 100644 --- a/event/bus.go +++ b/event/bus.go @@ -10,17 +10,25 @@ package event -import ( - "sync/atomic" -) - -type subscribeCommand *Subscription +// Bus is an event bus used to broadcasts messages to subscribers. +type Bus interface { + // Subscribe subscribes to the Channel, returning a channel by which events can + // be read from, or an error should one occur (e.g. if this object is closed). + // + // This function is non-blocking unless the subscription-buffer is full. + Subscribe(events ...string) (*Subscription, error) -type unsubscribeCommand *Subscription + // Unsubscribe unsubscribes from the Channel, closing the provided channel. + // + // Will do nothing if this object is already closed. + Unsubscribe(sub *Subscription) -type publishCommand Message + // Publish pushes the given item into this channel. Non-blocking. + Publish(msg Message) -type closeCommand struct{} + // Close closes this Channel, and any owned or subscribing channels. + Close() +} // Message contains event info. type Message struct { @@ -51,128 +59,3 @@ func (s *Subscription) Message() <-chan Message { func (s *Subscription) Events() []string { return s.events } - -// Bus is used to broadcast events to subscribers. -type Bus struct { - // subID is incremented for each subscriber and used to set subscriber ids. - subID atomic.Uint64 - // subs is a mapping of subscriber ids to subscriptions. - subs map[uint64]*Subscription - // events is a mapping of event names to subscriber ids. - events map[string]map[uint64]struct{} - // commandChannel manages all commands sent to this simpleChannel. - // - // It is important that all stuff gets sent through this single channel to ensure - // that the order of operations is preserved. - // - // WARNING: This does mean that non-event commands can block the database if the buffer - // size is breached (e.g. if many subscribe commands occupy the buffer). - commandChannel chan any - eventBufferSize int - hasClosedChan chan struct{} - isClosed atomic.Bool -} - -// NewBus creates a new event bus with the given commandBufferSize and -// eventBufferSize. -// -// Should the buffers be filled subsequent calls to functions on this object may start to block. -func NewBus(commandBufferSize int, eventBufferSize int) *Bus { - bus := Bus{ - subs: make(map[uint64]*Subscription), - events: make(map[string]map[uint64]struct{}), - commandChannel: make(chan any, commandBufferSize), - hasClosedChan: make(chan struct{}), - eventBufferSize: eventBufferSize, - } - go bus.handleChannel() - return &bus -} - -// Publish publishes the given event message to all subscribers. -func (b *Bus) Publish(msg Message) { - if b.isClosed.Load() { - return - } - b.commandChannel <- publishCommand(msg) -} - -// Subscribe returns a new channel that will receive all of the subscribed events. -func (b *Bus) Subscribe(events ...string) (*Subscription, error) { - if b.isClosed.Load() { - return nil, ErrSubscribedToClosedChan - } - - sub := &Subscription{ - id: b.subID.Add(1), - value: make(chan Message, b.eventBufferSize), - events: events, - } - - b.commandChannel <- subscribeCommand(sub) - return sub, nil -} - -// Unsubscribe unsubscribes from all events and closes the event channel of the given subscription. -func (b *Bus) Unsubscribe(sub *Subscription) { - if b.isClosed.Load() { - return - } - b.commandChannel <- unsubscribeCommand(sub) -} - -// Close closes the event bus by unsubscribing all subscribers. -func (b *Bus) Close() { - if b.isClosed.Load() { - return - } - b.isClosed.Store(true) - b.commandChannel <- closeCommand{} - - // Wait for the close command to be handled, in order, before returning - <-b.hasClosedChan -} - -func (b *Bus) handleChannel() { - for cmd := range b.commandChannel { - switch t := cmd.(type) { - case closeCommand: - for _, subscriber := range b.subs { - close(subscriber.value) - } - close(b.commandChannel) - close(b.hasClosedChan) - return - - case subscribeCommand: - for _, event := range t.events { - if _, ok := b.events[event]; !ok { - b.events[event] = make(map[uint64]struct{}) - } - b.events[event][t.id] = struct{}{} - } - b.subs[t.id] = t - - case unsubscribeCommand: - if _, ok := b.subs[t.id]; !ok { - continue // not subscribed - } - for _, event := range t.events { - delete(b.events[event], t.id) - } - delete(b.subs, t.id) - close(t.value) - - case publishCommand: - for id := range b.events[WildCardEventName] { - b.subs[id].value <- Message(t) - } - for id := range b.events[t.Name] { - if _, ok := b.events[WildCardEventName][id]; ok { - continue - } - b.subs[id].value <- Message(t) - } - } - } -} diff --git a/http/client.go b/http/client.go index 2843ee4f2d..e1fb1034ce 100644 --- a/http/client.go +++ b/http/client.go @@ -451,7 +451,7 @@ func (c *Client) Headstore() ds.Read { panic("client side database") } -func (c *Client) Events() *event.Bus { +func (c *Client) Events() event.Bus { panic("client side database") } diff --git a/internal/db/db.go b/internal/db/db.go index 39c121960e..861d2a1fb2 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -58,7 +58,7 @@ type db struct { rootstore datastore.RootStore multistore datastore.MultiStore - events *event.Bus + events event.Bus parser core.Parser @@ -109,7 +109,7 @@ func newDB( lensRegistry: lens, parser: parser, options: options, - events: event.NewBus(commandBufferSize, eventBufferSize), + events: event.NewBufferedBus(commandBufferSize, eventBufferSize), } // apply options @@ -251,7 +251,7 @@ func (db *db) initialize(ctx context.Context) error { } // Events returns the events Channel. -func (db *db) Events() *event.Bus { +func (db *db) Events() event.Bus { return db.events } diff --git a/tests/clients/cli/wrapper.go b/tests/clients/cli/wrapper.go index 18e306c0f4..1681dd6e7b 100644 --- a/tests/clients/cli/wrapper.go +++ b/tests/clients/cli/wrapper.go @@ -525,7 +525,7 @@ func (w *Wrapper) Close() { w.node.Close() } -func (w *Wrapper) Events() *event.Bus { +func (w *Wrapper) Events() event.Bus { return w.node.Events() } diff --git a/tests/clients/http/wrapper.go b/tests/clients/http/wrapper.go index 89b5bce5e7..7288781ba5 100644 --- a/tests/clients/http/wrapper.go +++ b/tests/clients/http/wrapper.go @@ -221,7 +221,7 @@ func (w *Wrapper) Close() { w.node.Close() } -func (w *Wrapper) Events() *event.Bus { +func (w *Wrapper) Events() event.Bus { return w.node.Events() } From d6a1be1f05f8f2244edc50333b007419f109f43e Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 12 Jun 2024 14:50:40 -0700 Subject: [PATCH 17/34] add event.Name type --- event/buffered_bus.go | 15 +++++++++------ event/bus.go | 11 ++++++----- event/event.go | 27 +++++++++++++++------------ internal/db/collection.go | 2 +- internal/db/collection_delete.go | 2 +- internal/db/db.go | 2 +- internal/db/merge.go | 2 +- internal/db/subscriptions.go | 2 +- net/node.go | 2 +- net/peer.go | 2 +- net/server.go | 6 +++--- tests/integration/events/utils.go | 2 +- tests/integration/utils2.go | 6 +++--- 13 files changed, 44 insertions(+), 37 deletions(-) diff --git a/event/buffered_bus.go b/event/buffered_bus.go index ed5102eb93..45a0c4a2e4 100644 --- a/event/buffered_bus.go +++ b/event/buffered_bus.go @@ -22,6 +22,9 @@ type publishCommand Message type closeCommand struct{} +// bufferedBus must implement the Bus interface +var _ (Bus) = (*bufferedBus)(nil) + // bufferedBus is a bus that uses a buffered channel to manage subscribers and publish messages. type bufferedBus struct { // subID is incremented for each subscriber and used to set subscriber ids. @@ -29,7 +32,7 @@ type bufferedBus struct { // subs is a mapping of subscriber ids to subscriptions. subs map[uint64]*Subscription // events is a mapping of event names to subscriber ids. - events map[string]map[uint64]struct{} + events map[Name]map[uint64]struct{} // commandChannel manages all commands sent to this simpleChannel. // // It is important that all stuff gets sent through this single channel to ensure @@ -47,10 +50,10 @@ type bufferedBus struct { // eventBufferSize. // // Should the buffers be filled, subsequent calls on this bus will block. -func NewBufferedBus(commandBufferSize int, eventBufferSize int) Bus { +func NewBufferedBus(commandBufferSize int, eventBufferSize int) *bufferedBus { bus := bufferedBus{ subs: make(map[uint64]*Subscription), - events: make(map[string]map[uint64]struct{}), + events: make(map[Name]map[uint64]struct{}), commandChannel: make(chan any, commandBufferSize), hasClosedChan: make(chan struct{}), eventBufferSize: eventBufferSize, @@ -66,7 +69,7 @@ func (b *bufferedBus) Publish(msg Message) { b.commandChannel <- publishCommand(msg) } -func (b *bufferedBus) Subscribe(events ...string) (*Subscription, error) { +func (b *bufferedBus) Subscribe(events ...Name) (*Subscription, error) { if b.isClosed.Load() { return nil, ErrSubscribedToClosedChan } @@ -127,11 +130,11 @@ func (b *bufferedBus) handleChannel() { close(t.value) case publishCommand: - for id := range b.events[WildCardEventName] { + for id := range b.events[WildCardName] { b.subs[id].value <- Message(t) } for id := range b.events[t.Name] { - if _, ok := b.events[WildCardEventName][id]; ok { + if _, ok := b.events[WildCardName][id]; ok { continue } b.subs[id].value <- Message(t) diff --git a/event/bus.go b/event/bus.go index d43b237db7..e202cc2f8b 100644 --- a/event/bus.go +++ b/event/bus.go @@ -16,7 +16,7 @@ type Bus interface { // be read from, or an error should one occur (e.g. if this object is closed). // // This function is non-blocking unless the subscription-buffer is full. - Subscribe(events ...string) (*Subscription, error) + Subscribe(events ...Name) (*Subscription, error) // Unsubscribe unsubscribes from the Channel, closing the provided channel. // @@ -33,13 +33,14 @@ type Bus interface { // Message contains event info. type Message struct { // Name is the name of the event this message was generated from. - Name string + Name Name + // Data contains optional event information. Data any } // NewMessage returns a new message with the given name and optional data. -func NewMessage(name string, data any) Message { +func NewMessage(name Name, data any) Message { return Message{name, data} } @@ -47,7 +48,7 @@ func NewMessage(name string, data any) Message { type Subscription struct { id uint64 value chan Message - events []string + events []Name } // Message returns the next event value from the subscription. @@ -56,6 +57,6 @@ func (s *Subscription) Message() <-chan Message { } // Events returns the names of all subscribed events. -func (s *Subscription) Events() []string { +func (s *Subscription) Events() []Name { return s.events } diff --git a/event/event.go b/event/event.go index a2a8464288..9c7c2e66b5 100644 --- a/event/event.go +++ b/event/event.go @@ -16,19 +16,22 @@ import ( "github.com/libp2p/go-libp2p/core/peer" ) +// Name identifies an event +type Name string + const ( - // WildCardEventName is the alias used to subscribe to all events. - WildCardEventName = "*" - // MergeEventName is the name of the net merge request event. - MergeEventName = "merge" - // MergeCompleteEventName is the name of the database merge complete event. - MergeCompleteEventName = "merge-complete" - // UpdateEventName is the name of the database update event. - UpdateEventName = "update" - // PubSubEventName is the name of the network pubsub event. - PubSubEventName = "pubsub" - // PeerEventName is the name of the network connect event. - PeerEventName = "peer" + // WildCardName is the alias used to subscribe to all events. + WildCardName = Name("*") + // MergeName is the name of the net merge request event. + MergeName = Name("merge") + // MergeCompleteName is the name of the database merge complete event. + MergeCompleteName = Name("merge-complete") + // UpdateName is the name of the database update event. + UpdateName = Name("update") + // PubSubName is the name of the network pubsub event. + PubSubName = Name("pubsub") + // PeerName is the name of the network connect event. + PeerName = Name("peer") ) // Peer is an event that is published when diff --git a/internal/db/collection.go b/internal/db/collection.go index 78200dc101..7e20f0da8f 100644 --- a/internal/db/collection.go +++ b/internal/db/collection.go @@ -685,7 +685,7 @@ func (c *collection) save( IsCreate: isCreate, } txn.OnSuccess(func() { - c.db.events.Publish(event.NewMessage(event.UpdateEventName, updateEvent)) + c.db.events.Publish(event.NewMessage(event.UpdateName, updateEvent)) }) txn.OnSuccess(func() { diff --git a/internal/db/collection_delete.go b/internal/db/collection_delete.go index b621103548..082a53caf2 100644 --- a/internal/db/collection_delete.go +++ b/internal/db/collection_delete.go @@ -174,7 +174,7 @@ func (c *collection) applyDelete( Block: b, } txn.OnSuccess(func() { - c.db.events.Publish(event.NewMessage(event.UpdateEventName, updateEvent)) + c.db.events.Publish(event.NewMessage(event.UpdateName, updateEvent)) }) return nil diff --git a/internal/db/db.go b/internal/db/db.go index 861d2a1fb2..bb4308c1d9 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -126,7 +126,7 @@ func newDB( return nil, err } - sub, err := db.events.Subscribe(event.MergeEventName) + sub, err := db.events.Subscribe(event.MergeName) if err != nil { return nil, err } diff --git a/internal/db/merge.go b/internal/db/merge.go index 39d822a946..fcc311eadc 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -64,7 +64,7 @@ func (db *db) handleMerges(ctx context.Context, sub *event.Subscription) { func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { // send a complete event so we can track merges in the integration tests - defer db.events.Publish(event.NewMessage(event.MergeCompleteEventName, dagMerge)) + defer db.events.Publish(event.NewMessage(event.MergeCompleteName, dagMerge)) ctx, txn, err := ensureContextTxn(ctx, db, false) if err != nil { diff --git a/internal/db/subscriptions.go b/internal/db/subscriptions.go index 1798e386a1..b469ac9c6e 100644 --- a/internal/db/subscriptions.go +++ b/internal/db/subscriptions.go @@ -32,7 +32,7 @@ func (db *db) handleSubscription(ctx context.Context, r *request.Request) (<-cha return nil, client.NewErrUnexpectedType[request.ObjectSubscription]("SubscriptionSelection", selections) } // subscribe to the subscription event bus so we don't block the system bus - sub, err := db.events.Subscribe(event.UpdateEventName) + sub, err := db.events.Subscribe(event.UpdateName) if err != nil { return nil, err } diff --git a/net/node.go b/net/node.go index 6f08dec00b..70914d472d 100644 --- a/net/node.go +++ b/net/node.go @@ -188,7 +188,7 @@ func NewNode( // publish subscribed events to the event bus go func() { for val := range sub.Out() { - db.Events().Publish(event1.NewMessage(event1.PeerEventName, val)) + db.Events().Publish(event1.NewMessage(event1.PeerName, val)) } }() diff --git a/net/peer.go b/net/peer.go index 32255afa43..fa1fb02263 100644 --- a/net/peer.go +++ b/net/peer.go @@ -144,7 +144,7 @@ func (p *Peer) Start() error { } if p.ps != nil { - sub, err := p.db.Events().Subscribe(event.UpdateEventName) + sub, err := p.db.Events().Subscribe(event.UpdateName) if err != nil { return err } diff --git a/net/server.go b/net/server.go index a1787098f3..511e0c1543 100644 --- a/net/server.go +++ b/net/server.go @@ -206,7 +206,7 @@ func (s *server) PushLog(ctx context.Context, req *pb.PushLogRequest) (*pb.PushL } if exists { // the integration tests expect every push log to emit a merge complete event - s.peer.db.Events().Publish(event.NewMessage(event.MergeCompleteEventName, evt)) + s.peer.db.Events().Publish(event.NewMessage(event.MergeCompleteName, evt)) return &pb.PushLogReply{}, nil } @@ -226,7 +226,7 @@ func (s *server) PushLog(ctx context.Context, req *pb.PushLogRequest) (*pb.PushL corelog.Any("CID", cid), ) } - s.peer.db.Events().Publish(event.NewMessage(event.MergeEventName, evt)) + s.peer.db.Events().Publish(event.NewMessage(event.MergeName, evt)) // Once processed, subscribe to the DocID topic on the pubsub network unless we already // suscribe to the collection. @@ -373,7 +373,7 @@ func (s *server) pubSubEventHandler(from libpeer.ID, topic string, msg []byte) { corelog.String("Topic", topic), corelog.String("Message", string(msg)), ) - evt := event.NewMessage(event.PubSubEventName, event.PubSub{ + evt := event.NewMessage(event.PubSubName, event.PubSub{ Peer: from, }) s.peer.db.Events().Publish(evt) diff --git a/tests/integration/events/utils.go b/tests/integration/events/utils.go index 74ed0a7da4..4988f25a66 100644 --- a/tests/integration/events/utils.go +++ b/tests/integration/events/utils.go @@ -81,7 +81,7 @@ func ExecuteRequestTestCase( testRoutineClosedChan := make(chan struct{}) closeTestRoutineChan := make(chan struct{}) - eventsSub, err := db.Events().Subscribe(event.UpdateEventName) + eventsSub, err := db.Events().Subscribe(event.UpdateName) require.NoError(t, err) indexOfNextExpectedUpdate := 0 diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 098cc3fda5..559d9d7816 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -659,7 +659,7 @@ func setStartingNodes( s.dbPaths = append(s.dbPaths, path) // subscribe to merge complete events - sub, err := c.Events().Subscribe(event.MergeCompleteEventName) + sub, err := c.Events().Subscribe(event.MergeCompleteName) require.NoError(s.t, err) s.eventSubs = append(s.eventSubs, sub) } @@ -715,7 +715,7 @@ func restartNodes( s.nodes[i] = c // subscribe to merge complete events - sub, err := c.Events().Subscribe(event.MergeCompleteEventName) + sub, err := c.Events().Subscribe(event.MergeCompleteName) require.NoError(s.t, err) s.eventSubs[i] = sub } @@ -821,7 +821,7 @@ func configureNode( s.dbPaths = append(s.dbPaths, path) // subscribe to merge complete events - sub, err := c.Events().Subscribe(event.MergeCompleteEventName) + sub, err := c.Events().Subscribe(event.MergeCompleteName) require.NoError(s.t, err) s.eventSubs = append(s.eventSubs, sub) } From 23d4d95fd19d25c0de6275fbd821133c2eac2d74 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 12 Jun 2024 15:04:34 -0700 Subject: [PATCH 18/34] fix event bus isClosed logic --- event/buffered_bus.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/event/buffered_bus.go b/event/buffered_bus.go index 45a0c4a2e4..fde2329336 100644 --- a/event/buffered_bus.go +++ b/event/buffered_bus.go @@ -33,7 +33,7 @@ type bufferedBus struct { subs map[uint64]*Subscription // events is a mapping of event names to subscriber ids. events map[Name]map[uint64]struct{} - // commandChannel manages all commands sent to this simpleChannel. + // commandChannel manages all commands sent to the bufferedBus. // // It is important that all stuff gets sent through this single channel to ensure // that the order of operations is preserved. @@ -90,10 +90,9 @@ func (b *bufferedBus) Unsubscribe(sub *Subscription) { } func (b *bufferedBus) Close() { - if b.isClosed.Load() { + if !b.isClosed.CompareAndSwap(false, true) { return } - b.isClosed.Store(true) b.commandChannel <- closeCommand{} // Wait for the close command to be handled, in order, before returning <-b.hasClosedChan From 64ebf8292e435d8ea4a64e8ab956313614b5bdc4 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 12 Jun 2024 15:12:22 -0700 Subject: [PATCH 19/34] remove WARNING text from bufferedBus docs --- event/buffered_bus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event/buffered_bus.go b/event/buffered_bus.go index fde2329336..fcd48e5b40 100644 --- a/event/buffered_bus.go +++ b/event/buffered_bus.go @@ -38,7 +38,7 @@ type bufferedBus struct { // It is important that all stuff gets sent through this single channel to ensure // that the order of operations is preserved. // - // WARNING: This does mean that non-event commands can block the database if the buffer + // This does mean that non-event commands can block the database if the buffer // size is breached (e.g. if many subscribe commands occupy the buffer). commandChannel chan any eventBufferSize int From 42c52d08fc394309d7ee1178c5882343a37a15d7 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 12 Jun 2024 15:18:15 -0700 Subject: [PATCH 20/34] update mocks --- client/mocks/db.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/client/mocks/db.go b/client/mocks/db.go index c56af31167..2cba0da3ed 100644 --- a/client/mocks/db.go +++ b/client/mocks/db.go @@ -360,15 +360,15 @@ func (_c *DB_Close_Call) RunAndReturn(run func()) *DB_Close_Call { } // Events provides a mock function with given fields: -func (_m *DB) Events() *event.Bus { +func (_m *DB) Events() event.Bus { ret := _m.Called() - var r0 *event.Bus - if rf, ok := ret.Get(0).(func() *event.Bus); ok { + var r0 event.Bus + if rf, ok := ret.Get(0).(func() event.Bus); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*event.Bus) + r0 = ret.Get(0).(event.Bus) } } @@ -392,12 +392,12 @@ func (_c *DB_Events_Call) Run(run func()) *DB_Events_Call { return _c } -func (_c *DB_Events_Call) Return(_a0 *event.Bus) *DB_Events_Call { +func (_c *DB_Events_Call) Return(_a0 event.Bus) *DB_Events_Call { _c.Call.Return(_a0) return _c } -func (_c *DB_Events_Call) RunAndReturn(run func() *event.Bus) *DB_Events_Call { +func (_c *DB_Events_Call) RunAndReturn(run func() event.Bus) *DB_Events_Call { _c.Call.Return(run) return _c } From 7ac87b987001df613fd6513c041822929c3c2d94 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 12 Jun 2024 15:51:34 -0700 Subject: [PATCH 21/34] fix waitForMerge node address --- net/node.go | 8 ++++---- tests/integration/p2p.go | 7 ++----- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/net/node.go b/net/node.go index 70914d472d..a2d27bde6b 100644 --- a/net/node.go +++ b/net/node.go @@ -31,7 +31,7 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" record "github.com/libp2p/go-libp2p-record" libp2pCrypto "github.com/libp2p/go-libp2p/core/crypto" - "github.com/libp2p/go-libp2p/core/event" + libp2pEvent "github.com/libp2p/go-libp2p/core/event" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" @@ -47,7 +47,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/crypto" - event1 "github.com/sourcenetwork/defradb/event" + "github.com/sourcenetwork/defradb/event" ) var _ client.P2P = (*Node)(nil) @@ -181,14 +181,14 @@ func NewNode( return nil, fin.Cleanup(err) } - sub, err := h.EventBus().Subscribe(&event.EvtPeerConnectednessChanged{}) + sub, err := h.EventBus().Subscribe(&libp2pEvent.EvtPeerConnectednessChanged{}) if err != nil { return nil, fin.Cleanup(err) } // publish subscribed events to the event bus go func() { for val := range sub.Out() { - db.Events().Publish(event1.NewMessage(event1.PeerName, val)) + db.Events().Publish(event.NewMessage(event.PeerName, val)) } }() diff --git a/tests/integration/p2p.go b/tests/integration/p2p.go index cd1dbd3b5c..d990a3d322 100644 --- a/tests/integration/p2p.go +++ b/tests/integration/p2p.go @@ -501,14 +501,11 @@ func waitForMerge( targetToSourceEvents []int, nodeSynced chan struct{}, ) { - sourceNode := s.nodes[sourceNodeID] - targetNode := s.nodes[targetNodeID] - sourceSub := s.eventSubs[sourceNodeID] targetSub := s.eventSubs[targetNodeID] - sourcePeerInfo := sourceNode.PeerInfo() - targetPeerInfo := targetNode.PeerInfo() + sourcePeerInfo := s.nodeAddresses[sourceNodeID] + targetPeerInfo := s.nodeAddresses[targetNodeID] for waitIndex := 0; waitIndex < len(sourceToTargetEvents); waitIndex++ { for i := 0; i < targetToSourceEvents[waitIndex]; i++ { From 311a37d358d5556f73b5758248e7c15c289e468b Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 12 Jun 2024 17:05:11 -0700 Subject: [PATCH 22/34] revert p2p waitForMerge change. add more info to cli peer info panic --- tests/clients/cli/wrapper.go | 4 ++-- tests/integration/p2p.go | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/clients/cli/wrapper.go b/tests/clients/cli/wrapper.go index 1681dd6e7b..501557b391 100644 --- a/tests/clients/cli/wrapper.go +++ b/tests/clients/cli/wrapper.go @@ -65,11 +65,11 @@ func (w *Wrapper) PeerInfo() peer.AddrInfo { data, err := w.cmd.execute(context.Background(), args) if err != nil { - panic(fmt.Sprintf("failed to get peer info: %v", err)) + panic(fmt.Sprintf("failed to get peer info: %v %s", err, data)) } var info peer.AddrInfo if err := json.Unmarshal(data, &info); err != nil { - panic(fmt.Sprintf("failed to get peer info: %v", err)) + panic(fmt.Sprintf("failed to get peer info: %v %s", err, data)) } return info } diff --git a/tests/integration/p2p.go b/tests/integration/p2p.go index d990a3d322..cd1dbd3b5c 100644 --- a/tests/integration/p2p.go +++ b/tests/integration/p2p.go @@ -501,11 +501,14 @@ func waitForMerge( targetToSourceEvents []int, nodeSynced chan struct{}, ) { + sourceNode := s.nodes[sourceNodeID] + targetNode := s.nodes[targetNodeID] + sourceSub := s.eventSubs[sourceNodeID] targetSub := s.eventSubs[targetNodeID] - sourcePeerInfo := s.nodeAddresses[sourceNodeID] - targetPeerInfo := s.nodeAddresses[targetNodeID] + sourcePeerInfo := sourceNode.PeerInfo() + targetPeerInfo := targetNode.PeerInfo() for waitIndex := 0; waitIndex < len(sourceToTargetEvents); waitIndex++ { for i := 0; i < targetToSourceEvents[waitIndex]; i++ { From cfa0ef9aa4f6105853d5f68c41d872ac405c2dbf Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 12 Jun 2024 18:31:09 -0700 Subject: [PATCH 23/34] fix node constructor --- net/node.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/net/node.go b/net/node.go index a2d27bde6b..e1557374cb 100644 --- a/net/node.go +++ b/net/node.go @@ -192,13 +192,15 @@ func NewNode( } }() - return &Node{ + node = &Node{ Peer: peer, DB: db, ctx: ctx, cancel: cancel, dhtClose: ddht.Close, - }, nil + } + + return } // Bootstrap connects to the given peers. From 03573853318c583adea54f485af0a8eb59926d1e Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 12 Jun 2024 19:30:13 -0700 Subject: [PATCH 24/34] skip waitForMerge when no waitForSync present --- tests/integration/p2p.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tests/integration/p2p.go b/tests/integration/p2p.go index cd1dbd3b5c..7a2d22a9ef 100644 --- a/tests/integration/p2p.go +++ b/tests/integration/p2p.go @@ -175,6 +175,7 @@ func setupPeerWaitSync( nodeCollections := map[int][]int{} waitIndex := 0 + skipWaitForMerge := true for i := startIndex; i < len(s.testCase.Actions); i++ { switch action := s.testCase.Actions[i].(type) { case SubscribeToCollection: @@ -247,14 +248,18 @@ func setupPeerWaitSync( case WaitForSync: waitIndex += 1 + skipWaitForMerge = false targetToSourceEvents = append(targetToSourceEvents, 0) sourceToTargetEvents = append(sourceToTargetEvents, 0) } } - nodeSynced := make(chan struct{}) - go waitForMerge(s, cfg.SourceNodeID, cfg.TargetNodeID, sourceToTargetEvents, targetToSourceEvents, nodeSynced) - s.syncChans = append(s.syncChans, nodeSynced) + // skip waiting for a merge if we aren't interested in waiting for a sync to complete + if !skipWaitForMerge { + nodeSynced := make(chan struct{}) + go waitForMerge(s, cfg.SourceNodeID, cfg.TargetNodeID, sourceToTargetEvents, targetToSourceEvents, nodeSynced) + s.syncChans = append(s.syncChans, nodeSynced) + } } // collectionSubscribedTo returns true if the collection on the given node @@ -323,6 +328,7 @@ func setupReplicatorWaitSync( docIDsSyncedToSource := map[int]struct{}{} waitIndex := 0 currentDocID := 0 + skipWaitForMerge := true for i := startIndex; i < len(s.testCase.Actions); i++ { switch action := s.testCase.Actions[i].(type) { case CreateDoc: @@ -360,14 +366,18 @@ func setupReplicatorWaitSync( case WaitForSync: waitIndex += 1 + skipWaitForMerge = false targetToSourceEvents = append(targetToSourceEvents, 0) sourceToTargetEvents = append(sourceToTargetEvents, 0) } } - nodeSynced := make(chan struct{}) - go waitForMerge(s, cfg.SourceNodeID, cfg.TargetNodeID, sourceToTargetEvents, targetToSourceEvents, nodeSynced) - s.syncChans = append(s.syncChans, nodeSynced) + // skip waiting for a merge if we aren't interested in waiting for a sync to complete + if !skipWaitForMerge { + nodeSynced := make(chan struct{}) + go waitForMerge(s, cfg.SourceNodeID, cfg.TargetNodeID, sourceToTargetEvents, targetToSourceEvents, nodeSynced) + s.syncChans = append(s.syncChans, nodeSynced) + } } // subscribeToCollection sets up a collection subscription on the given node/collection. From 35133905c75f1d68ecfe0cb873e5468a4b94d801 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 12 Jun 2024 20:10:42 -0700 Subject: [PATCH 25/34] revert skipWaitForMerge. use peer info from state --- tests/clients/cli/wrapper.go | 4 ++-- tests/integration/p2p.go | 29 ++++++++--------------------- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/tests/clients/cli/wrapper.go b/tests/clients/cli/wrapper.go index 501557b391..1681dd6e7b 100644 --- a/tests/clients/cli/wrapper.go +++ b/tests/clients/cli/wrapper.go @@ -65,11 +65,11 @@ func (w *Wrapper) PeerInfo() peer.AddrInfo { data, err := w.cmd.execute(context.Background(), args) if err != nil { - panic(fmt.Sprintf("failed to get peer info: %v %s", err, data)) + panic(fmt.Sprintf("failed to get peer info: %v", err)) } var info peer.AddrInfo if err := json.Unmarshal(data, &info); err != nil { - panic(fmt.Sprintf("failed to get peer info: %v %s", err, data)) + panic(fmt.Sprintf("failed to get peer info: %v", err)) } return info } diff --git a/tests/integration/p2p.go b/tests/integration/p2p.go index 7a2d22a9ef..d990a3d322 100644 --- a/tests/integration/p2p.go +++ b/tests/integration/p2p.go @@ -175,7 +175,6 @@ func setupPeerWaitSync( nodeCollections := map[int][]int{} waitIndex := 0 - skipWaitForMerge := true for i := startIndex; i < len(s.testCase.Actions); i++ { switch action := s.testCase.Actions[i].(type) { case SubscribeToCollection: @@ -248,18 +247,14 @@ func setupPeerWaitSync( case WaitForSync: waitIndex += 1 - skipWaitForMerge = false targetToSourceEvents = append(targetToSourceEvents, 0) sourceToTargetEvents = append(sourceToTargetEvents, 0) } } - // skip waiting for a merge if we aren't interested in waiting for a sync to complete - if !skipWaitForMerge { - nodeSynced := make(chan struct{}) - go waitForMerge(s, cfg.SourceNodeID, cfg.TargetNodeID, sourceToTargetEvents, targetToSourceEvents, nodeSynced) - s.syncChans = append(s.syncChans, nodeSynced) - } + nodeSynced := make(chan struct{}) + go waitForMerge(s, cfg.SourceNodeID, cfg.TargetNodeID, sourceToTargetEvents, targetToSourceEvents, nodeSynced) + s.syncChans = append(s.syncChans, nodeSynced) } // collectionSubscribedTo returns true if the collection on the given node @@ -328,7 +323,6 @@ func setupReplicatorWaitSync( docIDsSyncedToSource := map[int]struct{}{} waitIndex := 0 currentDocID := 0 - skipWaitForMerge := true for i := startIndex; i < len(s.testCase.Actions); i++ { switch action := s.testCase.Actions[i].(type) { case CreateDoc: @@ -366,18 +360,14 @@ func setupReplicatorWaitSync( case WaitForSync: waitIndex += 1 - skipWaitForMerge = false targetToSourceEvents = append(targetToSourceEvents, 0) sourceToTargetEvents = append(sourceToTargetEvents, 0) } } - // skip waiting for a merge if we aren't interested in waiting for a sync to complete - if !skipWaitForMerge { - nodeSynced := make(chan struct{}) - go waitForMerge(s, cfg.SourceNodeID, cfg.TargetNodeID, sourceToTargetEvents, targetToSourceEvents, nodeSynced) - s.syncChans = append(s.syncChans, nodeSynced) - } + nodeSynced := make(chan struct{}) + go waitForMerge(s, cfg.SourceNodeID, cfg.TargetNodeID, sourceToTargetEvents, targetToSourceEvents, nodeSynced) + s.syncChans = append(s.syncChans, nodeSynced) } // subscribeToCollection sets up a collection subscription on the given node/collection. @@ -511,14 +501,11 @@ func waitForMerge( targetToSourceEvents []int, nodeSynced chan struct{}, ) { - sourceNode := s.nodes[sourceNodeID] - targetNode := s.nodes[targetNodeID] - sourceSub := s.eventSubs[sourceNodeID] targetSub := s.eventSubs[targetNodeID] - sourcePeerInfo := sourceNode.PeerInfo() - targetPeerInfo := targetNode.PeerInfo() + sourcePeerInfo := s.nodeAddresses[sourceNodeID] + targetPeerInfo := s.nodeAddresses[targetNodeID] for waitIndex := 0; waitIndex < len(sourceToTargetEvents); waitIndex++ { for i := 0; i < targetToSourceEvents[waitIndex]; i++ { From 184f0cc1e33b4c7c50b85a60b918a98f4365de47 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Thu, 13 Jun 2024 09:56:37 -0700 Subject: [PATCH 26/34] test db merge sequential --- internal/db/merge.go | 22 ++++++++++------------ internal/db/subscriptions.go | 1 - tests/integration/utils2.go | 5 ----- 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/internal/db/merge.go b/internal/db/merge.go index fcc311eadc..6cb264e6b3 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -46,18 +46,16 @@ func (db *db) handleMerges(ctx context.Context, sub *event.Subscription) { if !ok { continue } - go func() { - err := db.executeMerge(ctx, merge) - if err != nil { - log.ErrorContextE( - ctx, - "Failed to execute merge", - err, - corelog.String("CID", merge.Cid.String()), - corelog.String("Error", err.Error()), - ) - } - }() + err := db.executeMerge(ctx, merge) + if err != nil { + log.ErrorContextE( + ctx, + "Failed to execute merge", + err, + corelog.String("CID", merge.Cid.String()), + corelog.String("Error", err.Error()), + ) + } } } } diff --git a/internal/db/subscriptions.go b/internal/db/subscriptions.go index b469ac9c6e..491ae44468 100644 --- a/internal/db/subscriptions.go +++ b/internal/db/subscriptions.go @@ -31,7 +31,6 @@ func (db *db) handleSubscription(ctx context.Context, r *request.Request) (<-cha if !ok { return nil, client.NewErrUnexpectedType[request.ObjectSubscription]("SubscriptionSelection", selections) } - // subscribe to the subscription event bus so we don't block the system bus sub, err := db.events.Subscribe(event.UpdateName) if err != nil { return nil, err diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 559d9d7816..80bc4920d8 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -657,11 +657,6 @@ func setStartingNodes( s.nodes = append(s.nodes, c) s.dbPaths = append(s.dbPaths, path) - - // subscribe to merge complete events - sub, err := c.Events().Subscribe(event.MergeCompleteName) - require.NoError(s.t, err) - s.eventSubs = append(s.eventSubs, sub) } } From 55aac76d5f209df8c6a9b048a0ebba1ad0f6aa2a Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 17 Jun 2024 11:00:11 -0700 Subject: [PATCH 27/34] add mutex to buffered bus --- event/buffered_bus.go | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/event/buffered_bus.go b/event/buffered_bus.go index fcd48e5b40..e92914fa7a 100644 --- a/event/buffered_bus.go +++ b/event/buffered_bus.go @@ -11,6 +11,7 @@ package event import ( + "sync" "sync/atomic" ) @@ -43,7 +44,8 @@ type bufferedBus struct { commandChannel chan any eventBufferSize int hasClosedChan chan struct{} - isClosed atomic.Bool + isClosed bool + mutex sync.RWMutex } // NewBufferedBus creates a new event bus with the given commandBufferSize and @@ -63,14 +65,20 @@ func NewBufferedBus(commandBufferSize int, eventBufferSize int) *bufferedBus { } func (b *bufferedBus) Publish(msg Message) { - if b.isClosed.Load() { + b.mutex.RLock() + defer b.mutex.RUnlock() + + if b.isClosed { return } b.commandChannel <- publishCommand(msg) } func (b *bufferedBus) Subscribe(events ...Name) (*Subscription, error) { - if b.isClosed.Load() { + b.mutex.RLock() + defer b.mutex.RUnlock() + + if b.isClosed { return nil, ErrSubscribedToClosedChan } sub := &Subscription{ @@ -83,16 +91,23 @@ func (b *bufferedBus) Subscribe(events ...Name) (*Subscription, error) { } func (b *bufferedBus) Unsubscribe(sub *Subscription) { - if b.isClosed.Load() { + b.mutex.RLock() + defer b.mutex.RUnlock() + + if b.isClosed { return } b.commandChannel <- unsubscribeCommand(sub) } func (b *bufferedBus) Close() { - if !b.isClosed.CompareAndSwap(false, true) { + b.mutex.Lock() + defer b.mutex.Unlock() + + if b.isClosed { return } + b.isClosed = true b.commandChannel <- closeCommand{} // Wait for the close command to be handled, in order, before returning <-b.hasClosedChan From e6128ef8477a1149b7a155ce3e3aebb43614b624 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 17 Jun 2024 12:14:33 -0700 Subject: [PATCH 28/34] document bus mutex --- event/buffered_bus.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/event/buffered_bus.go b/event/buffered_bus.go index e92914fa7a..5ab7f0aafa 100644 --- a/event/buffered_bus.go +++ b/event/buffered_bus.go @@ -45,7 +45,8 @@ type bufferedBus struct { eventBufferSize int hasClosedChan chan struct{} isClosed bool - mutex sync.RWMutex + // closeMutex is only locked when the bus is closing. + closeMutex sync.RWMutex } // NewBufferedBus creates a new event bus with the given commandBufferSize and @@ -65,8 +66,8 @@ func NewBufferedBus(commandBufferSize int, eventBufferSize int) *bufferedBus { } func (b *bufferedBus) Publish(msg Message) { - b.mutex.RLock() - defer b.mutex.RUnlock() + b.closeMutex.RLock() + defer b.closeMutex.RUnlock() if b.isClosed { return @@ -75,8 +76,8 @@ func (b *bufferedBus) Publish(msg Message) { } func (b *bufferedBus) Subscribe(events ...Name) (*Subscription, error) { - b.mutex.RLock() - defer b.mutex.RUnlock() + b.closeMutex.RLock() + defer b.closeMutex.RUnlock() if b.isClosed { return nil, ErrSubscribedToClosedChan @@ -91,8 +92,8 @@ func (b *bufferedBus) Subscribe(events ...Name) (*Subscription, error) { } func (b *bufferedBus) Unsubscribe(sub *Subscription) { - b.mutex.RLock() - defer b.mutex.RUnlock() + b.closeMutex.RLock() + defer b.closeMutex.RUnlock() if b.isClosed { return @@ -101,8 +102,8 @@ func (b *bufferedBus) Unsubscribe(sub *Subscription) { } func (b *bufferedBus) Close() { - b.mutex.Lock() - defer b.mutex.Unlock() + b.closeMutex.Lock() + defer b.closeMutex.Unlock() if b.isClosed { return From 801354cb33afe08b6d2de6ab92a146c3262d8252 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 17 Jun 2024 12:22:17 -0700 Subject: [PATCH 29/34] add buffered bus tests --- event/buffered_bus_test.go | 42 +++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/event/buffered_bus_test.go b/event/buffered_bus_test.go index 3b2a0fd1ed..76c21cefda 100644 --- a/event/buffered_bus_test.go +++ b/event/buffered_bus_test.go @@ -18,7 +18,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestSimplePushIsNotBlockedWithoutSubscribers(t *testing.T) { +func TestBufferedBusPushIsNotBlockedWithoutSubscribers(t *testing.T) { bus := NewBufferedBus(0, 0) defer bus.Close() @@ -29,7 +29,7 @@ func TestSimplePushIsNotBlockedWithoutSubscribers(t *testing.T) { assert.True(t, true) } -func TestSimpleSubscribersAreNotBlockedAfterClose(t *testing.T) { +func TestBufferedBusSubscribersAreNotBlockedAfterClose(t *testing.T) { bus := NewBufferedBus(0, 0) defer bus.Close() @@ -44,7 +44,39 @@ func TestSimpleSubscribersAreNotBlockedAfterClose(t *testing.T) { assert.True(t, true) } -func TestSimpleEachSubscribersRecievesEachItem(t *testing.T) { +func TestBufferedBusUnsubscribeTwice(t *testing.T) { + bus := NewBufferedBus(0, 0) + defer bus.Close() + + sub, err := bus.Subscribe(WildCardName) + assert.NoError(t, err) + + bus.Unsubscribe(sub) + bus.Unsubscribe(sub) +} + +func TestBufferedBusWildCardDeduplicates(t *testing.T) { + bus := NewBufferedBus(0, 0) + defer bus.Close() + + sub, err := bus.Subscribe("test", WildCardName) + assert.NoError(t, err) + + msg := NewMessage("test", 1) + bus.Publish(msg) + + evt := <-sub.Message() + assert.Equal(t, evt, msg) + + select { + case <-sub.Message(): + t.Errorf("should not receive duplicate message") + case <-time.After(1 * time.Second): + // message is deduplicated + } +} + +func TestBufferedBusEachSubscribersRecievesEachItem(t *testing.T) { bus := NewBufferedBus(0, 0) defer bus.Close() @@ -98,7 +130,7 @@ func TestSimpleEachSubscribersRecievesEachItem(t *testing.T) { assert.Equal(t, msg2, event2) } -func TestSimpleEachSubscribersRecievesEachItemGivenBufferedEventChan(t *testing.T) { +func TestBufferedBusEachSubscribersRecievesEachItemGivenBufferedEventChan(t *testing.T) { bus := NewBufferedBus(0, 2) defer bus.Close() @@ -127,7 +159,7 @@ func TestSimpleEachSubscribersRecievesEachItemGivenBufferedEventChan(t *testing. assert.Equal(t, msg2, output2Ch2) } -func TestSimpleSubscribersDontRecieveItemsAfterUnsubscribing(t *testing.T) { +func TestBufferedBusSubscribersDontRecieveItemsAfterUnsubscribing(t *testing.T) { bus := NewBufferedBus(0, 0) defer bus.Close() From 2dfa4d273e36212dfe57010cb8b8724afe00aa83 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 17 Jun 2024 14:39:44 -0700 Subject: [PATCH 30/34] fix buffered bus test names --- event/buffered_bus_test.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/event/buffered_bus_test.go b/event/buffered_bus_test.go index 76c21cefda..e976186485 100644 --- a/event/buffered_bus_test.go +++ b/event/buffered_bus_test.go @@ -18,7 +18,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestBufferedBusPushIsNotBlockedWithoutSubscribers(t *testing.T) { +func TestBufferedBus_PushIsNotBlockedWithoutSubscribers_Succeed(t *testing.T) { bus := NewBufferedBus(0, 0) defer bus.Close() @@ -29,12 +29,12 @@ func TestBufferedBusPushIsNotBlockedWithoutSubscribers(t *testing.T) { assert.True(t, true) } -func TestBufferedBusSubscribersAreNotBlockedAfterClose(t *testing.T) { +func TestBufferedBus_SubscribersAreNotBlockedAfterClose_Succeed(t *testing.T) { bus := NewBufferedBus(0, 0) defer bus.Close() sub, err := bus.Subscribe("test") - assert.Nil(t, err) + assert.NoError(t, err) bus.Close() @@ -44,7 +44,7 @@ func TestBufferedBusSubscribersAreNotBlockedAfterClose(t *testing.T) { assert.True(t, true) } -func TestBufferedBusUnsubscribeTwice(t *testing.T) { +func TestBufferedBus_UnsubscribeTwice_Succeed(t *testing.T) { bus := NewBufferedBus(0, 0) defer bus.Close() @@ -55,7 +55,7 @@ func TestBufferedBusUnsubscribeTwice(t *testing.T) { bus.Unsubscribe(sub) } -func TestBufferedBusWildCardDeduplicates(t *testing.T) { +func TestBufferedBus_WildCardDeduplicates_Succeed(t *testing.T) { bus := NewBufferedBus(0, 0) defer bus.Close() @@ -71,12 +71,12 @@ func TestBufferedBusWildCardDeduplicates(t *testing.T) { select { case <-sub.Message(): t.Errorf("should not receive duplicate message") - case <-time.After(1 * time.Second): + case <-time.After(100 * time.Millisecond): // message is deduplicated } } -func TestBufferedBusEachSubscribersRecievesEachItem(t *testing.T) { +func TestBufferedBus_EachSubscribersRecievesEachItem_Succeed(t *testing.T) { bus := NewBufferedBus(0, 0) defer bus.Close() @@ -84,10 +84,10 @@ func TestBufferedBusEachSubscribersRecievesEachItem(t *testing.T) { msg2 := NewMessage("test", 2) sub1, err := bus.Subscribe("test") - assert.Nil(t, err) + assert.NoError(t, err) sub2, err := bus.Subscribe("test") - assert.Nil(t, err) + assert.NoError(t, err) // ordering of publish is not deterministic // so capture each in a go routine @@ -130,7 +130,7 @@ func TestBufferedBusEachSubscribersRecievesEachItem(t *testing.T) { assert.Equal(t, msg2, event2) } -func TestBufferedBusEachSubscribersRecievesEachItemGivenBufferedEventChan(t *testing.T) { +func TestBufferedBus_EachSubscribersRecievesEachItemGivenBufferedEventChan_Succeed(t *testing.T) { bus := NewBufferedBus(0, 2) defer bus.Close() @@ -138,9 +138,9 @@ func TestBufferedBusEachSubscribersRecievesEachItemGivenBufferedEventChan(t *tes msg2 := NewMessage("test", 2) sub1, err := bus.Subscribe("test") - assert.Nil(t, err) + assert.NoError(t, err) sub2, err := bus.Subscribe("test") - assert.Nil(t, err) + assert.NoError(t, err) // both inputs are added first before read, using the internal chan buffer bus.Publish(msg1) @@ -159,12 +159,12 @@ func TestBufferedBusEachSubscribersRecievesEachItemGivenBufferedEventChan(t *tes assert.Equal(t, msg2, output2Ch2) } -func TestBufferedBusSubscribersDontRecieveItemsAfterUnsubscribing(t *testing.T) { +func TestBufferedBus_SubscribersDontRecieveItemsAfterUnsubscribing_Succeed(t *testing.T) { bus := NewBufferedBus(0, 0) defer bus.Close() sub, err := bus.Subscribe("test") - assert.Nil(t, err) + assert.NoError(t, err) bus.Unsubscribe(sub) msg := NewMessage("test", 1) From 56136c5151c9f5c63d519d9ab1eb08a870b1b16b Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 17 Jun 2024 14:52:05 -0700 Subject: [PATCH 31/34] remove event.Bus interface --- client/db.go | 2 +- event/buffered_bus.go | 159 ------------------ event/bus.go | 171 ++++++++++++++++---- event/{buffered_bus_test.go => bus_test.go} | 28 ++-- event/event.go | 26 +++ http/client.go | 2 +- internal/db/db.go | 6 +- tests/clients/cli/wrapper.go | 2 +- tests/clients/http/wrapper.go | 2 +- 9 files changed, 183 insertions(+), 215 deletions(-) delete mode 100644 event/buffered_bus.go rename event/{buffered_bus_test.go => bus_test.go} (80%) diff --git a/client/db.go b/client/db.go index 4ec9643fff..e52dfed60a 100644 --- a/client/db.go +++ b/client/db.go @@ -75,7 +75,7 @@ type DB interface { // // It may be used to monitor database events - a new event will be yielded for each mutation. // Note: it does not copy the queue, just the reference to it. - Events() event.Bus + Events() *event.Bus // MaxTxnRetries returns the number of retries that this DefraDB instance has been configured to // make in the event of a transaction conflict in certain scenarios. diff --git a/event/buffered_bus.go b/event/buffered_bus.go deleted file mode 100644 index 5ab7f0aafa..0000000000 --- a/event/buffered_bus.go +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2024 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package event - -import ( - "sync" - "sync/atomic" -) - -type subscribeCommand *Subscription - -type unsubscribeCommand *Subscription - -type publishCommand Message - -type closeCommand struct{} - -// bufferedBus must implement the Bus interface -var _ (Bus) = (*bufferedBus)(nil) - -// bufferedBus is a bus that uses a buffered channel to manage subscribers and publish messages. -type bufferedBus struct { - // subID is incremented for each subscriber and used to set subscriber ids. - subID atomic.Uint64 - // subs is a mapping of subscriber ids to subscriptions. - subs map[uint64]*Subscription - // events is a mapping of event names to subscriber ids. - events map[Name]map[uint64]struct{} - // commandChannel manages all commands sent to the bufferedBus. - // - // It is important that all stuff gets sent through this single channel to ensure - // that the order of operations is preserved. - // - // This does mean that non-event commands can block the database if the buffer - // size is breached (e.g. if many subscribe commands occupy the buffer). - commandChannel chan any - eventBufferSize int - hasClosedChan chan struct{} - isClosed bool - // closeMutex is only locked when the bus is closing. - closeMutex sync.RWMutex -} - -// NewBufferedBus creates a new event bus with the given commandBufferSize and -// eventBufferSize. -// -// Should the buffers be filled, subsequent calls on this bus will block. -func NewBufferedBus(commandBufferSize int, eventBufferSize int) *bufferedBus { - bus := bufferedBus{ - subs: make(map[uint64]*Subscription), - events: make(map[Name]map[uint64]struct{}), - commandChannel: make(chan any, commandBufferSize), - hasClosedChan: make(chan struct{}), - eventBufferSize: eventBufferSize, - } - go bus.handleChannel() - return &bus -} - -func (b *bufferedBus) Publish(msg Message) { - b.closeMutex.RLock() - defer b.closeMutex.RUnlock() - - if b.isClosed { - return - } - b.commandChannel <- publishCommand(msg) -} - -func (b *bufferedBus) Subscribe(events ...Name) (*Subscription, error) { - b.closeMutex.RLock() - defer b.closeMutex.RUnlock() - - if b.isClosed { - return nil, ErrSubscribedToClosedChan - } - sub := &Subscription{ - id: b.subID.Add(1), - value: make(chan Message, b.eventBufferSize), - events: events, - } - b.commandChannel <- subscribeCommand(sub) - return sub, nil -} - -func (b *bufferedBus) Unsubscribe(sub *Subscription) { - b.closeMutex.RLock() - defer b.closeMutex.RUnlock() - - if b.isClosed { - return - } - b.commandChannel <- unsubscribeCommand(sub) -} - -func (b *bufferedBus) Close() { - b.closeMutex.Lock() - defer b.closeMutex.Unlock() - - if b.isClosed { - return - } - b.isClosed = true - b.commandChannel <- closeCommand{} - // Wait for the close command to be handled, in order, before returning - <-b.hasClosedChan -} - -func (b *bufferedBus) handleChannel() { - for cmd := range b.commandChannel { - switch t := cmd.(type) { - case closeCommand: - for _, subscriber := range b.subs { - close(subscriber.value) - } - close(b.commandChannel) - close(b.hasClosedChan) - return - - case subscribeCommand: - for _, event := range t.events { - if _, ok := b.events[event]; !ok { - b.events[event] = make(map[uint64]struct{}) - } - b.events[event][t.id] = struct{}{} - } - b.subs[t.id] = t - - case unsubscribeCommand: - if _, ok := b.subs[t.id]; !ok { - continue // not subscribed - } - for _, event := range t.events { - delete(b.events[event], t.id) - } - delete(b.subs, t.id) - close(t.value) - - case publishCommand: - for id := range b.events[WildCardName] { - b.subs[id].value <- Message(t) - } - for id := range b.events[t.Name] { - if _, ok := b.events[WildCardName][id]; ok { - continue - } - b.subs[id].value <- Message(t) - } - } - } -} diff --git a/event/bus.go b/event/bus.go index e202cc2f8b..371173686d 100644 --- a/event/bus.go +++ b/event/bus.go @@ -10,53 +10,154 @@ package event -// Bus is an event bus used to broadcasts messages to subscribers. -type Bus interface { - // Subscribe subscribes to the Channel, returning a channel by which events can - // be read from, or an error should one occur (e.g. if this object is closed). - // - // This function is non-blocking unless the subscription-buffer is full. - Subscribe(events ...Name) (*Subscription, error) +import ( + "sync" + "sync/atomic" +) - // Unsubscribe unsubscribes from the Channel, closing the provided channel. - // - // Will do nothing if this object is already closed. - Unsubscribe(sub *Subscription) +type subscribeCommand *Subscription + +type unsubscribeCommand *Subscription + +type publishCommand Message - // Publish pushes the given item into this channel. Non-blocking. - Publish(msg Message) +type closeCommand struct{} + +// Bus uses a buffered channel to manage subscribers and publish messages. +type Bus struct { + // subID is incremented for each subscriber and used to set subscriber ids. + subID atomic.Uint64 + // subs is a mapping of subscriber ids to subscriptions. + subs map[uint64]*Subscription + // events is a mapping of event names to subscriber ids. + events map[Name]map[uint64]struct{} + // commandChannel manages all commands sent to the bufferedBus. + // + // It is important that all stuff gets sent through this single channel to ensure + // that the order of operations is preserved. + // + // This does mean that non-event commands can block the database if the buffer + // size is breached (e.g. if many subscribe commands occupy the buffer). + commandChannel chan any + eventBufferSize int + hasClosedChan chan struct{} + isClosed bool + // closeMutex is only locked when the bus is closing. + closeMutex sync.RWMutex +} - // Close closes this Channel, and any owned or subscribing channels. - Close() +// NewBus creates a new event bus with the given commandBufferSize and +// eventBufferSize. +// +// Should the buffers be filled, subsequent calls on this bus will block. +func NewBus(commandBufferSize int, eventBufferSize int) *Bus { + bus := Bus{ + subs: make(map[uint64]*Subscription), + events: make(map[Name]map[uint64]struct{}), + commandChannel: make(chan any, commandBufferSize), + hasClosedChan: make(chan struct{}), + eventBufferSize: eventBufferSize, + } + go bus.handleChannel() + return &bus } -// Message contains event info. -type Message struct { - // Name is the name of the event this message was generated from. - Name Name +// Publish broadcasts the given message to the bus subscribers. Non-blocking. +func (b *Bus) Publish(msg Message) { + b.closeMutex.RLock() + defer b.closeMutex.RUnlock() - // Data contains optional event information. - Data any + if b.isClosed { + return + } + b.commandChannel <- publishCommand(msg) } -// NewMessage returns a new message with the given name and optional data. -func NewMessage(name Name, data any) Message { - return Message{name, data} +// Subscribe returns a new subscription that will receive all of the events +// contained in the given list of events. +func (b *Bus) Subscribe(events ...Name) (*Subscription, error) { + b.closeMutex.RLock() + defer b.closeMutex.RUnlock() + + if b.isClosed { + return nil, ErrSubscribedToClosedChan + } + sub := &Subscription{ + id: b.subID.Add(1), + value: make(chan Message, b.eventBufferSize), + events: events, + } + b.commandChannel <- subscribeCommand(sub) + return sub, nil } -// Subscription is a read-only event stream. -type Subscription struct { - id uint64 - value chan Message - events []Name +// Unsubscribe removes all event subscriptions and closes the subscription channel. +// +// Will do nothing if this object is already closed. +func (b *Bus) Unsubscribe(sub *Subscription) { + b.closeMutex.RLock() + defer b.closeMutex.RUnlock() + + if b.isClosed { + return + } + b.commandChannel <- unsubscribeCommand(sub) } -// Message returns the next event value from the subscription. -func (s *Subscription) Message() <-chan Message { - return s.value +// Close unsubscribes all active subscribers and closes the command channel. +func (b *Bus) Close() { + b.closeMutex.Lock() + defer b.closeMutex.Unlock() + + if b.isClosed { + return + } + b.isClosed = true + b.commandChannel <- closeCommand{} + // Wait for the close command to be handled, in order, before returning + <-b.hasClosedChan } -// Events returns the names of all subscribed events. -func (s *Subscription) Events() []Name { - return s.events +func (b *Bus) handleChannel() { + for cmd := range b.commandChannel { + switch t := cmd.(type) { + case closeCommand: + for _, subscriber := range b.subs { + close(subscriber.value) + } + close(b.commandChannel) + close(b.hasClosedChan) + return + + case subscribeCommand: + for _, event := range t.events { + if _, ok := b.events[event]; !ok { + b.events[event] = make(map[uint64]struct{}) + } + b.events[event][t.id] = struct{}{} + } + b.subs[t.id] = t + + case unsubscribeCommand: + if _, ok := b.subs[t.id]; !ok { + continue // not subscribed + } + for _, event := range t.events { + delete(b.events[event], t.id) + } + delete(b.subs, t.id) + close(t.value) + + case publishCommand: + for id := range b.events[WildCardName] { + b.subs[id].value <- Message(t) + } + for id := range b.events[t.Name] { + if _, ok := b.events[WildCardName][id]; ok { + continue + } + b.subs[id].value <- Message(t) + } + } + } } diff --git a/event/buffered_bus_test.go b/event/bus_test.go similarity index 80% rename from event/buffered_bus_test.go rename to event/bus_test.go index e976186485..4d5352bc94 100644 --- a/event/buffered_bus_test.go +++ b/event/bus_test.go @@ -18,8 +18,8 @@ import ( "github.com/stretchr/testify/assert" ) -func TestBufferedBus_PushIsNotBlockedWithoutSubscribers_Succeed(t *testing.T) { - bus := NewBufferedBus(0, 0) +func TestBus_PushIsNotBlockedWithoutSubscribers_Succeed(t *testing.T) { + bus := NewBus(0, 0) defer bus.Close() msg := NewMessage("test", 1) @@ -29,8 +29,8 @@ func TestBufferedBus_PushIsNotBlockedWithoutSubscribers_Succeed(t *testing.T) { assert.True(t, true) } -func TestBufferedBus_SubscribersAreNotBlockedAfterClose_Succeed(t *testing.T) { - bus := NewBufferedBus(0, 0) +func TestBus_SubscribersAreNotBlockedAfterClose_Succeed(t *testing.T) { + bus := NewBus(0, 0) defer bus.Close() sub, err := bus.Subscribe("test") @@ -44,8 +44,8 @@ func TestBufferedBus_SubscribersAreNotBlockedAfterClose_Succeed(t *testing.T) { assert.True(t, true) } -func TestBufferedBus_UnsubscribeTwice_Succeed(t *testing.T) { - bus := NewBufferedBus(0, 0) +func TestBus_UnsubscribeTwice_Succeed(t *testing.T) { + bus := NewBus(0, 0) defer bus.Close() sub, err := bus.Subscribe(WildCardName) @@ -55,8 +55,8 @@ func TestBufferedBus_UnsubscribeTwice_Succeed(t *testing.T) { bus.Unsubscribe(sub) } -func TestBufferedBus_WildCardDeduplicates_Succeed(t *testing.T) { - bus := NewBufferedBus(0, 0) +func TestBus_WildCardDeduplicates_Succeed(t *testing.T) { + bus := NewBus(0, 0) defer bus.Close() sub, err := bus.Subscribe("test", WildCardName) @@ -76,8 +76,8 @@ func TestBufferedBus_WildCardDeduplicates_Succeed(t *testing.T) { } } -func TestBufferedBus_EachSubscribersRecievesEachItem_Succeed(t *testing.T) { - bus := NewBufferedBus(0, 0) +func TestBus_EachSubscribersRecievesEachItem_Succeed(t *testing.T) { + bus := NewBus(0, 0) defer bus.Close() msg1 := NewMessage("test", 1) @@ -130,8 +130,8 @@ func TestBufferedBus_EachSubscribersRecievesEachItem_Succeed(t *testing.T) { assert.Equal(t, msg2, event2) } -func TestBufferedBus_EachSubscribersRecievesEachItemGivenBufferedEventChan_Succeed(t *testing.T) { - bus := NewBufferedBus(0, 2) +func TestBus_EachSubscribersRecievesEachItemGivenBufferedEventChan_Succeed(t *testing.T) { + bus := NewBus(0, 2) defer bus.Close() msg1 := NewMessage("test", 1) @@ -159,8 +159,8 @@ func TestBufferedBus_EachSubscribersRecievesEachItemGivenBufferedEventChan_Succe assert.Equal(t, msg2, output2Ch2) } -func TestBufferedBus_SubscribersDontRecieveItemsAfterUnsubscribing_Succeed(t *testing.T) { - bus := NewBufferedBus(0, 0) +func TestBus_SubscribersDontRecieveItemsAfterUnsubscribing_Succeed(t *testing.T) { + bus := NewBus(0, 0) defer bus.Close() sub, err := bus.Subscribe("test") diff --git a/event/event.go b/event/event.go index cf29859012..e9afdf1a57 100644 --- a/event/event.go +++ b/event/event.go @@ -84,3 +84,29 @@ type Merge struct { // SchemaRoot is the root identifier of the schema that defined the shape of the document that was updated. SchemaRoot string } + +// Message contains event info. +type Message struct { + // Name is the name of the event this message was generated from. + Name Name + + // Data contains optional event information. + Data any +} + +// NewMessage returns a new message with the given name and optional data. +func NewMessage(name Name, data any) Message { + return Message{name, data} +} + +// Subscription is a read-only event stream. +type Subscription struct { + id uint64 + value chan Message + events []Name +} + +// Message returns the next event value from the subscription. +func (s *Subscription) Message() <-chan Message { + return s.value +} diff --git a/http/client.go b/http/client.go index e1fb1034ce..2843ee4f2d 100644 --- a/http/client.go +++ b/http/client.go @@ -451,7 +451,7 @@ func (c *Client) Headstore() ds.Read { panic("client side database") } -func (c *Client) Events() event.Bus { +func (c *Client) Events() *event.Bus { panic("client side database") } diff --git a/internal/db/db.go b/internal/db/db.go index bb4308c1d9..574e748819 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -58,7 +58,7 @@ type db struct { rootstore datastore.RootStore multistore datastore.MultiStore - events event.Bus + events *event.Bus parser core.Parser @@ -109,7 +109,7 @@ func newDB( lensRegistry: lens, parser: parser, options: options, - events: event.NewBufferedBus(commandBufferSize, eventBufferSize), + events: event.NewBus(commandBufferSize, eventBufferSize), } // apply options @@ -251,7 +251,7 @@ func (db *db) initialize(ctx context.Context) error { } // Events returns the events Channel. -func (db *db) Events() event.Bus { +func (db *db) Events() *event.Bus { return db.events } diff --git a/tests/clients/cli/wrapper.go b/tests/clients/cli/wrapper.go index 1681dd6e7b..18e306c0f4 100644 --- a/tests/clients/cli/wrapper.go +++ b/tests/clients/cli/wrapper.go @@ -525,7 +525,7 @@ func (w *Wrapper) Close() { w.node.Close() } -func (w *Wrapper) Events() event.Bus { +func (w *Wrapper) Events() *event.Bus { return w.node.Events() } diff --git a/tests/clients/http/wrapper.go b/tests/clients/http/wrapper.go index 7288781ba5..89b5bce5e7 100644 --- a/tests/clients/http/wrapper.go +++ b/tests/clients/http/wrapper.go @@ -221,7 +221,7 @@ func (w *Wrapper) Close() { w.node.Close() } -func (w *Wrapper) Events() event.Bus { +func (w *Wrapper) Events() *event.Bus { return w.node.Events() } From 726be78ba8557c7ce8b7e14ecff31b19a54d1857 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 17 Jun 2024 14:55:02 -0700 Subject: [PATCH 32/34] make mocks --- client/mocks/db.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/client/mocks/db.go b/client/mocks/db.go index 2cba0da3ed..c56af31167 100644 --- a/client/mocks/db.go +++ b/client/mocks/db.go @@ -360,15 +360,15 @@ func (_c *DB_Close_Call) RunAndReturn(run func()) *DB_Close_Call { } // Events provides a mock function with given fields: -func (_m *DB) Events() event.Bus { +func (_m *DB) Events() *event.Bus { ret := _m.Called() - var r0 event.Bus - if rf, ok := ret.Get(0).(func() event.Bus); ok { + var r0 *event.Bus + if rf, ok := ret.Get(0).(func() *event.Bus); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(event.Bus) + r0 = ret.Get(0).(*event.Bus) } } @@ -392,12 +392,12 @@ func (_c *DB_Events_Call) Run(run func()) *DB_Events_Call { return _c } -func (_c *DB_Events_Call) Return(_a0 event.Bus) *DB_Events_Call { +func (_c *DB_Events_Call) Return(_a0 *event.Bus) *DB_Events_Call { _c.Call.Return(_a0) return _c } -func (_c *DB_Events_Call) RunAndReturn(run func() event.Bus) *DB_Events_Call { +func (_c *DB_Events_Call) RunAndReturn(run func() *event.Bus) *DB_Events_Call { _c.Call.Return(run) return _c } From d0ad720754fcca066d0a216c29f31c0557783bb8 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 18 Jun 2024 15:58:47 -0700 Subject: [PATCH 33/34] update test names --- event/bus_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/event/bus_test.go b/event/bus_test.go index 4d5352bc94..b927e8bec4 100644 --- a/event/bus_test.go +++ b/event/bus_test.go @@ -18,7 +18,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestBus_PushIsNotBlockedWithoutSubscribers_Succeed(t *testing.T) { +func TestBus_IfPublishingWithoutSubscribers_ItShouldNotBlock(t *testing.T) { bus := NewBus(0, 0) defer bus.Close() @@ -29,7 +29,7 @@ func TestBus_PushIsNotBlockedWithoutSubscribers_Succeed(t *testing.T) { assert.True(t, true) } -func TestBus_SubscribersAreNotBlockedAfterClose_Succeed(t *testing.T) { +func TestBus_IfClosingAfterSubscribing_ItShouldNotBlock(t *testing.T) { bus := NewBus(0, 0) defer bus.Close() @@ -44,7 +44,7 @@ func TestBus_SubscribersAreNotBlockedAfterClose_Succeed(t *testing.T) { assert.True(t, true) } -func TestBus_UnsubscribeTwice_Succeed(t *testing.T) { +func TestBus_IfSubscriptionIsUnsubscribedTwice_ItShouldNotPanic(t *testing.T) { bus := NewBus(0, 0) defer bus.Close() @@ -55,7 +55,7 @@ func TestBus_UnsubscribeTwice_Succeed(t *testing.T) { bus.Unsubscribe(sub) } -func TestBus_WildCardDeduplicates_Succeed(t *testing.T) { +func TestBus_IfSubscribedToWildCard_ItShouldNotReceiveMessageTwice(t *testing.T) { bus := NewBus(0, 0) defer bus.Close() @@ -76,7 +76,7 @@ func TestBus_WildCardDeduplicates_Succeed(t *testing.T) { } } -func TestBus_EachSubscribersRecievesEachItem_Succeed(t *testing.T) { +func TestBus_IfMultipleSubscriptionsToTheSameEvent_EachSubscriberRecievesEachEvent(t *testing.T) { bus := NewBus(0, 0) defer bus.Close() @@ -130,7 +130,7 @@ func TestBus_EachSubscribersRecievesEachItem_Succeed(t *testing.T) { assert.Equal(t, msg2, event2) } -func TestBus_EachSubscribersRecievesEachItemGivenBufferedEventChan_Succeed(t *testing.T) { +func TestBus_IfMultipleBufferedSubscribersWithMultipleEvents_EachSubscriberRecievesEachItem(t *testing.T) { bus := NewBus(0, 2) defer bus.Close() @@ -159,7 +159,7 @@ func TestBus_EachSubscribersRecievesEachItemGivenBufferedEventChan_Succeed(t *te assert.Equal(t, msg2, output2Ch2) } -func TestBus_SubscribersDontRecieveItemsAfterUnsubscribing_Succeed(t *testing.T) { +func TestBus_IfSubscribedThenUnsubscribe_SubscriptionShouldNotReceiveEvent(t *testing.T) { bus := NewBus(0, 0) defer bus.Close() From 7680133e172dc34d72a3aa88f992a3ca7df64ddc Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Thu, 20 Jun 2024 09:50:53 -0700 Subject: [PATCH 34/34] use session for dag sync --- net/server.go | 15 --------------- net/sync_dag.go | 3 +++ 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/net/server.go b/net/server.go index 9e09164b37..413f391064 100644 --- a/net/server.go +++ b/net/server.go @@ -337,18 +337,3 @@ func peerIDFromContext(ctx context.Context) (libpeer.ID, error) { } return pid, nil } - -// KEEPING AS REFERENCE -// -// logFromProto returns a thread log from a proto log. -// func logFromProto(l *pb.Log) thread.LogInfo { -// return thread.LogInfo{ -// ID: l.ID.ID, -// PubKey: l.PubKey.PubKey, -// Addrs: addrsFromProto(l.Addrs), -// Head: thread.Head{ -// ID: l.Head.Cid, -// Counter: l.Counter, -// }, -// } -// } diff --git a/net/sync_dag.go b/net/sync_dag.go index 6e9801ebd7..c15ebb7552 100644 --- a/net/sync_dag.go +++ b/net/sync_dag.go @@ -43,7 +43,10 @@ func syncDAG(ctx context.Context, bserv blockservice.BlockService, block *corebl ctx, cancel := context.WithTimeout(ctx, syncDAGTimeout) defer cancel() + // use a session to make remote fetches more efficient + ctx = blockservice.ContextWithSession(ctx, bserv) store := &bsrvadapter.Adapter{Wrapped: bserv} + lsys := cidlink.DefaultLinkSystem() lsys.SetWriteStorage(store) lsys.SetReadStorage(store)