From 8b1a8d3ba133a12059c060460178a26c213250b8 Mon Sep 17 00:00:00 2001 From: Andrew Gillis Date: Mon, 15 Jan 2024 16:27:51 -0800 Subject: [PATCH] Sync by segment when listing many ads (#88) --- go.mod | 4 ++-- go.sum | 8 ++++---- pkg/adpub/client.go | 16 +++++++++++++++- pkg/adpub/client_store.go | 28 +++++++--------------------- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index 945b8b6..69fe9cf 100644 --- a/go.mod +++ b/go.mod @@ -9,11 +9,11 @@ require ( github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/go-car/v2 v2.13.1 github.com/ipld/go-ipld-prime v0.21.0 - github.com/ipni/go-libipni v0.5.8 + github.com/ipni/go-libipni v0.5.9 github.com/libp2p/go-libp2p v0.32.2 github.com/mattn/go-isatty v0.0.20 github.com/montanaflynn/stats v0.7.0 - github.com/multiformats/go-multiaddr v0.12.0 + github.com/multiformats/go-multiaddr v0.12.1 github.com/multiformats/go-multicodec v0.9.0 github.com/multiformats/go-multihash v0.2.3 github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index 0065835..6f1f0b4 100644 --- a/go.sum +++ b/go.sum @@ -269,8 +269,8 @@ github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYt github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E= github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= -github.com/ipni/go-libipni v0.5.8 h1:iSob6WRjdRuKpDZmwYw5aBD13utosnEGlRRun/21ZCs= -github.com/ipni/go-libipni v0.5.8/go.mod h1:QvWhyJAY6CZ6IoQNw0DvZQW2nDJWxt/diwxv8udoAIw= +github.com/ipni/go-libipni v0.5.9 h1:AlYlqZScX2jusGXXWkW5j6OMUtMKgQKNcl1Mi8g3glA= +github.com/ipni/go-libipni v0.5.9/go.mod h1:c8mHa6J9iFREpDB29GlPIsbvztRq6bnhg5zJKrnvdUg= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= @@ -376,8 +376,8 @@ github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9 github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= -github.com/multiformats/go-multiaddr v0.12.0 h1:1QlibTFkoXJuDjjYsMHhE73TnzJQl8FSWatk/0gxGzE= -github.com/multiformats/go-multiaddr v0.12.0/go.mod h1:WmZXgObOQOYp9r3cslLlppkrz1FYSHmE834dfz/lWu8= +github.com/multiformats/go-multiaddr v0.12.1 h1:vm+BA/WZA8QZDp1pF1FWhi5CT3g1tbi5GJmqpb6wnlk= +github.com/multiformats/go-multiaddr v0.12.1/go.mod h1:7mPkiBMmLeFipt+nNSq9pHZUeJSt8lHBgH6yhj0YQzE= github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= diff --git a/pkg/adpub/client.go b/pkg/adpub/client.go index 6a23d56..745b1e1 100644 --- a/pkg/adpub/client.go +++ b/pkg/adpub/client.go @@ -18,6 +18,8 @@ import ( "github.com/libp2p/go-libp2p/core/peer" ) +const syncSegmentSize = 2048 + type Client interface { GetAdvertisement(context.Context, cid.Cid) (*Advertisement, error) Close() error @@ -81,7 +83,19 @@ func NewClient(addrInfo peer.AddrInfo, options ...Option) (Client, error) { } func (c *client) List(ctx context.Context, latestCid cid.Cid, n int, w io.Writer) error { - latestCid, err := c.sub.SyncAdChain(ctx, c.publisher, dagsync.WithHeadAdCid(latestCid), dagsync.ScopedDepthLimit(int64(n))) + opts := []dagsync.SyncOption{dagsync.WithHeadAdCid(latestCid), dagsync.ScopedDepthLimit(int64(n))} + if n > syncSegmentSize { + prevAdCid := func(adCid cid.Cid) (cid.Cid, error) { + ad, err := c.store.loadAd(ctx, adCid) + if err != nil { + return cid.Undef, err + } + return ad.PreviousCid(), nil + } + opts = append(opts, dagsync.ScopedSegmentDepthLimit(syncSegmentSize)) + opts = append(opts, dagsync.ScopedBlockHook(dagsync.MakeGeneralBlockHook(prevAdCid))) + } + latestCid, err := c.sub.SyncAdChain(ctx, c.publisher, opts...) if err != nil { return err } diff --git a/pkg/adpub/client_store.go b/pkg/adpub/client_store.go index 9c99fc4..7c4ca34 100644 --- a/pkg/adpub/client_store.go +++ b/pkg/adpub/client_store.go @@ -107,25 +107,16 @@ func (s *ClientStore) getEntriesChunk(ctx context.Context, target cid.Cid) (cid. return next, chunk.Entries, nil } -func (s *ClientStore) getAdvertisement(ctx context.Context, id cid.Cid) (*Advertisement, error) { +func (s *ClientStore) loadAd(ctx context.Context, id cid.Cid) (schema.Advertisement, error) { val, err := s.Batching.Get(ctx, datastore.NewKey(id.String())) if err != nil { - return nil, err - } - - nb := schema.AdvertisementPrototype.NewBuilder() - decoder, err := multicodec.LookupDecoder(id.Prefix().Codec) - if err != nil { - return nil, err - } - - err = decoder(nb, bytes.NewBuffer(val)) - if err != nil { - return nil, err + return schema.Advertisement{}, err } - node := nb.Build() + return schema.BytesToAdvertisement(id, val) +} - ad, err := schema.UnwrapAdvertisement(node) +func (s *ClientStore) getAdvertisement(ctx context.Context, id cid.Cid) (*Advertisement, error) { + ad, err := s.loadAd(ctx, id) if err != nil { return nil, err } @@ -135,18 +126,13 @@ func (s *ClientStore) getAdvertisement(ctx context.Context, id cid.Cid) (*Advert return nil, err } - var prevCid cid.Cid - if ad.PreviousID != nil { - prevCid = ad.PreviousID.(cidlink.Link).Cid - } - a := &Advertisement{ ID: id, ProviderID: dprovid, ContextID: ad.ContextID, Metadata: ad.Metadata, Addresses: ad.Addresses, - PreviousID: prevCid, + PreviousID: ad.PreviousCid(), IsRemove: ad.IsRm, ExtendedProvider: ad.ExtendedProvider, }