Sync by segment when listing many ads (#88)

ipni · Jan 16, 2024 · 8b1a8d3 · 8b1a8d3
1 parent 1c98f22
commit 8b1a8d3
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 28 deletions.
diff --git a/go.mod b/go.mod
@@ -9,11 +9,11 @@ require (
 	github.com/ipfs/go-log/v2 v2.5.1
 	github.com/ipld/go-car/v2 v2.13.1
 	github.com/ipld/go-ipld-prime v0.21.0
-	github.com/ipni/go-libipni v0.5.8
+	github.com/ipni/go-libipni v0.5.9
 	github.com/libp2p/go-libp2p v0.32.2
 	github.com/mattn/go-isatty v0.0.20
 	github.com/montanaflynn/stats v0.7.0
-	github.com/multiformats/go-multiaddr v0.12.0
+	github.com/multiformats/go-multiaddr v0.12.1
 	github.com/multiformats/go-multicodec v0.9.0
 	github.com/multiformats/go-multihash v0.2.3
 	github.com/stretchr/testify v1.8.4

diff --git a/go.sum b/go.sum
@@ -269,8 +269,8 @@ github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYt
 github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E=
 github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ=
 github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
-github.com/ipni/go-libipni v0.5.8 h1:iSob6WRjdRuKpDZmwYw5aBD13utosnEGlRRun/21ZCs=
-github.com/ipni/go-libipni v0.5.8/go.mod h1:QvWhyJAY6CZ6IoQNw0DvZQW2nDJWxt/diwxv8udoAIw=
+github.com/ipni/go-libipni v0.5.9 h1:AlYlqZScX2jusGXXWkW5j6OMUtMKgQKNcl1Mi8g3glA=
+github.com/ipni/go-libipni v0.5.9/go.mod h1:c8mHa6J9iFREpDB29GlPIsbvztRq6bnhg5zJKrnvdUg=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=
 github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
 github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
@@ -376,8 +376,8 @@ github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9
 github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4=
 github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo=
 github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4=
-github.com/multiformats/go-multiaddr v0.12.0 h1:1QlibTFkoXJuDjjYsMHhE73TnzJQl8FSWatk/0gxGzE=
-github.com/multiformats/go-multiaddr v0.12.0/go.mod h1:WmZXgObOQOYp9r3cslLlppkrz1FYSHmE834dfz/lWu8=
+github.com/multiformats/go-multiaddr v0.12.1 h1:vm+BA/WZA8QZDp1pF1FWhi5CT3g1tbi5GJmqpb6wnlk=
+github.com/multiformats/go-multiaddr v0.12.1/go.mod h1:7mPkiBMmLeFipt+nNSq9pHZUeJSt8lHBgH6yhj0YQzE=
 github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A=
 github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk=
 github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E=

diff --git a/pkg/adpub/client.go b/pkg/adpub/client.go
@@ -18,6 +18,8 @@ import (
 	"github.com/libp2p/go-libp2p/core/peer"
 )
 
+const syncSegmentSize = 2048
+
 type Client interface {
 	GetAdvertisement(context.Context, cid.Cid) (*Advertisement, error)
 	Close() error
@@ -81,7 +83,19 @@ func NewClient(addrInfo peer.AddrInfo, options ...Option) (Client, error) {
 }
 
 func (c *client) List(ctx context.Context, latestCid cid.Cid, n int, w io.Writer) error {
-	latestCid, err := c.sub.SyncAdChain(ctx, c.publisher, dagsync.WithHeadAdCid(latestCid), dagsync.ScopedDepthLimit(int64(n)))
+	opts := []dagsync.SyncOption{dagsync.WithHeadAdCid(latestCid), dagsync.ScopedDepthLimit(int64(n))}
+	if n > syncSegmentSize {
+		prevAdCid := func(adCid cid.Cid) (cid.Cid, error) {
+			ad, err := c.store.loadAd(ctx, adCid)
+			if err != nil {
+				return cid.Undef, err
+			}
+			return ad.PreviousCid(), nil
+		}
+		opts = append(opts, dagsync.ScopedSegmentDepthLimit(syncSegmentSize))
+		opts = append(opts, dagsync.ScopedBlockHook(dagsync.MakeGeneralBlockHook(prevAdCid)))
+	}
+	latestCid, err := c.sub.SyncAdChain(ctx, c.publisher, opts...)
 	if err != nil {
 		return err
 	}

diff --git a/pkg/adpub/client_store.go b/pkg/adpub/client_store.go
@@ -107,25 +107,16 @@ func (s *ClientStore) getEntriesChunk(ctx context.Context, target cid.Cid) (cid.
 	return next, chunk.Entries, nil
 }
 
-func (s *ClientStore) getAdvertisement(ctx context.Context, id cid.Cid) (*Advertisement, error) {
+func (s *ClientStore) loadAd(ctx context.Context, id cid.Cid) (schema.Advertisement, error) {
 	val, err := s.Batching.Get(ctx, datastore.NewKey(id.String()))
 	if err != nil {
-		return nil, err
-	}
-
-	nb := schema.AdvertisementPrototype.NewBuilder()
-	decoder, err := multicodec.LookupDecoder(id.Prefix().Codec)
-	if err != nil {
-		return nil, err
-	}
-
-	err = decoder(nb, bytes.NewBuffer(val))
-	if err != nil {
-		return nil, err
+		return schema.Advertisement{}, err
 	}
-	node := nb.Build()
+	return schema.BytesToAdvertisement(id, val)
+}
 
-	ad, err := schema.UnwrapAdvertisement(node)
+func (s *ClientStore) getAdvertisement(ctx context.Context, id cid.Cid) (*Advertisement, error) {
+	ad, err := s.loadAd(ctx, id)
 	if err != nil {
 		return nil, err
 	}
@@ -135,18 +126,13 @@ func (s *ClientStore) getAdvertisement(ctx context.Context, id cid.Cid) (*Advert
 		return nil, err
 	}
 
-	var prevCid cid.Cid
-	if ad.PreviousID != nil {
-		prevCid = ad.PreviousID.(cidlink.Link).Cid
-	}
-
 	a := &Advertisement{
 		ID:               id,
 		ProviderID:       dprovid,
 		ContextID:        ad.ContextID,
 		Metadata:         ad.Metadata,
 		Addresses:        ad.Addresses,
-		PreviousID:       prevCid,
+		PreviousID:       ad.PreviousCid(),
 		IsRemove:         ad.IsRm,
 		ExtendedProvider: ad.ExtendedProvider,
 	}