Merge pull request #1887 from ipfs/feat/ipns-cache

cache ipns entries to speed things up a little

core/commands/ipns.go

@@ -45,6 +45,7 @@ Resolve the value of another name:
 	},
 	Options: []cmds.Option{
 		cmds.BoolOption("recursive", "r", "Resolve until the result is not an IPNS name"),
+		cmds.BoolOption("nocache", "n", "Do not used cached entries"),
 	},
 	Run: func(req cmds.Request, res cmds.Response) {
 
@@ -62,13 +63,27 @@ Resolve the value of another name:
 			}
 		}
 
-		router := n.Routing
-		if local, _, _ := req.Option("local").Bool(); local {
-			router = offline.NewOfflineRouter(n.Repo.Datastore(), n.PrivateKey)
+		nocache, _, _ := req.Option("nocache").Bool()
+		local, _, _ := req.Option("local").Bool()
+
+		// default to nodes namesys resolver
+		var resolver namesys.Resolver = n.Namesys
+
+		if local && nocache {
+			res.SetError(errors.New("cannot specify both local and nocache"), cmds.ErrNormal)
+			return
 		}
 
-		var name string
+		if local {
+			offroute := offline.NewOfflineRouter(n.Repo.Datastore(), n.PrivateKey)
+			resolver = namesys.NewRoutingResolver(offroute, 0)
+		}
 
+		if nocache {
+			resolver = namesys.NewNameSystem(n.Routing, n.Repo.Datastore(), 0)
+		}
+
+		var name string
 		if len(req.Arguments()) == 0 {
 			if n.Identity == "" {
 				res.SetError(errors.New("Identity not loaded!"), cmds.ErrNormal)
@@ -86,7 +101,10 @@ Resolve the value of another name:
 			depth = namesys.DefaultDepthLimit
 		}
 
-		resolver := namesys.NewRoutingResolver(router)
+		if !strings.HasPrefix(name, "/ipns/") {
+			name = "/ipns/" + name
+		}
+
 		output, err := resolver.ResolveN(req.Context(), name, depth)
 		if err != nil {
 			res.SetError(err, cmds.ErrNormal)

core/commands/publish.go

@@ -52,6 +52,7 @@ Publish an <ipfs-path> to another public key (not implemented):
 	Options: []cmds.Option{
 		cmds.BoolOption("resolve", "resolve given path before publishing (default=true)"),
 		cmds.StringOption("lifetime", "t", "time duration that the record will be valid for (default: 24hrs)"),
+		cmds.StringOption("ttl", "time duration this record should be cached for (caution: experimental)"),
 	},
 	Run: func(req cmds.Request, res cmds.Response) {
 		log.Debug("Begin Publish")
@@ -96,7 +97,18 @@ Publish an <ipfs-path> to another public key (not implemented):
 			popts.pubValidTime = d
 		}
 
-		output, err := publish(req.Context(), n, n.PrivateKey, path.Path(pstr), popts)
+		ctx := req.Context()
+		if ttl, found, _ := req.Option("ttl").String(); found {
+			d, err := time.ParseDuration(ttl)
+			if err != nil {
+				res.SetError(err, cmds.ErrNormal)
+				return
+			}
+
+			ctx = context.WithValue(ctx, "ipns-publish-ttl", d)
+		}
+
+		output, err := publish(ctx, n, n.PrivateKey, path.Path(pstr), popts)
 		if err != nil {
 			res.SetError(err, cmds.ErrNormal)
 			return

core/core.go

@@ -226,8 +226,13 @@ func (n *IpfsNode) startOnlineServicesWithHost(ctx context.Context, host p2phost
 	bitswapNetwork := bsnet.NewFromIpfsHost(n.PeerHost, n.Routing)
 	n.Exchange = bitswap.New(ctx, n.Identity, bitswapNetwork, n.Blockstore, alwaysSendToPeer)
 
+	size, err := n.getCacheSize()
+	if err != nil {
+		return err
+	}
+
 	// setup name system
-	n.Namesys = namesys.NewNameSystem(n.Routing, n.Repo.Datastore())
+	n.Namesys = namesys.NewNameSystem(n.Routing, n.Repo.Datastore(), size)
 
 	// setup ipns republishing
 	err = n.setupIpnsRepublisher()
@@ -238,6 +243,23 @@ func (n *IpfsNode) startOnlineServicesWithHost(ctx context.Context, host p2phost
 	return nil
 }
 
+// getCacheSize returns cache life and cache size
+func (n *IpfsNode) getCacheSize() (int, error) {
+	cfg, err := n.Repo.Config()
+	if err != nil {
+		return 0, err
+	}
+
+	cs := cfg.Ipns.ResolveCacheSize
+	if cs == 0 {
+		cs = 128
+	}
+	if cs < 0 {
+		return 0, fmt.Errorf("cannot specify negative resolve cache size")
+	}
+	return cs, nil
+}
+
 func (n *IpfsNode) setupIpnsRepublisher() error {
 	cfg, err := n.Repo.Config()
 	if err != nil {
@@ -456,7 +478,12 @@ func (n *IpfsNode) SetupOfflineRouting() error {
 
 	n.Routing = offroute.NewOfflineRouter(n.Repo.Datastore(), n.PrivateKey)
 
-	n.Namesys = namesys.NewNameSystem(n.Routing, n.Repo.Datastore())
+	size, err := n.getCacheSize()
+	if err != nil {
+		return err
+	}
+
+	n.Namesys = namesys.NewNameSystem(n.Routing, n.Repo.Datastore(), size)
 
 	return nil
 }

fuse/ipns/ipns_test.go

@@ -113,7 +113,7 @@ func setupIpnsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.M
 		}
 
 		node.Routing = offroute.NewOfflineRouter(node.Repo.Datastore(), node.PrivateKey)
-		node.Namesys = namesys.NewNameSystem(node.Routing, node.Repo.Datastore())
+		node.Namesys = namesys.NewNameSystem(node.Routing, node.Repo.Datastore(), 0)
 
 		ipnsfs, err := nsfs.NewFilesystem(context.Background(), node.DAG, node.Namesys, node.Pinning, node.PrivateKey)
 		if err != nil {

namesys/namesys.go

@@ -26,19 +26,21 @@ type mpns struct {
 }
 
 // NewNameSystem will construct the IPFS naming system based on Routing
-func NewNameSystem(r routing.IpfsRouting, ds ds.Datastore) NameSystem {
+func NewNameSystem(r routing.IpfsRouting, ds ds.Datastore, cachesize int) NameSystem {
 	return &mpns{
 		resolvers: map[string]resolver{
 			"dns":      newDNSResolver(),
 			"proquint": new(ProquintResolver),
-			"dht":      newRoutingResolver(r),
+			"dht":      NewRoutingResolver(r, cachesize),
 		},
 		publishers: map[string]Publisher{
 			"/ipns/": NewRoutingPublisher(r, ds),
 		},
 	}
 }
 
+const DefaultResolverCacheTTL = time.Minute
+
 // Resolve implements Resolver.
 func (ns *mpns) Resolve(ctx context.Context, name string) (path.Path, error) {
 	return ns.ResolveN(ctx, name, DefaultDepthLimit)

namesys/pb/namesys.proto

@@ -12,4 +12,6 @@ message IpnsEntry {
 	optional bytes validity = 4;
 
 	optional uint64 sequence = 5;
+
+	optional uint64 ttl = 6;
 }

namesys/publisher.go

@@ -121,13 +121,31 @@ func (p *ipnsPublisher) getPreviousSeqNo(ctx context.Context, ipnskey key.Key) (
 	return e.GetSequence(), nil
 }
 
+// setting the TTL on published records is an experimental feature.
+// as such, i'm using the context to wire it through to avoid changing too
+// much code along the way.
+func checkCtxTTL(ctx context.Context) (time.Duration, bool) {
+	v := ctx.Value("ipns-publish-ttl")
+	if v == nil {
+		return 0, false
+	}
+
+	d, ok := v.(time.Duration)
+	return d, ok
+}
+
 func PutRecordToRouting(ctx context.Context, k ci.PrivKey, value path.Path, seqnum uint64, eol time.Time, r routing.IpfsRouting, id peer.ID) error {
 	namekey, ipnskey := IpnsKeysForID(id)
 	entry, err := CreateRoutingEntryData(k, value, seqnum, eol)
 	if err != nil {
 		return err
 	}
 
+	ttl, ok := checkCtxTTL(ctx)
+	if ok {
+		entry.Ttl = proto.Uint64(uint64(ttl.Nanoseconds()))
+	}
+
 	err = PublishEntry(ctx, r, ipnskey, entry)
 	if err != nil {
 		return err

namesys/republisher/repub_test.go

@@ -18,6 +18,8 @@ import (
 )
 
 func TestRepublish(t *testing.T) {
+	// set cache life to zero for testing low-period repubs
+
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -34,6 +36,8 @@ func TestRepublish(t *testing.T) {
 			t.Fatal(err)
 		}
 
+		nd.Namesys = namesys.NewNameSystem(nd.Routing, nd.Repo.Datastore(), 0)
+
 		nodes = append(nodes, nd)
 	}
 

namesys/resolve_test.go

@@ -19,7 +19,7 @@ func TestRoutingResolve(t *testing.T) {
 	d := mockrouting.NewServer().Client(testutil.RandIdentityOrFatal(t))
 	dstore := ds.NewMapDatastore()
 
-	resolver := NewRoutingResolver(d)
+	resolver := NewRoutingResolver(d, 0)
 	publisher := NewRoutingPublisher(d, dstore)
 
 	privk, pubk, err := testutil.RandTestKeyPair(512)
@@ -53,7 +53,7 @@ func TestPrexistingExpiredRecord(t *testing.T) {
 	dstore := ds.NewMapDatastore()
 	d := mockrouting.NewServer().ClientWithDatastore(context.Background(), testutil.RandIdentityOrFatal(t), dstore)
 
-	resolver := NewRoutingResolver(d)
+	resolver := NewRoutingResolver(d, 0)
 	publisher := NewRoutingPublisher(d, dstore)
 
 	privk, pubk, err := testutil.RandTestKeyPair(512)
@@ -90,7 +90,7 @@ func TestPrexistingRecord(t *testing.T) {
 	dstore := ds.NewMapDatastore()
 	d := mockrouting.NewServer().ClientWithDatastore(context.Background(), testutil.RandIdentityOrFatal(t), dstore)
 
-	resolver := NewRoutingResolver(d)
+	resolver := NewRoutingResolver(d, 0)
 	publisher := NewRoutingPublisher(d, dstore)
 
 	privk, pubk, err := testutil.RandTestKeyPair(512)