From 943c471016040b9d7ce8b68e13b3767d6c319530 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 1 Mar 2023 02:52:19 -0500 Subject: [PATCH] feat: update go-libipfs and switch gateway code to use go-libipfs implementation --- core/corehttp/gateway.go | 154 ++++++++++++++++--------- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 +- go.mod | 2 +- go.sum | 4 +- 5 files changed, 104 insertions(+), 62 deletions(-) diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index c20ab6e4a4f5..cde6000e19dc 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -2,20 +2,25 @@ package corehttp import ( "context" + "errors" "fmt" "io" "net" "net/http" - cid "github.com/ipfs/go-cid" - "github.com/ipfs/go-libipfs/blocks" + "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-cid" + offline "github.com/ipfs/go-ipfs-exchange-offline" + offlineroute "github.com/ipfs/go-ipfs-routing/offline" "github.com/ipfs/go-libipfs/files" - "github.com/ipfs/go-libipfs/gateway" "github.com/ipfs/go-namesys" iface "github.com/ipfs/interface-go-ipfs-core" - options "github.com/ipfs/interface-go-ipfs-core/options" - nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/ipfs/kubo/core/node" + "github.com/libp2p/go-libp2p/core/routing" + + "github.com/ipfs/go-libipfs/gateway" + options "github.com/ipfs/interface-go-ipfs-core/options" version "github.com/ipfs/kubo" config "github.com/ipfs/kubo/config" core "github.com/ipfs/kubo/core" @@ -47,7 +52,7 @@ func GatewayOption(writable bool, paths ...string) ServeOption { Headers: headers, } - gwAPI, err := newGatewayAPI(n) + gwAPI, err := newGatewayBackend(n) if err != nil { return nil, err } @@ -95,7 +100,7 @@ func HostnameOption() ServeOption { return nil, err } - gwAPI, err := newGatewayAPI(n) + gwAPI, err := newGatewayBackend(n) if err != nil { return nil, err } @@ -118,83 +123,120 @@ func VersionOption() ServeOption { } } -type gatewayAPI struct { - ns namesys.NameSystem - api iface.CoreAPI - offlineAPI iface.CoreAPI -} - -func newGatewayAPI(n *core.IpfsNode) (*gatewayAPI, error) { +func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) { cfg, err := n.Repo.Config() if err != nil { return nil, err } - api, err := coreapi.NewCoreAPI(n, options.Api.FetchBlocks(!cfg.Gateway.NoFetch)) - if err != nil { - return nil, err + bserv := n.Blocks + var vsRouting routing.ValueStore = n.Routing + nsys := n.Namesys + if cfg.Gateway.NoFetch { + bserv = blockservice.New(bserv.Blockstore(), offline.Exchange(bserv.Blockstore())) + + cs := cfg.Ipns.ResolveCacheSize + if cs == 0 { + cs = node.DefaultIpnsCacheSize + } + if cs < 0 { + return nil, fmt.Errorf("cannot specify negative resolve cache size") + } + + vsRouting = offlineroute.NewOfflineRouter(n.Repo.Datastore(), n.RecordValidator) + nsys, err = namesys.NewNameSystem(vsRouting, + namesys.WithDatastore(n.Repo.Datastore()), + namesys.WithDNSResolver(n.DNSResolver), + namesys.WithCache(cs)) + if err != nil { + return nil, fmt.Errorf("error constructing namesys: %w", err) + } } - offlineAPI, err := api.WithOptions(options.Api.Offline(true)) + + gw, err := gateway.NewBlocksGateway(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys)) if err != nil { return nil, err } + return &offlineGatewayErrWrapper{gwimpl: gw}, nil +} + +type offlineGatewayErrWrapper struct { + gwimpl gateway.IPFSBackend +} + +func offlineErrWrap(err error) error { + if errors.Is(err, iface.ErrOffline) { + return fmt.Errorf("%s : %w", err.Error(), gateway.ErrServiceUnavailable) + } + return err +} - return &gatewayAPI{ - ns: n.Namesys, - api: api, - offlineAPI: offlineAPI, - }, nil +func (o *offlineGatewayErrWrapper) Get(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { + md, n, err := o.gwimpl.Get(ctx, path) + err = offlineErrWrap(err) + return md, n, err } -func (gw *gatewayAPI) GetUnixFsNode(ctx context.Context, pth path.Resolved) (files.Node, error) { - return gw.api.Unixfs().Get(ctx, pth) +func (o *offlineGatewayErrWrapper) GetRange(ctx context.Context, path gateway.ImmutablePath, ranges ...gateway.GetRange) (gateway.ContentPathMetadata, files.File, error) { + md, n, err := o.gwimpl.GetRange(ctx, path, ranges...) + err = offlineErrWrap(err) + return md, n, err } -func (gw *gatewayAPI) LsUnixFsDir(ctx context.Context, pth path.Resolved) (<-chan iface.DirEntry, error) { - // Optimization: use Unixfs.Ls without resolving children, but using the - // cumulative DAG size as the file size. This allows for a fast listing - // while keeping a good enough Size field. - return gw.api.Unixfs().Ls(ctx, pth, - options.Unixfs.ResolveChildren(false), - options.Unixfs.UseCumulativeSize(true), - ) +func (o *offlineGatewayErrWrapper) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { + md, n, err := o.gwimpl.GetAll(ctx, path) + err = offlineErrWrap(err) + return md, n, err } -func (gw *gatewayAPI) GetBlock(ctx context.Context, cid cid.Cid) (blocks.Block, error) { - r, err := gw.api.Block().Get(ctx, path.IpfsPath(cid)) - if err != nil { - return nil, err - } +func (o *offlineGatewayErrWrapper) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) { + md, n, err := o.gwimpl.GetBlock(ctx, path) + err = offlineErrWrap(err) + return md, n, err +} - data, err := io.ReadAll(r) - if err != nil { - return nil, err - } +func (o *offlineGatewayErrWrapper) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { + md, n, err := o.gwimpl.Head(ctx, path) + err = offlineErrWrap(err) + return md, n, err +} - return blocks.NewBlockWithCid(data, cid) +func (o *offlineGatewayErrWrapper) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) { + md, err := o.gwimpl.ResolvePath(ctx, path) + err = offlineErrWrap(err) + return md, err } -func (gw *gatewayAPI) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { - return gw.api.Routing().Get(ctx, "/ipns/"+c.String()) +func (o *offlineGatewayErrWrapper) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) { + md, data, errCh, err := o.gwimpl.GetCAR(ctx, path) + err = offlineErrWrap(err) + return md, data, errCh, err } -func (gw *gatewayAPI) GetDNSLinkRecord(ctx context.Context, hostname string) (path.Path, error) { - p, err := gw.ns.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) - if err == namesys.ErrResolveRecursion { - err = nil - } - return path.New(p.String()), err +func (o *offlineGatewayErrWrapper) IsCached(ctx context.Context, path path.Path) bool { + return o.gwimpl.IsCached(ctx, path) +} + +func (o *offlineGatewayErrWrapper) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { + rec, err := o.gwimpl.GetIPNSRecord(ctx, c) + err = offlineErrWrap(err) + return rec, err } -func (gw *gatewayAPI) IsCached(ctx context.Context, pth path.Path) bool { - _, err := gw.offlineAPI.Block().Stat(ctx, pth) - return err == nil +func (o *offlineGatewayErrWrapper) ResolveMutable(ctx context.Context, path path.Path) (gateway.ImmutablePath, error) { + imPath, err := o.gwimpl.ResolveMutable(ctx, path) + err = offlineErrWrap(err) + return imPath, err } -func (gw *gatewayAPI) ResolvePath(ctx context.Context, pth path.Path) (path.Resolved, error) { - return gw.api.ResolvePath(ctx, pth) +func (o *offlineGatewayErrWrapper) GetDNSLinkRecord(ctx context.Context, s string) (path.Path, error) { + p, err := o.gwimpl.GetDNSLinkRecord(ctx, s) + err = offlineErrWrap(err) + return p, err } +var _ gateway.IPFSBackend = (*offlineGatewayErrWrapper)(nil) + var defaultPaths = []string{"/ipfs/", "/ipns/", "/api/", "/p2p/"} var subdomainGatewaySpec = &gateway.Specification{ diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 1c19dd3eb949..dea25c10fbe3 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.18 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/go-libipfs v0.6.1 + github.com/ipfs/go-libipfs v0.6.2-0.20230309095205-cde04e59712b github.com/ipfs/interface-go-ipfs-core v0.11.0 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.26.2 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index c927b58d90e6..f120323e885a 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -569,8 +569,8 @@ github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2 github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg= github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A= github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24= -github.com/ipfs/go-libipfs v0.6.1 h1:OSO9cm1H3r4OXfP0MP1Q5UhTnhd2fByGl6CVYyz/Rhk= -github.com/ipfs/go-libipfs v0.6.1/go.mod h1:FmhKgxMOQA572TK5DA3MZ5GL44ZqsMHIrkgK4gLn4A8= +github.com/ipfs/go-libipfs v0.6.2-0.20230309095205-cde04e59712b h1:8/Y3maFcpkED+e0mMJXHG3BobNYsJxocFamzuhCBoyU= +github.com/ipfs/go-libipfs v0.6.2-0.20230309095205-cde04e59712b/go.mod h1:uC9VqJdOglX/UMYH4vX3R64ab0TqEj1qVsT4ncEQTFM= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A= diff --git a/go.mod b/go.mod index 673dd3793bde..30440b963aa4 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/ipfs/go-ipld-git v0.1.1 github.com/ipfs/go-ipld-legacy v0.1.1 github.com/ipfs/go-ipns v0.3.0 - github.com/ipfs/go-libipfs v0.6.1 + github.com/ipfs/go-libipfs v0.6.2-0.20230309095205-cde04e59712b github.com/ipfs/go-log v1.0.5 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-merkledag v0.9.0 diff --git a/go.sum b/go.sum index 16329d6ad3b7..f6683067a905 100644 --- a/go.sum +++ b/go.sum @@ -591,8 +591,8 @@ github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2 github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg= github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A= github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24= -github.com/ipfs/go-libipfs v0.6.1 h1:OSO9cm1H3r4OXfP0MP1Q5UhTnhd2fByGl6CVYyz/Rhk= -github.com/ipfs/go-libipfs v0.6.1/go.mod h1:FmhKgxMOQA572TK5DA3MZ5GL44ZqsMHIrkgK4gLn4A8= +github.com/ipfs/go-libipfs v0.6.2-0.20230309095205-cde04e59712b h1:8/Y3maFcpkED+e0mMJXHG3BobNYsJxocFamzuhCBoyU= +github.com/ipfs/go-libipfs v0.6.2-0.20230309095205-cde04e59712b/go.mod h1:uC9VqJdOglX/UMYH4vX3R64ab0TqEj1qVsT4ncEQTFM= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A=