diff --git a/examples/gateway/car-file-gateway/libp2pio.car b/examples/gateway/car-file-gateway/libp2pio.car new file mode 100644 index 000000000..6514eef6d Binary files /dev/null and b/examples/gateway/car-file-gateway/libp2pio.car differ diff --git a/examples/gateway/car/main.go b/examples/gateway/car/main.go index cd909c505..3f8e16b20 100644 --- a/examples/gateway/car/main.go +++ b/examples/gateway/car/main.go @@ -29,7 +29,7 @@ func main() { } defer f.Close() - gwAPI, err := common.NewBlocksGateway(blockService, nil) + gwAPI, err := gateway.NewBlocksGateway(blockService) if err != nil { log.Fatal(err) } diff --git a/examples/gateway/car/main_test.go b/examples/gateway/car/main_test.go index fe7174198..71bcfbd80 100644 --- a/examples/gateway/car/main_test.go +++ b/examples/gateway/car/main_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/ipfs/boxo/examples/gateway/common" + "github.com/ipfs/boxo/gateway" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/stretchr/testify/assert" @@ -22,7 +23,7 @@ func newTestServer() (*httptest.Server, io.Closer, error) { return nil, nil, err } - gateway, err := common.NewBlocksGateway(blockService, nil) + gateway, err := gateway.NewBlocksGateway(blockService) if err != nil { _ = f.Close() return nil, nil, err diff --git a/examples/gateway/common/blocks.go b/examples/gateway/common/blocks.go index d8c289f91..e95405de0 100644 --- a/examples/gateway/common/blocks.go +++ b/examples/gateway/common/blocks.go @@ -1,42 +1,12 @@ package common import ( - "context" - "errors" - "fmt" "net/http" - gopath "path" - "github.com/ipfs/boxo/blockservice" - blockstore "github.com/ipfs/boxo/blockstore" - iface "github.com/ipfs/boxo/coreiface" - nsopts "github.com/ipfs/boxo/coreiface/options/namesys" - ifacepath "github.com/ipfs/boxo/coreiface/path" - bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" - "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/gateway" - "github.com/ipfs/boxo/ipld/merkledag" - "github.com/ipfs/boxo/ipld/unixfs" - ufile "github.com/ipfs/boxo/ipld/unixfs/file" - uio "github.com/ipfs/boxo/ipld/unixfs/io" - "github.com/ipfs/boxo/namesys" - "github.com/ipfs/boxo/namesys/resolve" - ipfspath "github.com/ipfs/boxo/path" - "github.com/ipfs/boxo/path/resolver" - "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" - format "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-unixfsnode" - dagpb "github.com/ipld/go-codec-dagpb" - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/node/basicnode" - "github.com/ipld/go-ipld-prime/schema" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/core/routing" - mc "github.com/multiformats/go-multicodec" ) -func NewBlocksHandler(gw *BlocksGateway, port int) http.Handler { +func NewBlocksHandler(gw gateway.IPFSBackend, port int) http.Handler { headers := map[string][]string{} gateway.AddAccessControlHeaders(headers) @@ -50,202 +20,3 @@ func NewBlocksHandler(gw *BlocksGateway, port int) http.Handler { mux.Handle("/ipns/", gwHandler) return mux } - -type BlocksGateway struct { - blockStore blockstore.Blockstore - blockService blockservice.BlockService - dagService format.DAGService - resolver resolver.Resolver - - // Optional routing system to handle /ipns addresses. - namesys namesys.NameSystem - routing routing.ValueStore -} - -func NewBlocksGateway(blockService blockservice.BlockService, routing routing.ValueStore) (*BlocksGateway, error) { - // Setup the DAG services, which use the CAR block store. - dagService := merkledag.NewDAGService(blockService) - - // Setup the UnixFS resolver. - fetcherConfig := bsfetcher.NewFetcherConfig(blockService) - fetcherConfig.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { - if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { - return tlnkNd.LinkTargetNodePrototype(), nil - } - return basicnode.Prototype.Any, nil - }) - fetcher := fetcherConfig.WithReifier(unixfsnode.Reify) - resolver := resolver.NewBasicResolver(fetcher) - - // Setup a name system so that we are able to resolve /ipns links. - var ( - ns namesys.NameSystem - err error - ) - if routing != nil { - ns, err = namesys.NewNameSystem(routing) - if err != nil { - return nil, err - } - } - - return &BlocksGateway{ - blockStore: blockService.Blockstore(), - blockService: blockService, - dagService: dagService, - resolver: resolver, - routing: routing, - namesys: ns, - }, nil -} - -func (api *BlocksGateway) GetUnixFsNode(ctx context.Context, p ifacepath.Resolved) (files.Node, error) { - nd, err := api.resolveNode(ctx, p) - if err != nil { - return nil, err - } - - return ufile.NewUnixfsFile(ctx, api.dagService, nd) -} - -func (api *BlocksGateway) LsUnixFsDir(ctx context.Context, p ifacepath.Resolved) (<-chan iface.DirEntry, error) { - node, err := api.resolveNode(ctx, p) - if err != nil { - return nil, err - } - - dir, err := uio.NewDirectoryFromNode(api.dagService, node) - if err != nil { - return nil, err - } - - out := make(chan iface.DirEntry, uio.DefaultShardWidth) - - go func() { - defer close(out) - for l := range dir.EnumLinksAsync(ctx) { - select { - case out <- api.processLink(ctx, l): - case <-ctx.Done(): - return - } - } - }() - - return out, nil -} - -func (api *BlocksGateway) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { - return api.blockService.GetBlock(ctx, c) -} - -func (api *BlocksGateway) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { - if api.routing == nil { - return nil, routing.ErrNotSupported - } - - // Fails fast if the CID is not an encoded Libp2p Key, avoids wasteful - // round trips to the remote routing provider. - if mc.Code(c.Type()) != mc.Libp2pKey { - return nil, errors.New("provided cid is not an encoded libp2p key") - } - - // The value store expects the key itself to be encoded as a multihash. - id, err := peer.FromCid(c) - if err != nil { - return nil, err - } - - return api.routing.GetValue(ctx, "/ipns/"+string(id)) -} - -func (api *BlocksGateway) GetDNSLinkRecord(ctx context.Context, hostname string) (ifacepath.Path, error) { - if api.namesys != nil { - p, err := api.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) - if err == namesys.ErrResolveRecursion { - err = nil - } - return ifacepath.New(p.String()), err - } - - return nil, errors.New("not implemented") -} - -func (api *BlocksGateway) IsCached(ctx context.Context, p ifacepath.Path) bool { - rp, err := api.ResolvePath(ctx, p) - if err != nil { - return false - } - - has, _ := api.blockStore.Has(ctx, rp.Cid()) - return has -} - -func (api *BlocksGateway) ResolvePath(ctx context.Context, p ifacepath.Path) (ifacepath.Resolved, error) { - if _, ok := p.(ifacepath.Resolved); ok { - return p.(ifacepath.Resolved), nil - } - - err := p.IsValid() - if err != nil { - return nil, err - } - - ipath := ipfspath.Path(p.String()) - if ipath.Segments()[0] == "ipns" { - ipath, err = resolve.ResolveIPNS(ctx, api.namesys, ipath) - if err != nil { - return nil, err - } - } - - if ipath.Segments()[0] != "ipfs" { - return nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace()) - } - - node, rest, err := api.resolver.ResolveToLastNode(ctx, ipath) - if err != nil { - return nil, err - } - - root, err := cid.Parse(ipath.Segments()[1]) - if err != nil { - return nil, err - } - - return ifacepath.NewResolvedPath(ipath, node, root, gopath.Join(rest...)), nil -} - -func (api *BlocksGateway) resolveNode(ctx context.Context, p ifacepath.Path) (format.Node, error) { - rp, err := api.ResolvePath(ctx, p) - if err != nil { - return nil, err - } - - node, err := api.dagService.Get(ctx, rp.Cid()) - if err != nil { - return nil, fmt.Errorf("get node: %w", err) - } - return node, nil -} - -func (api *BlocksGateway) processLink(ctx context.Context, result unixfs.LinkResult) iface.DirEntry { - if result.Err != nil { - return iface.DirEntry{Err: result.Err} - } - - link := iface.DirEntry{ - Name: result.Link.Name, - Cid: result.Link.Cid, - } - - switch link.Cid.Type() { - case cid.Raw: - link.Type = iface.TFile - link.Size = result.Link.Size - case cid.DagProtobuf: - link.Size = result.Link.Size - } - - return link -} diff --git a/examples/gateway/proxy/main.go b/examples/gateway/proxy/main.go index 7cb8a9572..60e2a2c06 100644 --- a/examples/gateway/proxy/main.go +++ b/examples/gateway/proxy/main.go @@ -27,7 +27,7 @@ func main() { routing := newProxyRouting(*gatewayUrlPtr, nil) // Creates the gateway with the block service and the routing. - gwAPI, err := common.NewBlocksGateway(blockService, routing) + gwAPI, err := gateway.NewBlocksGateway(blockService, gateway.WithValueStore(routing)) if err != nil { log.Fatal(err) } diff --git a/examples/gateway/proxy/main_test.go b/examples/gateway/proxy/main_test.go index 2521ff5c6..55a113a4c 100644 --- a/examples/gateway/proxy/main_test.go +++ b/examples/gateway/proxy/main_test.go @@ -9,6 +9,7 @@ import ( "github.com/ipfs/boxo/blockservice" "github.com/ipfs/boxo/examples/gateway/common" offline "github.com/ipfs/boxo/exchange/offline" + "github.com/ipfs/boxo/gateway" "github.com/ipfs/go-block-format" "github.com/stretchr/testify/assert" ) @@ -22,12 +23,12 @@ func newProxyGateway(t *testing.T, rs *httptest.Server) *httptest.Server { blockService := blockservice.New(blockStore, offline.Exchange(blockStore)) routing := newProxyRouting(rs.URL, nil) - gateway, err := common.NewBlocksGateway(blockService, routing) + gw, err := gateway.NewBlocksGateway(blockService, gateway.WithValueStore(routing)) if err != nil { t.Error(err) } - handler := common.NewBlocksHandler(gateway, 0) + handler := common.NewBlocksHandler(gw, 0) ts := httptest.NewServer(handler) t.Cleanup(ts.Close) diff --git a/examples/go.mod b/examples/go.mod index b9d83269b..b59fa9ca7 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -8,9 +8,6 @@ require ( github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.0 github.com/ipfs/go-datastore v0.6.0 - github.com/ipfs/go-ipld-format v0.4.0 - github.com/ipfs/go-unixfsnode v1.6.0 - github.com/ipld/go-codec-dagpb v1.6.0 github.com/ipld/go-ipld-prime v0.20.0 github.com/libp2p/go-libp2p v0.26.3 github.com/libp2p/go-libp2p-routing-helpers v0.6.0 @@ -59,12 +56,15 @@ require ( github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect github.com/ipfs/go-ipld-cbor v0.0.6 // indirect + github.com/ipfs/go-ipld-format v0.4.0 // indirect github.com/ipfs/go-ipld-legacy v0.1.1 // indirect github.com/ipfs/go-ipns v0.3.0 // indirect github.com/ipfs/go-log v1.0.5 // indirect github.com/ipfs/go-log/v2 v2.5.1 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect github.com/ipfs/go-peertaskqueue v0.8.1 // indirect + github.com/ipfs/go-unixfsnode v1.6.0 // indirect + github.com/ipld/go-codec-dagpb v1.6.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect @@ -73,6 +73,7 @@ require ( github.com/koron/go-ssdp v0.0.3 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect + github.com/libp2p/go-doh-resolver v0.4.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect github.com/libp2p/go-libp2p-kad-dht v0.21.1 // indirect diff --git a/examples/go.sum b/examples/go.sum index 57d33cf13..453953be3 100644 --- a/examples/go.sum +++ b/examples/go.sum @@ -375,6 +375,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= +github.com/libp2p/go-doh-resolver v0.4.0 h1:gUBa1f1XsPwtpE1du0O+nnZCUqtG7oYi7Bb+0S7FQqw= +github.com/libp2p/go-doh-resolver v0.4.0/go.mod h1:v1/jwsFusgsWIGX/c6vCRrnJ60x7bhTiq/fs2qt0cAg= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= github.com/libp2p/go-libp2p v0.26.3 h1:6g/psubqwdaBqNNoidbRKSTBEYgaOuKBhHl8Q5tO+PM= @@ -450,6 +452,7 @@ github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU= github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= +github.com/multiformats/go-multiaddr-dns v0.3.0/go.mod h1:mNzQ4eTGDg0ll1N9jKPOUogZPoJ30W8a7zk66FQPpdQ= github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= diff --git a/files/readerfile.go b/files/readerfile.go index a03dae23f..7b4e07954 100644 --- a/files/readerfile.go +++ b/files/readerfile.go @@ -18,7 +18,18 @@ type ReaderFile struct { } func NewBytesFile(b []byte) File { - return &ReaderFile{"", NewReaderFile(bytes.NewReader(b)), nil, int64(len(b))} + return &ReaderFile{"", bytesReaderCloser{bytes.NewReader(b)}, nil, int64(len(b))} +} + +// TODO: Is this the best way to fix this bug? +// The bug is we want to be an io.ReadSeekCloser, but bytes.NewReader only gives a io.ReadSeeker and io.NopCloser +// effectively removes the io.Seeker ability from the passed in io.Reader +type bytesReaderCloser struct { + *bytes.Reader +} + +func (b bytesReaderCloser) Close() error { + return nil } func NewReaderFile(reader io.Reader) File { diff --git a/gateway/blocks_gateway.go b/gateway/blocks_gateway.go new file mode 100644 index 000000000..ce18bfd9d --- /dev/null +++ b/gateway/blocks_gateway.go @@ -0,0 +1,474 @@ +package gateway + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + gopath "path" + "strings" + + "go.uber.org/multierr" + + "github.com/ipfs/boxo/blockservice" + blockstore "github.com/ipfs/boxo/blockstore" + nsopts "github.com/ipfs/boxo/coreiface/options/namesys" + ifacepath "github.com/ipfs/boxo/coreiface/path" + bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" + "github.com/ipfs/boxo/files" + "github.com/ipfs/boxo/ipld/car" + "github.com/ipfs/boxo/ipld/merkledag" + ufile "github.com/ipfs/boxo/ipld/unixfs/file" + uio "github.com/ipfs/boxo/ipld/unixfs/io" + "github.com/ipfs/boxo/namesys" + "github.com/ipfs/boxo/namesys/resolve" + ipfspath "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path/resolver" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-unixfsnode" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/schema" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" + routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/routing" + mc "github.com/multiformats/go-multicodec" + + // Ensure basic codecs are registered. + _ "github.com/ipld/go-ipld-prime/codec/cbor" + _ "github.com/ipld/go-ipld-prime/codec/dagcbor" + _ "github.com/ipld/go-ipld-prime/codec/dagjson" + _ "github.com/ipld/go-ipld-prime/codec/json" +) + +type BlocksGateway struct { + blockStore blockstore.Blockstore + blockService blockservice.BlockService + dagService format.DAGService + resolver resolver.Resolver + + // Optional routing system to handle /ipns addresses. + namesys namesys.NameSystem + routing routing.ValueStore +} + +var _ IPFSBackend = (*BlocksGateway)(nil) + +type gwOptions struct { + ns namesys.NameSystem + vs routing.ValueStore +} + +// WithNameSystem sets the name system to use for the gateway. If not set it will use a default DNSLink resolver +// along with any configured ValueStore +func WithNameSystem(ns namesys.NameSystem) BlockGatewayOption { + return func(opts *gwOptions) error { + opts.ns = ns + return nil + } +} + +// WithValueStore sets the ValueStore to use for the gateway +func WithValueStore(vs routing.ValueStore) BlockGatewayOption { + return func(opts *gwOptions) error { + opts.vs = vs + return nil + } +} + +type BlockGatewayOption func(gwOptions *gwOptions) error + +func NewBlocksGateway(blockService blockservice.BlockService, opts ...BlockGatewayOption) (*BlocksGateway, error) { + var compiledOptions gwOptions + for _, o := range opts { + if err := o(&compiledOptions); err != nil { + return nil, err + } + } + + // Setup the DAG services, which use the CAR block store. + dagService := merkledag.NewDAGService(blockService) + + // Setup the UnixFS resolver. + fetcherConfig := bsfetcher.NewFetcherConfig(blockService) + fetcherConfig.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) + fetcher := fetcherConfig.WithReifier(unixfsnode.Reify) + r := resolver.NewBasicResolver(fetcher) + + // Setup a name system so that we are able to resolve /ipns links. + var ( + ns namesys.NameSystem + vs routing.ValueStore + ) + + vs = compiledOptions.vs + if vs == nil { + vs = routinghelpers.Null{} + } + + ns = compiledOptions.ns + if ns == nil { + dns, err := NewDNSResolver(nil, nil) + if err != nil { + return nil, err + } + + ns, err = namesys.NewNameSystem(vs, namesys.WithDNSResolver(dns)) + if err != nil { + return nil, err + } + } + + return &BlocksGateway{ + blockStore: blockService.Blockstore(), + blockService: blockService, + dagService: dagService, + resolver: r, + routing: vs, + namesys: ns, + }, nil +} + +func (api *BlocksGateway) Get(ctx context.Context, path ImmutablePath) (ContentPathMetadata, *GetResponse, error) { + md, nd, err := api.getNode(ctx, path) + if err != nil { + return md, nil, err + } + + rootCodec := nd.Cid().Prefix().GetCodec() + // This covers both Raw blocks and terminal IPLD codecs like dag-cbor and dag-json + // Note: while only cbor, json, dag-cbor, and dag-json are currently supported by gateways this could change + if rootCodec != uint64(mc.DagPb) { + return md, NewGetResponseFromFile(files.NewBytesFile(nd.RawData())), nil + } + + // This code path covers full graph, single file/directory, and range requests + f, err := ufile.NewUnixfsFile(ctx, api.dagService, nd) + // Note: there is an assumption here that non-UnixFS dag-pb should not be returned which is currently valid + if err != nil { + return md, nil, err + } + + if d, ok := f.(files.Directory); ok { + dir, err := uio.NewDirectoryFromNode(api.dagService, nd) + if err != nil { + return md, nil, err + } + sz, err := d.Size() + if err != nil { + return ContentPathMetadata{}, nil, fmt.Errorf("could not get cumulative directory DAG size: %w", err) + } + if sz < 0 { + return ContentPathMetadata{}, nil, fmt.Errorf("directory cumulative DAG size cannot be negative") + } + return md, NewGetResponseFromDirectoryListing(uint64(sz), dir.EnumLinksAsync(ctx)), nil + } + if file, ok := f.(files.File); ok { + return md, NewGetResponseFromFile(file), nil + } + + return ContentPathMetadata{}, nil, fmt.Errorf("data was not a valid file or directory: %w", ErrInternalServerError) // TODO: should there be a gateway invalid content type to abstract over the various IPLD error types? +} + +func (api *BlocksGateway) GetRange(ctx context.Context, path ImmutablePath, ranges ...GetRange) (ContentPathMetadata, files.File, error) { + md, nd, err := api.getNode(ctx, path) + if err != nil { + return md, nil, err + } + + // This code path covers full graph, single file/directory, and range requests + n, err := ufile.NewUnixfsFile(ctx, api.dagService, nd) + if err != nil { + return md, nil, err + } + f, ok := n.(files.File) + if !ok { + return ContentPathMetadata{}, nil, NewErrorResponse(fmt.Errorf("can only do range requests on files, but did not get a file"), http.StatusBadRequest) + } + + return md, f, nil +} + +func (api *BlocksGateway) GetAll(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.Node, error) { + md, nd, err := api.getNode(ctx, path) + if err != nil { + return md, nil, err + } + + // This code path covers full graph, single file/directory, and range requests + n, err := ufile.NewUnixfsFile(ctx, api.dagService, nd) + if err != nil { + return md, nil, err + } + return md, n, nil +} + +func (api *BlocksGateway) GetBlock(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.File, error) { + md, nd, err := api.getNode(ctx, path) + if err != nil { + return md, nil, err + } + + return md, files.NewBytesFile(nd.RawData()), nil +} + +func (api *BlocksGateway) Head(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.Node, error) { + md, nd, err := api.getNode(ctx, path) + if err != nil { + return md, nil, err + } + + rootCodec := nd.Cid().Prefix().GetCodec() + if rootCodec != uint64(mc.DagPb) { + return md, files.NewBytesFile(nd.RawData()), nil + } + + // TODO: We're not handling non-UnixFS dag-pb. There's a bit of a discrepancy between what we want from a HEAD request and a Resolve request here and we're using this for both + fileNode, err := ufile.NewUnixfsFile(ctx, api.dagService, nd) + if err != nil { + return ContentPathMetadata{}, nil, err + } + + return md, fileNode, nil +} + +func (api *BlocksGateway) GetCAR(ctx context.Context, path ImmutablePath) (ContentPathMetadata, io.ReadCloser, <-chan error, error) { + // Same go-car settings as dag.export command + store := dagStore{api: api, ctx: ctx} + + // TODO: When switching to exposing path blocks we'll want to add these as well + roots, lastSeg, err := api.getPathRoots(ctx, path) + if err != nil { + return ContentPathMetadata{}, nil, nil, err + } + + md := ContentPathMetadata{ + PathSegmentRoots: roots, + LastSegment: lastSeg, + } + + rootCid := lastSeg.Cid() + + // TODO: support selectors passed as request param: https://github.com/ipfs/kubo/issues/8769 + // TODO: this is very slow if blocks are remote due to linear traversal. Do we need deterministic traversals here? + dag := car.Dag{Root: rootCid, Selector: selectorparse.CommonSelector_ExploreAllRecursively} + c := car.NewSelectiveCar(ctx, store, []car.Dag{dag}, car.TraverseLinksOnlyOnce()) + r, w := io.Pipe() + + errCh := make(chan error, 1) + go func() { + carWriteErr := c.Write(w) + pipeCloseErr := w.Close() + errCh <- multierr.Combine(carWriteErr, pipeCloseErr) + close(errCh) + }() + + return md, r, errCh, nil +} + +func (api *BlocksGateway) getNode(ctx context.Context, path ImmutablePath) (ContentPathMetadata, format.Node, error) { + roots, lastSeg, err := api.getPathRoots(ctx, path) + if err != nil { + return ContentPathMetadata{}, nil, err + } + + md := ContentPathMetadata{ + PathSegmentRoots: roots, + LastSegment: lastSeg, + } + + lastRoot := lastSeg.Cid() + + nd, err := api.dagService.Get(ctx, lastRoot) + if err != nil { + return ContentPathMetadata{}, nil, err + } + + return md, nd, err +} + +func (api *BlocksGateway) getPathRoots(ctx context.Context, contentPath ImmutablePath) ([]cid.Cid, ifacepath.Resolved, error) { + /* + These are logical roots where each CID represent one path segment + and resolves to either a directory or the root block of a file. + The main purpose of this header is allow HTTP caches to do smarter decisions + around cache invalidation (eg. keep specific subdirectory/file if it did not change) + A good example is Wikipedia, which is HAMT-sharded, but we only care about + logical roots that represent each segment of the human-readable content + path: + Given contentPath = /ipns/en.wikipedia-on-ipfs.org/wiki/Block_of_Wikipedia_in_Turkey + rootCidList is a generated by doing `ipfs resolve -r` on each sub path: + /ipns/en.wikipedia-on-ipfs.org → bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze + /ipns/en.wikipedia-on-ipfs.org/wiki/ → bafybeihn2f7lhumh4grizksi2fl233cyszqadkn424ptjajfenykpsaiw4 + /ipns/en.wikipedia-on-ipfs.org/wiki/Block_of_Wikipedia_in_Turkey → bafkreibn6euazfvoghepcm4efzqx5l3hieof2frhp254hio5y7n3hv5rma + The result is an ordered array of values: + X-Ipfs-Roots: bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze,bafybeihn2f7lhumh4grizksi2fl233cyszqadkn424ptjajfenykpsaiw4,bafkreibn6euazfvoghepcm4efzqx5l3hieof2frhp254hio5y7n3hv5rma + Note that while the top one will change every time any article is changed, + the last root (responsible for specific article) may not change at all. + */ + var sp strings.Builder + var pathRoots []cid.Cid + contentPathStr := contentPath.String() + pathSegments := strings.Split(contentPathStr[6:], "/") + sp.WriteString(contentPathStr[:5]) // /ipfs or /ipns + var lastPath ifacepath.Resolved + for _, root := range pathSegments { + if root == "" { + continue + } + sp.WriteString("/") + sp.WriteString(root) + resolvedSubPath, err := api.resolvePath(ctx, ifacepath.New(sp.String())) + if err != nil { + // TODO: should we be more explicit here and is this part of the Gateway API contract? + // The issue here was that we returned datamodel.ErrWrongKind instead of this resolver error + if isErrNotFound(err) { + return nil, nil, resolver.ErrNoLink{Name: root, Node: lastPath.Cid()} + } + return nil, nil, err + } + lastPath = resolvedSubPath + pathRoots = append(pathRoots, lastPath.Cid()) + } + + pathRoots = pathRoots[:len(pathRoots)-1] + return pathRoots, lastPath, nil +} + +// FIXME(@Jorropo): https://github.com/ipld/go-car/issues/315 +type dagStore struct { + api *BlocksGateway + ctx context.Context +} + +func (ds dagStore) Get(_ context.Context, c cid.Cid) (blocks.Block, error) { + return ds.api.blockService.GetBlock(ds.ctx, c) +} + +func (api *BlocksGateway) ResolveMutable(ctx context.Context, p ifacepath.Path) (ImmutablePath, error) { + err := p.IsValid() + if err != nil { + return ImmutablePath{}, err + } + + ipath := ipfspath.Path(p.String()) + switch ipath.Segments()[0] { + case "ipns": + ipath, err = resolve.ResolveIPNS(ctx, api.namesys, ipath) + if err != nil { + return ImmutablePath{}, err + } + imPath, err := NewImmutablePath(ifacepath.New(ipath.String())) + if err != nil { + return ImmutablePath{}, err + } + return imPath, nil + case "ipfs": + imPath, err := NewImmutablePath(ifacepath.New(ipath.String())) + if err != nil { + return ImmutablePath{}, err + } + return imPath, nil + default: + return ImmutablePath{}, NewErrorResponse(fmt.Errorf("unsupported path namespace: %s", p.Namespace()), http.StatusNotImplemented) + } +} + +func (api *BlocksGateway) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { + if api.routing == nil { + return nil, NewErrorResponse(errors.New("IPNS Record responses are not supported by this gateway"), http.StatusNotImplemented) + } + + // Fails fast if the CID is not an encoded Libp2p Key, avoids wasteful + // round trips to the remote routing provider. + if mc.Code(c.Type()) != mc.Libp2pKey { + return nil, NewErrorResponse(errors.New("cid codec must be libp2p-key"), http.StatusBadRequest) + } + + // The value store expects the key itself to be encoded as a multihash. + id, err := peer.FromCid(c) + if err != nil { + return nil, err + } + + return api.routing.GetValue(ctx, "/ipns/"+string(id)) +} + +func (api *BlocksGateway) GetDNSLinkRecord(ctx context.Context, hostname string) (ifacepath.Path, error) { + if api.namesys != nil { + p, err := api.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) + if err == namesys.ErrResolveRecursion { + err = nil + } + return ifacepath.New(p.String()), err + } + + return nil, NewErrorResponse(errors.New("not implemented"), http.StatusNotImplemented) +} + +func (api *BlocksGateway) IsCached(ctx context.Context, p ifacepath.Path) bool { + rp, err := api.resolvePath(ctx, p) + if err != nil { + return false + } + + has, _ := api.blockStore.Has(ctx, rp.Cid()) + return has +} + +func (api *BlocksGateway) ResolvePath(ctx context.Context, path ImmutablePath) (ContentPathMetadata, error) { + roots, lastSeg, err := api.getPathRoots(ctx, path) + if err != nil { + return ContentPathMetadata{}, err + } + md := ContentPathMetadata{ + PathSegmentRoots: roots, + LastSegment: lastSeg, + } + return md, nil +} + +func (api *BlocksGateway) resolvePath(ctx context.Context, p ifacepath.Path) (ifacepath.Resolved, error) { + if _, ok := p.(ifacepath.Resolved); ok { + return p.(ifacepath.Resolved), nil + } + + err := p.IsValid() + if err != nil { + return nil, err + } + + ipath := ipfspath.Path(p.String()) + if ipath.Segments()[0] == "ipns" { + ipath, err = resolve.ResolveIPNS(ctx, api.namesys, ipath) + if err != nil { + return nil, err + } + } + + if ipath.Segments()[0] != "ipfs" { + return nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace()) + } + + node, rest, err := api.resolver.ResolveToLastNode(ctx, ipath) + if err != nil { + return nil, err + } + + root, err := cid.Parse(ipath.Segments()[1]) + if err != nil { + return nil, err + } + + return ifacepath.NewResolvedPath(ipath, node, root, gopath.Join(rest...)), nil +} diff --git a/gateway/dns.go b/gateway/dns.go new file mode 100644 index 000000000..504bb8311 --- /dev/null +++ b/gateway/dns.go @@ -0,0 +1,92 @@ +package gateway + +import ( + "fmt" + "strings" + + "github.com/libp2p/go-doh-resolver" + dns "github.com/miekg/dns" + madns "github.com/multiformats/go-multiaddr-dns" +) + +var defaultResolvers = map[string]string{ + "eth.": "https://resolver.cloudflare-eth.com/dns-query", + "crypto.": "https://resolver.cloudflare-eth.com/dns-query", +} + +func newResolver(url string, opts ...doh.Option) (madns.BasicResolver, error) { + if !strings.HasPrefix(url, "https://") { + return nil, fmt.Errorf("invalid resolver url: %s", url) + } + + return doh.NewResolver(url, opts...) +} + +// NewDNSResolver creates a new DNS resolver based on the default resolvers and +// the provided resolvers. +// +// The argument 'resolvers' is a map of FQDNs to URLs for custom DNS resolution. +// URLs starting with `https://` indicate DoH endpoints. Support for other resolver +// types may be added in the future. +// +// https://en.wikipedia.org/wiki/Fully_qualified_domain_name +// https://en.wikipedia.org/wiki/DNS_over_HTTPS +// +// Example: +// - Custom resolver for ENS: `eth.` → `https://eth.link/dns-query` +// - Override the default OS resolver: `.` → `https://doh.applied-privacy.net/query` +func NewDNSResolver(resolvers map[string]string, dohOpts ...doh.Option) (*madns.Resolver, error) { + var opts []madns.Option + var err error + + domains := make(map[string]struct{}) // to track overridden default resolvers + rslvrs := make(map[string]madns.BasicResolver) // to reuse resolvers for the same URL + + for domain, url := range resolvers { + if domain != "." && !dns.IsFqdn(domain) { + return nil, fmt.Errorf("invalid domain %s; must be FQDN", domain) + } + + domains[domain] = struct{}{} + if url == "" { + // allow overriding of implicit defaults with the default resolver + continue + } + + rslv, ok := rslvrs[url] + if !ok { + rslv, err = newResolver(url, dohOpts...) + if err != nil { + return nil, fmt.Errorf("bad resolver for %s: %w", domain, err) + } + rslvrs[url] = rslv + } + + if domain != "." { + opts = append(opts, madns.WithDomainResolver(domain, rslv)) + } else { + opts = append(opts, madns.WithDefaultResolver(rslv)) + } + } + + // fill in defaults if not overridden by the user + for domain, url := range defaultResolvers { + _, ok := domains[domain] + if ok { + continue + } + + rslv, ok := rslvrs[url] + if !ok { + rslv, err = newResolver(url) + if err != nil { + return nil, fmt.Errorf("bad resolver for %s: %w", domain, err) + } + rslvrs[url] = rslv + } + + opts = append(opts, madns.WithDomainResolver(domain, rslv)) + } + + return madns.NewResolver(opts...) +} diff --git a/gateway/errors.go b/gateway/errors.go index 191426742..a0ef4db79 100644 --- a/gateway/errors.go +++ b/gateway/errors.go @@ -11,6 +11,7 @@ import ( "github.com/ipfs/boxo/path/resolver" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" + "github.com/ipld/go-ipld-prime/datamodel" ) var ( @@ -160,15 +161,24 @@ func isErrNotFound(err error) bool { return true } - // Checks if err is a resolver.ErrNoLink. resolver.ErrNoLink does not implement - // the .Is interface and cannot be directly compared to. Therefore, errors.Is - // always returns false with it. + // Checks if err is of a type that does not implement the .Is interface and + // cannot be directly compared to. Therefore, errors.Is cannot be used. for { _, ok := err.(resolver.ErrNoLink) if ok { return true } + _, ok = err.(datamodel.ErrWrongKind) + if ok { + return true + } + + _, ok = err.(datamodel.ErrNotExists) + if ok { + return true + } + err = errors.Unwrap(err) if err == nil { return false diff --git a/gateway/gateway.go b/gateway/gateway.go index 99ca74783..86d2db8fb 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -2,14 +2,15 @@ package gateway import ( "context" + "fmt" + "io" "net/http" "sort" - iface "github.com/ipfs/boxo/coreiface" "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/boxo/files" - "github.com/ipfs/go-block-format" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/boxo/ipld/unixfs" + "github.com/ipfs/go-cid" ) // Config is the configuration used when creating a new gateway handler. @@ -17,34 +18,129 @@ type Config struct { Headers map[string][]string } -// API defines the minimal set of API services required for a gateway handler. -type API interface { - // GetUnixFsNode returns a read-only handle to a file tree referenced by a path. - GetUnixFsNode(context.Context, path.Resolved) (files.Node, error) +// TODO: Is this what we want for ImmutablePath? +type ImmutablePath struct { + p path.Path +} + +func NewImmutablePath(p path.Path) (ImmutablePath, error) { + if p.Mutable() { + return ImmutablePath{}, fmt.Errorf("path cannot be mutable") + } + return ImmutablePath{p: p}, nil +} + +func (i ImmutablePath) String() string { + return i.p.String() +} + +func (i ImmutablePath) Namespace() string { + return i.p.Namespace() +} + +func (i ImmutablePath) Mutable() bool { + return false +} + +func (i ImmutablePath) IsValid() error { + return i.p.IsValid() +} + +var _ path.Path = (*ImmutablePath)(nil) + +type ContentPathMetadata struct { + PathSegmentRoots []cid.Cid + LastSegment path.Resolved + ContentType string // Only used for UnixFS requests +} + +// GetRange describes a range request within a UnixFS file. From and To mostly follow HTTP Range Request semantics. +// From >= 0 and To = nil: Get the file (From, Length) +// From >= 0 and To >= 0: Get the range (From, To) +// From >= 0 and To <0: Get the range (From, Length - To) +type GetRange struct { + From uint64 + To *int64 +} + +type GetResponse struct { + bytes files.File + directoryMetadata *directoryMetadata +} - // LsUnixFsDir returns the list of links in a directory. - LsUnixFsDir(context.Context, path.Resolved) (<-chan iface.DirEntry, error) +type directoryMetadata struct { + dagSize uint64 + entries <-chan unixfs.LinkResult +} + +func NewGetResponseFromFile(file files.File) *GetResponse { + return &GetResponse{bytes: file} +} + +func NewGetResponseFromDirectoryListing(dagSize uint64, entries <-chan unixfs.LinkResult) *GetResponse { + return &GetResponse{directoryMetadata: &directoryMetadata{dagSize, entries}} +} - // GetBlock return a block from a certain CID. - GetBlock(context.Context, cid.Cid) (blocks.Block, error) +// IPFSBackend is the required set of functionality used to implement the IPFS HTTP Gateway specification. +// To signal error types to the gateway code (so that not everything is a HTTP 500) return an error wrapped with NewErrorResponse. +// There are also some existing error types that the gateway code knows how to handle (e.g. context.DeadlineExceeded +// and various IPLD pathing related errors). +type IPFSBackend interface { + // Get returns a UnixFS file, UnixFS directory, or an IPLD block depending on what the path is that has been + // requested. Directories' files.DirEntry objects do not need to contain content, but must contain Name, + // Size, and Cid. + Get(context.Context, ImmutablePath) (ContentPathMetadata, *GetResponse, error) + + // GetRange returns a full UnixFS file object. Ranges passed in are advisory for pre-fetching data, however + // consumers of this function may require extra data beyond the passed ranges (e.g. the initial bit of the file + // might be used for content type sniffing even if only the end of the file is requested). + GetRange(context.Context, ImmutablePath, ...GetRange) (ContentPathMetadata, files.File, error) + + // GetAll returns a UnixFS file or directory depending on what the path is that has been requested. Directories should + // include all content recursively. + GetAll(context.Context, ImmutablePath) (ContentPathMetadata, files.Node, error) + + // GetBlock returns a single block of data + GetBlock(context.Context, ImmutablePath) (ContentPathMetadata, files.File, error) + + // Head returns a file or directory depending on what the path is that has been requested. + // For UnixFS files should return a file which has the correct file size and either returns the ContentType in ContentPathMetadata or + // enough data (e.g. 3kiB) such that the content type can be determined by sniffing. + // For all other data types returning just size information is sufficient + // TODO: give function more explicit return types + Head(context.Context, ImmutablePath) (ContentPathMetadata, files.Node, error) + + // ResolvePath resolves the path using UnixFS resolver. If the path does not + // exist due to a missing link, it should return an error of type: + // NewErrorResponse(fmt.Errorf("no link named %q under %s", name, cid), http.StatusNotFound) + ResolvePath(context.Context, ImmutablePath) (ContentPathMetadata, error) + + // GetCAR returns a CAR file for the given immutable path + // Returns an initial error if there was an issue before the CAR streaming begins as well as a channel with a single + // that may contain a single error for if any errors occur during the streaming. If there was an initial error the + // error channel is nil + // TODO: Make this function signature better + GetCAR(context.Context, ImmutablePath) (ContentPathMetadata, io.ReadCloser, <-chan error, error) + + // IsCached returns whether or not the path exists locally. + IsCached(context.Context, path.Path) bool // GetIPNSRecord retrieves the best IPNS record for a given CID (libp2p-key) // from the routing system. GetIPNSRecord(context.Context, cid.Cid) ([]byte, error) + // ResolveMutable takes a mutable path and resolves it into an immutable one. This means recursively resolving any + // DNSLink or IPNS records. + // + // For example, given a mapping from `/ipns/dnslink.tld -> /ipns/ipns-id/mydirectory` and `/ipns/ipns-id` to + // `/ipfs/some-cid`, the result of passing `/ipns/dnslink.tld/myfile` would be `/ipfs/some-cid/mydirectory/myfile`. + ResolveMutable(context.Context, path.Path) (ImmutablePath, error) + // GetDNSLinkRecord returns the DNSLink TXT record for the provided FQDN. // Unlike ResolvePath, it does not perform recursive resolution. It only // checks for the existence of a DNSLink TXT record with path starting with // /ipfs/ or /ipns/ and returns the path as-is. GetDNSLinkRecord(context.Context, string) (path.Path, error) - - // IsCached returns whether or not the path exists locally. - IsCached(context.Context, path.Path) bool - - // ResolvePath resolves the path using UnixFS resolver. If the path does not - // exist due to a missing link, it should return an error of type: - // https://pkg.go.dev/github.com/ipfs/go-path@v0.3.0/resolver#ErrNoLink - ResolvePath(context.Context, path.Path) (path.Resolved, error) } // A helper function to clean up a set of headers: diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index 39924f085..aa36b47e9 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -8,36 +8,19 @@ import ( "net/http" "net/http/httptest" "os" - gopath "path" "regexp" "strings" "testing" "github.com/ipfs/boxo/blockservice" - blockstore "github.com/ipfs/boxo/blockstore" - iface "github.com/ipfs/boxo/coreiface" nsopts "github.com/ipfs/boxo/coreiface/options/namesys" ipath "github.com/ipfs/boxo/coreiface/path" offline "github.com/ipfs/boxo/exchange/offline" - bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" "github.com/ipfs/boxo/files" carblockstore "github.com/ipfs/boxo/ipld/car/v2/blockstore" - "github.com/ipfs/boxo/ipld/merkledag" - "github.com/ipfs/boxo/ipld/unixfs" - ufile "github.com/ipfs/boxo/ipld/unixfs/file" - uio "github.com/ipfs/boxo/ipld/unixfs/io" "github.com/ipfs/boxo/namesys" - "github.com/ipfs/boxo/namesys/resolve" path "github.com/ipfs/boxo/path" - "github.com/ipfs/boxo/path/resolver" - "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - format "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-unixfsnode" - dagpb "github.com/ipld/go-codec-dagpb" - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/node/basicnode" - "github.com/ipld/go-ipld-prime/schema" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/routing" "github.com/stretchr/testify/assert" @@ -88,13 +71,12 @@ func (m mockNamesys) GetResolver(subs string) (namesys.Resolver, bool) { } type mockAPI struct { - blockStore blockstore.Blockstore - blockService blockservice.BlockService - dagService format.DAGService - resolver resolver.Resolver - namesys mockNamesys + gw IPFSBackend + namesys mockNamesys } +var _ IPFSBackend = (*mockAPI)(nil) + func newMockAPI(t *testing.T) (*mockAPI, cid.Cid) { r, err := os.Open("./testdata/fixtures.car") assert.Nil(t, err) @@ -112,65 +94,45 @@ func newMockAPI(t *testing.T) (*mockAPI, cid.Cid) { assert.Len(t, cids, 1) blockService := blockservice.New(blockStore, offline.Exchange(blockStore)) - dagService := merkledag.NewDAGService(blockService) - fetcherConfig := bsfetcher.NewFetcherConfig(blockService) - fetcherConfig.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { - if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { - return tlnkNd.LinkTargetNodePrototype(), nil - } - return basicnode.Prototype.Any, nil - }) - fetcher := fetcherConfig.WithReifier(unixfsnode.Reify) - resolver := resolver.NewBasicResolver(fetcher) + n := mockNamesys{} + gwApi, err := NewBlocksGateway(blockService, WithNameSystem(n)) + if err != nil { + t.Fatal(err) + } return &mockAPI{ - blockStore: blockService.Blockstore(), - blockService: blockService, - dagService: dagService, - resolver: resolver, - namesys: mockNamesys{}, + gw: gwApi, + namesys: n, }, cids[0] } -func (api *mockAPI) GetUnixFsNode(ctx context.Context, p ipath.Resolved) (files.Node, error) { - nd, err := api.resolveNode(ctx, p) - if err != nil { - return nil, err - } - - return ufile.NewUnixfsFile(ctx, api.dagService, nd) +func (api *mockAPI) Get(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, *GetResponse, error) { + return api.gw.Get(ctx, immutablePath) } -func (api *mockAPI) LsUnixFsDir(ctx context.Context, p ipath.Resolved) (<-chan iface.DirEntry, error) { - node, err := api.resolveNode(ctx, p) - if err != nil { - return nil, err - } +func (api *mockAPI) GetRange(ctx context.Context, immutablePath ImmutablePath, ranges ...GetRange) (ContentPathMetadata, files.File, error) { + return api.gw.GetRange(ctx, immutablePath, ranges...) +} - dir, err := uio.NewDirectoryFromNode(api.dagService, node) - if err != nil { - return nil, err - } +func (api *mockAPI) GetAll(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.Node, error) { + return api.gw.GetAll(ctx, immutablePath) +} - out := make(chan iface.DirEntry, uio.DefaultShardWidth) +func (api *mockAPI) GetBlock(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.File, error) { + return api.gw.GetBlock(ctx, immutablePath) +} - go func() { - defer close(out) - for l := range dir.EnumLinksAsync(ctx) { - select { - case out <- api.processLink(ctx, l): - case <-ctx.Done(): - return - } - } - }() +func (api *mockAPI) Head(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.Node, error) { + return api.gw.Head(ctx, immutablePath) +} - return out, nil +func (api *mockAPI) GetCAR(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, io.ReadCloser, <-chan error, error) { + return api.gw.GetCAR(ctx, immutablePath) } -func (api *mockAPI) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { - return api.blockService.GetBlock(ctx, c) +func (api *mockAPI) ResolveMutable(ctx context.Context, p ipath.Path) (ImmutablePath, error) { + return api.gw.ResolveMutable(ctx, p) } func (api *mockAPI) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { @@ -190,82 +152,33 @@ func (api *mockAPI) GetDNSLinkRecord(ctx context.Context, hostname string) (ipat } func (api *mockAPI) IsCached(ctx context.Context, p ipath.Path) bool { - rp, err := api.ResolvePath(ctx, p) - if err != nil { - return false - } - - has, _ := api.blockStore.Has(ctx, rp.Cid()) - return has + return api.gw.IsCached(ctx, p) } -func (api *mockAPI) ResolvePath(ctx context.Context, ip ipath.Path) (ipath.Resolved, error) { - if _, ok := ip.(ipath.Resolved); ok { - return ip.(ipath.Resolved), nil - } - - err := ip.IsValid() - if err != nil { - return nil, err - } +func (api *mockAPI) ResolvePath(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, error) { + return api.gw.ResolvePath(ctx, immutablePath) +} - p := path.Path(ip.String()) - if p.Segments()[0] == "ipns" { - p, err = resolve.ResolveIPNS(ctx, api.namesys, p) +func (api *mockAPI) resolvePathNoRootsReturned(ctx context.Context, ip ipath.Path) (ipath.Resolved, error) { + var imPath ImmutablePath + var err error + if ip.Mutable() { + imPath, err = api.ResolveMutable(ctx, ip) + if err != nil { + return nil, err + } + } else { + imPath, err = NewImmutablePath(ip) if err != nil { return nil, err } } - if p.Segments()[0] != "ipfs" { - return nil, fmt.Errorf("unsupported path namespace: %s", ip.Namespace()) - } - - node, rest, err := api.resolver.ResolveToLastNode(ctx, p) - if err != nil { - return nil, err - } - - root, err := cid.Parse(p.Segments()[1]) - if err != nil { - return nil, err - } - - return ipath.NewResolvedPath(p, node, root, gopath.Join(rest...)), nil -} - -func (api *mockAPI) resolveNode(ctx context.Context, p ipath.Path) (format.Node, error) { - rp, err := api.ResolvePath(ctx, p) + md, err := api.ResolvePath(ctx, imPath) if err != nil { return nil, err } - - node, err := api.dagService.Get(ctx, rp.Cid()) - if err != nil { - return nil, fmt.Errorf("get node: %w", err) - } - return node, nil -} - -func (api *mockAPI) processLink(ctx context.Context, result unixfs.LinkResult) iface.DirEntry { - if result.Err != nil { - return iface.DirEntry{Err: result.Err} - } - - link := iface.DirEntry{ - Name: result.Link.Name, - Cid: result.Link.Cid, - } - - switch link.Cid.Type() { - case cid.Raw: - link.Type = iface.TFile - link.Size = result.Link.Size - case cid.DagProtobuf: - link.Size = result.Link.Size - } - - return link + return md.LastSegment, nil } func doWithoutRedirect(req *http.Request) (*http.Response, error) { @@ -288,7 +201,7 @@ func newTestServerAndNode(t *testing.T, ns mockNamesys) (*httptest.Server, *mock return ts, api, root } -func newTestServer(t *testing.T, api API) *httptest.Server { +func newTestServer(t *testing.T, api IPFSBackend) *httptest.Server { config := Config{Headers: map[string][]string{}} AddAccessControlHeaders(config.Headers) @@ -316,7 +229,7 @@ func TestGatewayGet(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - k, err := api.ResolvePath(ctx, ipath.Join(ipath.IpfsPath(root), t.Name(), "fnord")) + k, err := api.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), t.Name(), "fnord")) assert.Nil(t, err) api.namesys["/ipns/example.com"] = path.FromCid(k.Cid()) @@ -423,7 +336,7 @@ func TestIPNSHostnameRedirect(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - k, err := api.ResolvePath(ctx, ipath.Join(ipath.IpfsPath(root), t.Name())) + k, err := api.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), t.Name())) assert.Nil(t, err) t.Logf("k: %s\n", k) @@ -478,14 +391,14 @@ func TestIPNSHostnameBacklinks(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - k, err := api.ResolvePath(ctx, ipath.Join(ipath.IpfsPath(root), t.Name())) + k, err := api.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), t.Name())) assert.Nil(t, err) // create /ipns/example.net/foo/ - k2, err := api.ResolvePath(ctx, ipath.Join(k, "foo? #<'")) + k2, err := api.resolvePathNoRootsReturned(ctx, ipath.Join(k, "foo? #<'")) assert.Nil(t, err) - k3, err := api.ResolvePath(ctx, ipath.Join(k, "foo? #<'/bar")) + k3, err := api.resolvePathNoRootsReturned(ctx, ipath.Join(k, "foo? #<'/bar")) assert.Nil(t, err) t.Logf("k: %s\n", k) @@ -562,7 +475,7 @@ func TestPretty404(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - k, err := api.ResolvePath(ctx, ipath.Join(ipath.IpfsPath(root), t.Name())) + k, err := api.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), t.Name())) assert.Nil(t, err) host := "example.net" diff --git a/gateway/handler.go b/gateway/handler.go index 02b907f6f..4c3fe29fd 100644 --- a/gateway/handler.go +++ b/gateway/handler.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "errors" "fmt" "html/template" "io" @@ -15,11 +16,9 @@ import ( "strings" "time" - coreiface "github.com/ipfs/boxo/coreiface" ipath "github.com/ipfs/boxo/coreiface/path" cid "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" - mc "github.com/multiformats/go-multicodec" prometheus "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -64,7 +63,7 @@ type redirectTemplateData struct { // (it serves requests like GET /ipfs/QmVRzPKPzNtSrEzBFm2UZfxmPAgnaLke4DMcerbsGGSaFe/link) type handler struct { config Config - api API + api IPFSBackend // generic metrics firstContentBlockGetMetric *prometheus.HistogramVec @@ -196,11 +195,11 @@ func newHistogramMetric(name string, help string) *prometheus.HistogramVec { // NewHandler returns an http.Handler that can act as a gateway to IPFS content // offlineApi is a version of the API that should not make network requests for missing data -func NewHandler(c Config, api API) http.Handler { +func NewHandler(c Config, api IPFSBackend) http.Handler { return newHandler(c, api) } -func newHandler(c Config, api API) *handler { +func newHandler(c Config, api IPFSBackend) *handler { i := &handler{ config: c, api: api, @@ -346,34 +345,53 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { } trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("ResponseFormat", responseFormat)) - resolvedPath, contentPath, ok := i.handlePathResolution(w, r, responseFormat, contentPath, logger) - if !ok { + i.addUserHeaders(w) // ok, _now_ write user's headers. + w.Header().Set("X-Ipfs-Path", contentPath.String()) + + // TODO: Why did the previous code do path resolution, was that a bug? + // TODO: Does If-None-Match apply here? + if responseFormat == "application/vnd.ipfs.ipns-record" { + logger.Debugw("serving ipns record", "path", contentPath) + success := i.serveIpnsRecord(r.Context(), w, r, contentPath, begin, logger) + if success { + i.getMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + } + return } - trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("ResolvedPath", resolvedPath.String())) - // Detect when If-None-Match HTTP header allows returning HTTP 304 Not Modified - if inm := r.Header.Get("If-None-Match"); inm != "" { - pathCid := resolvedPath.Cid() - // need to check against both File and Dir Etag variants - // because this inexpensive check happens before we do any I/O - cidEtag := getEtag(r, pathCid) - dirEtag := getDirListingEtag(pathCid) - if etagMatch(inm, cidEtag, dirEtag) { - // Finish early if client already has a matching Etag - w.WriteHeader(http.StatusNotModified) + var immutableContentPath ImmutablePath + if contentPath.Mutable() { + immutableContentPath, err = i.api.ResolveMutable(r.Context(), contentPath) + if err != nil { + // Note: webError will replace http.StatusInternalServerError with a more appropriate error (e.g. StatusNotFound, StatusRequestTimeout, StatusServiceUnavailable, etc.) if necessary + err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) + webError(w, err, http.StatusInternalServerError) + return + } + } else { + immutableContentPath, err = NewImmutablePath(contentPath) + if err != nil { + err = fmt.Errorf("path was expected to be immutable, but was not %s: %w", debugStr(contentPath.String()), err) + webError(w, err, http.StatusInternalServerError) return } } - if err := i.handleGettingFirstBlock(r, begin, contentPath, resolvedPath); err != nil { - webRequestError(w, err) + // Detect when If-None-Match HTTP header allows returning HTTP 304 Not Modified + ifNoneMatchResolvedPath, ok := i.handleIfNoneMatch(w, r, responseFormat, contentPath, immutableContentPath, logger) + if !ok { return } - if err := i.setCommonHeaders(w, r, contentPath); err != nil { - webRequestError(w, err) - return + // If we already did the path resolution no need to do it again + maybeResolvedImPath := immutableContentPath + if ifNoneMatchResolvedPath != nil { + maybeResolvedImPath, err = NewImmutablePath(ifNoneMatchResolvedPath) + if err != nil { + webError(w, err, http.StatusInternalServerError) + return + } } var success bool @@ -381,30 +399,21 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { // Support custom response formats passed via ?format or Accept HTTP header switch responseFormat { case "", "application/json", "application/cbor": - switch mc.Code(resolvedPath.Cid().Prefix().Codec) { - case mc.Json, mc.DagJson, mc.Cbor, mc.DagCbor: - logger.Debugw("serving codec", "path", contentPath) - success = i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat) - default: - logger.Debugw("serving unixfs", "path", contentPath) - success = i.serveUnixFS(r.Context(), w, r, resolvedPath, contentPath, begin, logger) - } + success = i.serveDefaults(r.Context(), w, r, maybeResolvedImPath, immutableContentPath, contentPath, begin, responseFormat, logger) case "application/vnd.ipld.raw": logger.Debugw("serving raw block", "path", contentPath) - success = i.serveRawBlock(r.Context(), w, r, resolvedPath, contentPath, begin) + success = i.serveRawBlock(r.Context(), w, r, maybeResolvedImPath, contentPath, begin) case "application/vnd.ipld.car": logger.Debugw("serving car stream", "path", contentPath) carVersion := formatParams["version"] - success = i.serveCAR(r.Context(), w, r, resolvedPath, contentPath, carVersion, begin) + success = i.serveCAR(r.Context(), w, r, maybeResolvedImPath, contentPath, carVersion, begin) case "application/x-tar": logger.Debugw("serving tar file", "path", contentPath) - success = i.serveTAR(r.Context(), w, r, resolvedPath, contentPath, begin, logger) + success = i.serveTAR(r.Context(), w, r, maybeResolvedImPath, contentPath, begin, logger) case "application/vnd.ipld.dag-json", "application/vnd.ipld.dag-cbor": logger.Debugw("serving codec", "path", contentPath) - success = i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat) + success = i.serveCodec(r.Context(), w, r, maybeResolvedImPath, contentPath, begin, responseFormat) case "application/vnd.ipfs.ipns-record": - logger.Debugw("serving ipns record", "path", contentPath) - success = i.serveIpnsRecord(r.Context(), w, r, resolvedPath, contentPath, begin, logger) default: // catch-all for unsuported application/vnd.* err := fmt.Errorf("unsupported format %q", responseFormat) webError(w, err, http.StatusBadRequest) @@ -492,7 +501,7 @@ func setContentDispositionHeader(w http.ResponseWriter, filename string, disposi } // Set X-Ipfs-Roots with logical CID array for efficient HTTP cache invalidation. -func (i *handler) buildIpfsRootsHeader(contentPath string, r *http.Request) (string, error) { +func (i *handler) setIpfsRootsHeader(w http.ResponseWriter, pathMetadata ContentPathMetadata) *ErrorResponse { /* These are logical roots where each CID represent one path segment and resolves to either a directory or the root block of a file. @@ -515,24 +524,16 @@ func (i *handler) buildIpfsRootsHeader(contentPath string, r *http.Request) (str Note that while the top one will change every time any article is changed, the last root (responsible for specific article) may not change at all. */ - var sp strings.Builder + var pathRoots []string - pathSegments := strings.Split(contentPath[6:], "/") - sp.WriteString(contentPath[:5]) // /ipfs or /ipns - for _, root := range pathSegments { - if root == "" { - continue - } - sp.WriteString("/") - sp.WriteString(root) - resolvedSubPath, err := i.api.ResolvePath(r.Context(), ipath.New(sp.String())) - if err != nil { - return "", err - } - pathRoots = append(pathRoots, resolvedSubPath.Cid().String()) + for _, c := range pathMetadata.PathSegmentRoots { + pathRoots = append(pathRoots, c.String()) } + pathRoots = append(pathRoots, pathMetadata.LastSegment.Cid().String()) rootCidList := strings.Join(pathRoots, ",") // convention from rfc2616#sec4.2 - return rootCidList, nil + + w.Header().Set("X-Ipfs-Roots", rootCidList) + return nil } func getFilename(contentPath ipath.Path) string { @@ -674,6 +675,13 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string] return "", nil, nil } +// check if request was for one of known explicit formats, +// or should use the default, implicit Web+UnixFS behaviors. +func isWebRequest(responseFormat string) bool { + // The implicit response format is "" + return responseFormat == "" +} + // returns unquoted path with all special characters revealed as \u codes func debugStr(path string) string { q := fmt.Sprintf("%+q", path) @@ -683,48 +691,83 @@ func debugStr(path string) string { return q } -// Resolve the provided contentPath including any special handling related to -// the requested responseFormat. Returned ok flag indicates if gateway handler -// should continue processing the request. -func (i *handler) handlePathResolution(w http.ResponseWriter, r *http.Request, responseFormat string, contentPath ipath.Path, logger *zap.SugaredLogger) (resolvedPath ipath.Resolved, newContentPath ipath.Path, ok bool) { - // Attempt to resolve the provided path. - resolvedPath, err := i.api.ResolvePath(r.Context(), contentPath) +func (i *handler) handleIfNoneMatch(w http.ResponseWriter, r *http.Request, responseFormat string, contentPath ipath.Path, imPath ImmutablePath, logger *zap.SugaredLogger) (ipath.Resolved, bool) { + // Detect when If-None-Match HTTP header allows returning HTTP 304 Not Modified + if inm := r.Header.Get("If-None-Match"); inm != "" { + pathMetadata, err := i.api.ResolvePath(r.Context(), imPath) + if err != nil { + // Note: webError will replace http.StatusInternalServerError with a more appropriate error (e.g. StatusNotFound, StatusRequestTimeout, StatusServiceUnavailable, etc.) if necessary + err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) + webError(w, err, http.StatusInternalServerError) + return nil, false + } + + resolvedPath := pathMetadata.LastSegment + pathCid := resolvedPath.Cid() + // need to check against both File and Dir Etag variants + // because this inexpensive check happens before we do any I/O + cidEtag := getEtag(r, pathCid) + dirEtag := getDirListingEtag(pathCid) + if etagMatch(inm, cidEtag, dirEtag) { + // Finish early if client already has a matching Etag + w.WriteHeader(http.StatusNotModified) + return nil, false + } + + return resolvedPath, true + } + return nil, true +} - switch err { - case nil: - return resolvedPath, contentPath, true - case coreiface.ErrOffline: +// handleRequestErrors is used when request type is other than Web+UnixFS +func (i *handler) handleRequestErrors(w http.ResponseWriter, contentPath ipath.Path, err error) bool { + if err == nil { + return true + } + // Note: webError will replace http.StatusInternalServerError with a more appropriate error (e.g. StatusNotFound, StatusRequestTimeout, StatusServiceUnavailable, etc.) if necessary + err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) + webError(w, err, http.StatusInternalServerError) + return false +} + +// handleWebRequestErrors is used when request type is Web+UnixFS and err could +// be a 404 (Not Found) that should be recovered via _redirects file (IPIP-290) +func (i *handler) handleWebRequestErrors(w http.ResponseWriter, r *http.Request, maybeResolvedImPath, immutableContentPath ImmutablePath, contentPath ipath.Path, err error, logger *zap.SugaredLogger) (ImmutablePath, bool) { + if err == nil { + return maybeResolvedImPath, true + } + + if errors.Is(err, ErrServiceUnavailable) { err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) webError(w, err, http.StatusServiceUnavailable) - return nil, nil, false - default: - // The path can't be resolved. - if isUnixfsResponseFormat(responseFormat) { - // If we have origin isolation (subdomain gw, DNSLink website), - // and response type is UnixFS (default for website hosting) - // check for presence of _redirects file and apply rules defined there. - // See: https://github.com/ipfs/specs/pull/290 - if hasOriginIsolation(r) { - resolvedPath, newContentPath, ok, hadMatchingRule := i.serveRedirectsIfPresent(w, r, resolvedPath, contentPath, logger) - if hadMatchingRule { - logger.Debugw("applied a rule from _redirects file") - return resolvedPath, newContentPath, ok - } - } - - // if Accept is text/html, see if ipfs-404.html is present - // This logic isn't documented and will likely be removed at some point. - // Any 404 logic in _redirects above will have already run by this time, so it's really an extra fall back - if i.serveLegacy404IfPresent(w, r, contentPath) { - logger.Debugw("served legacy 404") - return nil, nil, false - } + return ImmutablePath{}, false + } + + // If we have origin isolation (subdomain gw, DNSLink website), + // and response type is UnixFS (default for website hosting) + // we can leverage the presence of an _redirects file and apply rules defined there. + // See: https://github.com/ipfs/specs/pull/290 + if hasOriginIsolation(r) { + newContentPath, ok, hadMatchingRule := i.serveRedirectsIfPresent(w, r, maybeResolvedImPath, immutableContentPath, contentPath, logger) + if hadMatchingRule { + logger.Debugw("applied a rule from _redirects file") + return newContentPath, ok } + } - err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) - webError(w, err, http.StatusInternalServerError) - return nil, nil, false + // if Accept is text/html, see if ipfs-404.html is present + // This logic isn't documented and will likely be removed at some point. + // Any 404 logic in _redirects above will have already run by this time, so it's really an extra fall back + // PLEASE do not use this for new websites, + // follow https://docs.ipfs.tech/how-to/websites-on-ipfs/redirects-and-custom-404s/ instead. + if i.serveLegacy404IfPresent(w, r, immutableContentPath) { + logger.Debugw("served legacy 404") + return ImmutablePath{}, false } + + err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) + webError(w, err, http.StatusInternalServerError) + return ImmutablePath{}, false } // Detect 'Cache-Control: only-if-cached' in request and return data if it is already in the local datastore. @@ -845,35 +888,6 @@ func handleSuperfluousNamespace(w http.ResponseWriter, r *http.Request, contentP return true } -func (i *handler) handleGettingFirstBlock(r *http.Request, begin time.Time, contentPath ipath.Path, resolvedPath ipath.Resolved) *ErrorResponse { - // Update the global metric of the time it takes to read the final root block of the requested resource - // NOTE: for legacy reasons this happens before we go into content-type specific code paths - _, err := i.api.GetBlock(r.Context(), resolvedPath.Cid()) - if err != nil { - err = fmt.Errorf("could not get block %s: %w", resolvedPath.Cid().String(), err) - return NewErrorResponse(err, http.StatusInternalServerError) - } - ns := contentPath.Namespace() - timeToGetFirstContentBlock := time.Since(begin).Seconds() - i.unixfsGetMetric.WithLabelValues(ns).Observe(timeToGetFirstContentBlock) // deprecated, use firstContentBlockGetMetric instead - i.firstContentBlockGetMetric.WithLabelValues(ns).Observe(timeToGetFirstContentBlock) - return nil -} - -func (i *handler) setCommonHeaders(w http.ResponseWriter, r *http.Request, contentPath ipath.Path) *ErrorResponse { - i.addUserHeaders(w) // ok, _now_ write user's headers. - w.Header().Set("X-Ipfs-Path", contentPath.String()) - - if rootCids, err := i.buildIpfsRootsHeader(contentPath.String(), r); err == nil { - w.Header().Set("X-Ipfs-Roots", rootCids) - } else { // this should never happen, as we resolved the contentPath already - err = fmt.Errorf("error while resolving X-Ipfs-Roots: %w", err) - return NewErrorResponse(err, http.StatusInternalServerError) - } - - return nil -} - // spanTrace starts a new span using the standard IPFS tracing conventions. func spanTrace(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { return otel.Tracer("boxo").Start(ctx, fmt.Sprintf("%s.%s", " Gateway", spanName), opts...) diff --git a/gateway/handler_block.go b/gateway/handler_block.go index 773088c17..47d74fe9e 100644 --- a/gateway/handler_block.go +++ b/gateway/handler_block.go @@ -1,9 +1,7 @@ package gateway import ( - "bytes" "context" - "fmt" "net/http" "time" @@ -13,18 +11,22 @@ import ( ) // serveRawBlock returns bytes behind a raw block -func (i *handler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time) bool { - ctx, span := spanTrace(ctx, "ServeRawBlock", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) +func (i *handler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *http.Request, imPath ImmutablePath, contentPath ipath.Path, begin time.Time) bool { + ctx, span := spanTrace(ctx, "ServeRawBlock", trace.WithAttributes(attribute.String("path", imPath.String()))) defer span.End() - blockCid := resolvedPath.Cid() - block, err := i.api.GetBlock(ctx, blockCid) - if err != nil { - err = fmt.Errorf("error getting block %s: %w", blockCid.String(), err) - webError(w, err, http.StatusInternalServerError) + pathMetadata, data, err := i.api.GetBlock(ctx, imPath) + if !i.handleRequestErrors(w, contentPath, err) { return false } - content := bytes.NewReader(block.RawData()) + defer data.Close() + + if err := i.setIpfsRootsHeader(w, pathMetadata); err != nil { + webRequestError(w, err) + return false + } + + blockCid := pathMetadata.LastSegment.Cid() // Set Content-Disposition var name string @@ -42,7 +44,7 @@ func (i *handler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *h // ServeContent will take care of // If-None-Match+Etag, Content-Length and range requests - _, dataSent, _ := ServeContent(w, r, name, modtime, content) + _, dataSent, _ := ServeContent(w, r, name, modtime, data) if dataSent { // Update metrics diff --git a/gateway/handler_car.go b/gateway/handler_car.go index 6ebc4e675..0d51cfe47 100644 --- a/gateway/handler_car.go +++ b/gateway/handler_car.go @@ -3,21 +3,19 @@ package gateway import ( "context" "fmt" + "io" "net/http" "time" ipath "github.com/ipfs/boxo/coreiface/path" - gocar "github.com/ipfs/boxo/ipld/car" - blocks "github.com/ipfs/go-block-format" - cid "github.com/ipfs/go-cid" - selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" + "go.uber.org/multierr" ) // serveCAR returns a CAR stream for specific DAG+selector -func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, carVersion string, begin time.Time) bool { - ctx, span := spanTrace(ctx, "ServeCAR", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) +func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.Request, imPath ImmutablePath, contentPath ipath.Path, carVersion string, begin time.Time) bool { + ctx, span := spanTrace(ctx, "ServeCAR", trace.WithAttributes(attribute.String("path", imPath.String()))) defer span.End() ctx, cancel := context.WithCancel(ctx) @@ -31,7 +29,19 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R webError(w, err, http.StatusBadRequest) return false } - rootCid := resolvedPath.Cid() + + pathMetadata, carFile, errCh, err := i.api.GetCAR(ctx, imPath) + if !i.handleRequestErrors(w, contentPath, err) { + return false + } + defer carFile.Close() + + if err := i.setIpfsRootsHeader(w, pathMetadata); err != nil { + webRequestError(w, err) + return false + } + + rootCid := pathMetadata.LastSegment.Cid() // Set Content-Disposition var name string @@ -67,19 +77,14 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R w.Header().Set("Content-Type", "application/vnd.ipld.car; version=1") w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^) - // Same go-car settings as dag.export command - store := dagStore{api: i.api, ctx: ctx} - - // TODO: support selectors passed as request param: https://github.com/ipfs/kubo/issues/8769 - dag := gocar.Dag{Root: rootCid, Selector: selectorparse.CommonSelector_ExploreAllRecursively} - car := gocar.NewSelectiveCar(ctx, store, []gocar.Dag{dag}, gocar.TraverseLinksOnlyOnce()) - - if err := car.Write(w); err != nil { + _, copyErr := io.Copy(w, carFile) + carErr := <-errCh + if copyErr != nil || carErr != nil { // We return error as a trailer, however it is not something browsers can access // (https://github.com/mdn/browser-compat-data/issues/14703) // Due to this, we suggest client always verify that // the received CAR stream response is matching requested DAG selector - w.Header().Set("X-Stream-Error", err.Error()) + w.Header().Set("X-Stream-Error", multierr.Combine(err, copyErr).Error()) return false } @@ -87,13 +92,3 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R i.carStreamGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) return true } - -// FIXME(@Jorropo): https://github.com/ipld/go-car/issues/315 -type dagStore struct { - api API - ctx context.Context -} - -func (ds dagStore) Get(_ context.Context, c cid.Cid) (blocks.Block, error) { - return ds.api.GetBlock(ds.ctx, c) -} diff --git a/gateway/handler_codec.go b/gateway/handler_codec.go index e2dcb1d87..c301de724 100644 --- a/gateway/handler_codec.go +++ b/gateway/handler_codec.go @@ -4,13 +4,14 @@ import ( "bytes" "context" "fmt" + "io" "net/http" "strings" "time" ipath "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/boxo/gateway/assets" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/multicodec" "github.com/ipld/go-ipld-prime/node/basicnode" mc "github.com/multiformats/go-multicodec" @@ -56,11 +57,31 @@ var contentTypeToExtension = map[string]string{ "application/vnd.ipld.dag-cbor": ".cbor", } -func (i *handler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, requestedContentType string) bool { - ctx, span := spanTrace(ctx, "ServeCodec", trace.WithAttributes(attribute.String("path", resolvedPath.String()), attribute.String("requestedContentType", requestedContentType))) +func (i *handler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, imPath ImmutablePath, contentPath ipath.Path, begin time.Time, requestedContentType string) bool { + ctx, span := spanTrace(ctx, "ServeCodec", trace.WithAttributes(attribute.String("path", imPath.String()), attribute.String("requestedContentType", requestedContentType))) defer span.End() - cidCodec := mc.Code(resolvedPath.Cid().Prefix().Codec) + pathMetadata, data, err := i.api.GetBlock(ctx, imPath) + if !i.handleRequestErrors(w, contentPath, err) { + return false + } + defer data.Close() + + if err := i.setIpfsRootsHeader(w, pathMetadata); err != nil { + webRequestError(w, err) + return false + } + + resolvedPath := pathMetadata.LastSegment + return i.renderCodec(ctx, w, r, resolvedPath, data, contentPath, begin, requestedContentType) +} + +func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, blockData io.ReadSeekCloser, contentPath ipath.Path, begin time.Time, requestedContentType string) bool { + ctx, span := spanTrace(ctx, "RenderCodec", trace.WithAttributes(attribute.String("path", resolvedPath.String()), attribute.String("requestedContentType", requestedContentType))) + defer span.End() + + blockCid := resolvedPath.Cid() + cidCodec := mc.Code(blockCid.Prefix().Codec) responseContentType := requestedContentType // If the resolved path still has some remainder, return error for now. @@ -103,7 +124,7 @@ func (i *handler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http } else { // This covers CIDs with codec 'json' and 'cbor' as those do not have // an explicit requested content type. - return i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, name, modtime, begin) + return i.serveCodecRaw(ctx, w, r, blockData, contentPath, name, modtime, begin) } } @@ -113,7 +134,7 @@ func (i *handler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http if ok { for _, skipCodec := range skipCodecs { if skipCodec == cidCodec { - return i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, name, modtime, begin) + return i.serveCodecRaw(ctx, w, r, blockData, contentPath, name, modtime, begin) } } } @@ -129,7 +150,7 @@ func (i *handler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http } // This handles DAG-* conversions and validations. - return i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, toCodec, modtime, begin) + return i.serveCodecConverted(ctx, w, r, blockCid, blockData, contentPath, toCodec, modtime, begin) } func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path) bool { @@ -165,19 +186,10 @@ func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r * } // serveCodecRaw returns the raw block without any conversion -func (i *handler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, name string, modtime, begin time.Time) bool { - blockCid := resolvedPath.Cid() - block, err := i.api.GetBlock(ctx, blockCid) - if err != nil { - err = fmt.Errorf("error getting block %s: %w", blockCid.String(), err) - webError(w, err, http.StatusInternalServerError) - return false - } - content := bytes.NewReader(block.RawData()) - +func (i *handler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, blockData io.ReadSeekCloser, contentPath ipath.Path, name string, modtime, begin time.Time) bool { // ServeContent will take care of // If-None-Match+Etag, Content-Length and range requests - _, dataSent, _ := ServeContent(w, r, name, modtime, content) + _, dataSent, _ := ServeContent(w, r, name, modtime, blockData) if dataSent { // Update metrics @@ -188,15 +200,7 @@ func (i *handler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *h } // serveCodecConverted returns payload converted to codec specified in toCodec -func (i *handler) serveCodecConverted(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, toCodec mc.Code, modtime, begin time.Time) bool { - blockCid := resolvedPath.Cid() - block, err := i.api.GetBlock(ctx, blockCid) - if err != nil { - err = fmt.Errorf("error getting block %s: %w", blockCid.String(), err) - webError(w, err, http.StatusInternalServerError) - return false - } - +func (i *handler) serveCodecConverted(ctx context.Context, w http.ResponseWriter, r *http.Request, blockCid cid.Cid, blockData io.ReadSeekCloser, contentPath ipath.Path, toCodec mc.Code, modtime, begin time.Time) bool { codec := blockCid.Prefix().Codec decoder, err := multicodec.LookupDecoder(codec) if err != nil { @@ -205,7 +209,7 @@ func (i *handler) serveCodecConverted(ctx context.Context, w http.ResponseWriter } node := basicnode.Prototype.Any.NewBuilder() - err = decoder(node, bytes.NewReader(block.RawData())) + err = decoder(node, blockData) if err != nil { webError(w, err, http.StatusInternalServerError) return false diff --git a/gateway/handler_defaults.go b/gateway/handler_defaults.go new file mode 100644 index 000000000..205ac6064 --- /dev/null +++ b/gateway/handler_defaults.go @@ -0,0 +1,207 @@ +package gateway + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/textproto" + "strconv" + "strings" + "time" + + "github.com/ipfs/boxo/files" + mc "github.com/multiformats/go-multicodec" + + ipath "github.com/ipfs/boxo/coreiface/path" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" +) + +func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *http.Request, maybeResolvedImPath ImmutablePath, immutableContentPath ImmutablePath, contentPath ipath.Path, begin time.Time, requestedContentType string, logger *zap.SugaredLogger) bool { + ctx, span := spanTrace(ctx, "ServeDefaults", trace.WithAttributes(attribute.String("path", contentPath.String()))) + defer span.End() + + var ( + pathMetadata ContentPathMetadata + bytesResponse files.File + isDirectoryHeadRequest bool + directoryMetadata *directoryMetadata + err error + ) + + switch r.Method { + case http.MethodHead: + var data files.Node + pathMetadata, data, err = i.api.Head(ctx, maybeResolvedImPath) + if !i.handleRequestErrors(w, contentPath, err) { + return false + } + defer data.Close() + if _, ok := data.(files.Directory); ok { + isDirectoryHeadRequest = true + } else if f, ok := data.(files.File); ok { + bytesResponse = f + } else { + webError(w, fmt.Errorf("unsupported response type"), http.StatusInternalServerError) + return false + } + case http.MethodGet: + // TODO: refactor below: we should not have 2x20 duplicated flow control when the only difference is ranges. + rangeHeader := r.Header.Get("Range") + if rangeHeader == "" { + var getResp *GetResponse + // TODO: passing resolved path here, instead of contentPath is harming content routing. Knowing original immutableContentPath will allow backend to find providers for parents, even when internal CIDs are not announced, and will provide better key for caching related DAGs. + pathMetadata, getResp, err = i.api.Get(ctx, maybeResolvedImPath) + if err != nil { + if isWebRequest(requestedContentType) { + forwardedPath, continueProcessing := i.handleWebRequestErrors(w, r, maybeResolvedImPath, immutableContentPath, contentPath, err, logger) + if !continueProcessing { + return false + } + pathMetadata, getResp, err = i.api.Get(ctx, forwardedPath) + if err != nil { + err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) + webError(w, err, http.StatusInternalServerError) + } + } else { + if !i.handleRequestErrors(w, contentPath, err) { + return false + } + } + } + if getResp.bytes != nil { + bytesResponse = getResp.bytes + defer bytesResponse.Close() + } else { + directoryMetadata = getResp.directoryMetadata + } + } else { + // TODO: Add tests for range parsing + var ranges []GetRange + ranges, err = parseRange(rangeHeader) + if err != nil { + webError(w, fmt.Errorf("invalid range request: %w", err), http.StatusBadRequest) + return false + } + pathMetadata, bytesResponse, err = i.api.GetRange(ctx, maybeResolvedImPath, ranges...) + if err != nil { + if isWebRequest(requestedContentType) { + forwardedPath, continueProcessing := i.handleWebRequestErrors(w, r, maybeResolvedImPath, immutableContentPath, contentPath, err, logger) + if !continueProcessing { + return false + } + pathMetadata, bytesResponse, err = i.api.GetRange(ctx, forwardedPath, ranges...) + if err != nil { + err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) + webError(w, err, http.StatusInternalServerError) + } + } else { + if !i.handleRequestErrors(w, contentPath, err) { + return false + } + } + } + defer bytesResponse.Close() + } + default: + // This shouldn't be possible to reach which is why it is a 500 rather than 4XX error + webError(w, fmt.Errorf("invalid method: cannot use this HTTP method with the given request"), http.StatusInternalServerError) + return false + } + + // TODO: check if we have a bug when maybeResolvedImPath is resolved and i.setIpfsRootsHeader works with pathMetadata returned by GetRange(maybeResolvedImPath) + if err := i.setIpfsRootsHeader(w, pathMetadata); err != nil { + webRequestError(w, err) + return false + } + + resolvedPath := pathMetadata.LastSegment + switch mc.Code(resolvedPath.Cid().Prefix().Codec) { + case mc.Json, mc.DagJson, mc.Cbor, mc.DagCbor: + if bytesResponse == nil { // This should never happen + webError(w, fmt.Errorf("decoding error: data not usable as a file"), http.StatusInternalServerError) + return false + } + logger.Debugw("serving codec", "path", contentPath) + return i.renderCodec(r.Context(), w, r, resolvedPath, bytesResponse, contentPath, begin, requestedContentType) + default: + logger.Debugw("serving unixfs", "path", contentPath) + ctx, span := spanTrace(ctx, "ServeUnixFS", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) + defer span.End() + + // Handling Unixfs file + if bytesResponse != nil { + logger.Debugw("serving unixfs file", "path", contentPath) + return i.serveFile(ctx, w, r, resolvedPath, contentPath, bytesResponse, pathMetadata.ContentType, begin) + } + + // Handling Unixfs directory + if directoryMetadata != nil || isDirectoryHeadRequest { + logger.Debugw("serving unixfs directory", "path", contentPath) + return i.serveDirectory(ctx, w, r, resolvedPath, contentPath, isDirectoryHeadRequest, directoryMetadata, begin, logger) + } + + webError(w, fmt.Errorf("unsupported UnixFS type"), http.StatusInternalServerError) + return false + } +} + +// parseRange parses a Range header string as per RFC 7233. +func parseRange(s string) ([]GetRange, error) { + if s == "" { + return nil, nil // header not present + } + const b = "bytes=" + if !strings.HasPrefix(s, b) { + return nil, errors.New("invalid range") + } + var ranges []GetRange + for _, ra := range strings.Split(s[len(b):], ",") { + ra = textproto.TrimString(ra) + if ra == "" { + continue + } + start, end, ok := strings.Cut(ra, "-") + if !ok { + return nil, errors.New("invalid range") + } + start, end = textproto.TrimString(start), textproto.TrimString(end) + var r GetRange + if start == "" { + r.From = 0 + // If no start is specified, end specifies the + // range start relative to the end of the file, + // and we are dealing with + // which has to be a non-negative integer as per + // RFC 7233 Section 2.1 "Byte-Ranges". + if end == "" || end[0] == '-' { + return nil, errors.New("invalid range") + } + i, err := strconv.ParseInt(end, 10, 64) + if i < 0 || err != nil { + return nil, errors.New("invalid range") + } + r.To = &i + } else { + i, err := strconv.ParseUint(start, 10, 64) + if err != nil { + return nil, errors.New("invalid range") + } + r.From = i + if end == "" { + // If no end is specified, range extends to end of the file. + r.To = nil + } else { + i, err := strconv.ParseInt(end, 10, 64) + if err != nil || i < 0 || r.From > uint64(i) { + return nil, errors.New("invalid range") + } + r.To = &i + } + } + ranges = append(ranges, r) + } + return ranges, nil +} diff --git a/gateway/handler_ipns_record.go b/gateway/handler_ipns_record.go index a1487f0c8..b6c010f65 100644 --- a/gateway/handler_ipns_record.go +++ b/gateway/handler_ipns_record.go @@ -5,9 +5,11 @@ import ( "errors" "fmt" "net/http" + "strconv" "strings" "time" + "github.com/cespare/xxhash" "github.com/gogo/protobuf/proto" ipath "github.com/ipfs/boxo/coreiface/path" ipns_pb "github.com/ipfs/boxo/ipns/pb" @@ -17,8 +19,8 @@ import ( "go.uber.org/zap" ) -func (i *handler) serveIpnsRecord(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) bool { - ctx, span := spanTrace(ctx, "ServeIPNSRecord", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) +func (i *handler) serveIpnsRecord(ctx context.Context, w http.ResponseWriter, r *http.Request, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) bool { + ctx, span := spanTrace(ctx, "ServeIPNSRecord", trace.WithAttributes(attribute.String("path", contentPath.String()))) defer span.End() if contentPath.Namespace() != "ipns" { @@ -59,7 +61,8 @@ func (i *handler) serveIpnsRecord(ctx context.Context, w http.ResponseWriter, r // TTL is not present, we use the Last-Modified tag. We are tracking IPNS // caching on: https://github.com/ipfs/kubo/issues/1818. // TODO: use addCacheControlHeaders once #1818 is fixed. - w.Header().Set("Etag", getEtag(r, resolvedPath.Cid())) + recordEtag := strconv.FormatUint(xxhash.Sum64(rawRecord), 32) + w.Header().Set("Etag", recordEtag) if record.Ttl != nil { seconds := int(time.Duration(*record.Ttl).Seconds()) w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", seconds)) diff --git a/gateway/handler_tar.go b/gateway/handler_tar.go index 7d835bb33..551e50031 100644 --- a/gateway/handler_tar.go +++ b/gateway/handler_tar.go @@ -3,7 +3,6 @@ package gateway import ( "context" "fmt" - "html" "net/http" "time" @@ -16,23 +15,25 @@ import ( var unixEpochTime = time.Unix(0, 0) -func (i *handler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) bool { - ctx, span := spanTrace(ctx, "ServeTAR", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) +func (i *handler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.Request, imPath ImmutablePath, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) bool { + ctx, span := spanTrace(ctx, "ServeTAR", trace.WithAttributes(attribute.String("path", imPath.String()))) defer span.End() ctx, cancel := context.WithCancel(ctx) defer cancel() - // Get Unixfs file - file, err := i.api.GetUnixFsNode(ctx, resolvedPath) - if err != nil { - err = fmt.Errorf("error getting UnixFS node for %s: %w", html.EscapeString(contentPath.String()), err) - webError(w, err, http.StatusInternalServerError) + // Get Unixfs file (or directory) + pathMetadata, file, err := i.api.GetAll(ctx, imPath) + if !i.handleRequestErrors(w, contentPath, err) { return false } defer file.Close() - rootCid := resolvedPath.Cid() + if err := i.setIpfsRootsHeader(w, pathMetadata); err != nil { + webRequestError(w, err) + return false + } + rootCid := pathMetadata.LastSegment.Cid() // Set Cache-Control and read optional Last-Modified time modtime := addCacheControlHeaders(w, r, contentPath, rootCid) diff --git a/gateway/handler_test.go b/gateway/handler_test.go index af881b1cf..0d158042d 100644 --- a/gateway/handler_test.go +++ b/gateway/handler_test.go @@ -4,16 +4,15 @@ import ( "context" "errors" "fmt" + "io" "net/http" "testing" "time" - iface "github.com/ipfs/boxo/coreiface" ipath "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/path/resolver" - "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" "github.com/stretchr/testify/assert" @@ -46,16 +45,32 @@ type errorMockAPI struct { err error } -func (api *errorMockAPI) GetUnixFsNode(context.Context, ipath.Resolved) (files.Node, error) { - return nil, api.err +func (api *errorMockAPI) Get(ctx context.Context, path ImmutablePath) (ContentPathMetadata, *GetResponse, error) { + return ContentPathMetadata{}, nil, api.err } -func (api *errorMockAPI) LsUnixFsDir(ctx context.Context, p ipath.Resolved) (<-chan iface.DirEntry, error) { - return nil, api.err +func (api *errorMockAPI) GetRange(ctx context.Context, path ImmutablePath, getRange ...GetRange) (ContentPathMetadata, files.File, error) { + return ContentPathMetadata{}, nil, api.err } -func (api *errorMockAPI) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { - return nil, api.err +func (api *errorMockAPI) GetAll(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.Node, error) { + return ContentPathMetadata{}, nil, api.err +} + +func (api *errorMockAPI) GetBlock(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.File, error) { + return ContentPathMetadata{}, nil, api.err +} + +func (api *errorMockAPI) Head(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.Node, error) { + return ContentPathMetadata{}, nil, api.err +} + +func (api *errorMockAPI) GetCAR(ctx context.Context, path ImmutablePath) (ContentPathMetadata, io.ReadCloser, <-chan error, error) { + return ContentPathMetadata{}, nil, nil, api.err +} + +func (api *errorMockAPI) ResolveMutable(ctx context.Context, path ipath.Path) (ImmutablePath, error) { + return ImmutablePath{}, api.err } func (api *errorMockAPI) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { @@ -70,8 +85,8 @@ func (api *errorMockAPI) IsCached(ctx context.Context, p ipath.Path) bool { return false } -func (api *errorMockAPI) ResolvePath(ctx context.Context, ip ipath.Path) (ipath.Resolved, error) { - return nil, api.err +func (api *errorMockAPI) ResolvePath(ctx context.Context, path ImmutablePath) (ContentPathMetadata, error) { + return ContentPathMetadata{}, api.err } func TestGatewayBadRequestInvalidPath(t *testing.T) { @@ -146,15 +161,31 @@ type panicMockAPI struct { panicOnHostnameHandler bool } -func (api *panicMockAPI) GetUnixFsNode(context.Context, ipath.Resolved) (files.Node, error) { +func (api *panicMockAPI) Get(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, *GetResponse, error) { + panic("i am panicking") +} + +func (api *panicMockAPI) GetRange(ctx context.Context, immutablePath ImmutablePath, ranges ...GetRange) (ContentPathMetadata, files.File, error) { + panic("i am panicking") +} + +func (api *panicMockAPI) GetAll(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.Node, error) { + panic("i am panicking") +} + +func (api *panicMockAPI) GetBlock(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.File, error) { + panic("i am panicking") +} + +func (api *panicMockAPI) Head(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.Node, error) { panic("i am panicking") } -func (api *panicMockAPI) LsUnixFsDir(ctx context.Context, p ipath.Resolved) (<-chan iface.DirEntry, error) { +func (api *panicMockAPI) GetCAR(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, io.ReadCloser, <-chan error, error) { panic("i am panicking") } -func (api *panicMockAPI) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { +func (api *panicMockAPI) ResolveMutable(ctx context.Context, p ipath.Path) (ImmutablePath, error) { panic("i am panicking") } @@ -177,7 +208,7 @@ func (api *panicMockAPI) IsCached(ctx context.Context, p ipath.Path) bool { panic("i am panicking") } -func (api *panicMockAPI) ResolvePath(ctx context.Context, ip ipath.Path) (ipath.Resolved, error) { +func (api *panicMockAPI) ResolvePath(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, error) { panic("i am panicking") } diff --git a/gateway/handler_unixfs.go b/gateway/handler_unixfs.go deleted file mode 100644 index f5a96b146..000000000 --- a/gateway/handler_unixfs.go +++ /dev/null @@ -1,44 +0,0 @@ -package gateway - -import ( - "context" - "fmt" - "net/http" - "time" - - ipath "github.com/ipfs/boxo/coreiface/path" - "github.com/ipfs/boxo/files" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - "go.uber.org/zap" -) - -func (i *handler) serveUnixFS(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) bool { - ctx, span := spanTrace(ctx, "ServeUnixFS", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) - defer span.End() - - // Handling UnixFS - dr, err := i.api.GetUnixFsNode(ctx, resolvedPath) - if err != nil { - err = fmt.Errorf("error while getting UnixFS node: %w", err) - webError(w, err, http.StatusInternalServerError) - return false - } - defer dr.Close() - - // Handling Unixfs file - if f, ok := dr.(files.File); ok { - logger.Debugw("serving unixfs file", "path", contentPath) - return i.serveFile(ctx, w, r, resolvedPath, contentPath, f, begin) - } - - // Handling Unixfs directory - dir, ok := dr.(files.Directory) - if !ok { - webError(w, fmt.Errorf("unsupported UnixFS type"), http.StatusInternalServerError) - return false - } - - logger.Debugw("serving unixfs directory", "path", contentPath) - return i.serveDirectory(ctx, w, r, resolvedPath, contentPath, dir, begin, logger) -} diff --git a/gateway/handler_unixfs__redirects.go b/gateway/handler_unixfs__redirects.go index afd0c7c3c..3747d85d6 100644 --- a/gateway/handler_unixfs__redirects.go +++ b/gateway/handler_unixfs__redirects.go @@ -8,10 +8,12 @@ import ( "strconv" "strings" + "go.uber.org/zap" + ipath "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/boxo/files" + redirects "github.com/ipfs/go-ipfs-redirects-file" - "go.uber.org/zap" ) // Resolving a UnixFS path involves determining if the provided `path.Path` exists and returning the `path.Resolved` @@ -36,46 +38,58 @@ import ( // // Note that for security reasons, redirect rules are only processed when the request has origin isolation. // See https://github.com/ipfs/specs/pull/290 for more information. -func (i *handler) serveRedirectsIfPresent(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, logger *zap.SugaredLogger) (newResolvedPath ipath.Resolved, newContentPath ipath.Path, continueProcessing bool, hadMatchingRule bool) { - redirectsFile := i.getRedirectsFile(r, contentPath, logger) - if redirectsFile != nil { - redirectRules, err := i.getRedirectRules(r, redirectsFile) - if err != nil { - webError(w, err, http.StatusInternalServerError) - return nil, nil, false, true - } +func (i *handler) serveRedirectsIfPresent(w http.ResponseWriter, r *http.Request, maybeResolvedImPath, immutableContentPath ImmutablePath, contentPath ipath.Path, logger *zap.SugaredLogger) (newContentPath ImmutablePath, continueProcessing bool, hadMatchingRule bool) { + // contentPath is the full ipfs path to the requested resource, + // regardless of whether path or subdomain resolution is used. + rootPath := getRootPath(immutableContentPath) + redirectsPath := ipath.Join(rootPath, "_redirects") + imRedirectsPath, err := NewImmutablePath(redirectsPath) + if err != nil { + err = fmt.Errorf("trouble processing _redirects path %q: %w", redirectsPath, err) + webError(w, err, http.StatusInternalServerError) + return ImmutablePath{}, false, true + } - redirected, newPath, err := i.handleRedirectsFileRules(w, r, contentPath, redirectRules) + foundRedirect, redirectRules, err := i.getRedirectRules(r, imRedirectsPath) + if err != nil { + err = fmt.Errorf("trouble processing _redirects file at %q: %w", redirectsPath, err) + webError(w, err, http.StatusInternalServerError) + return ImmutablePath{}, false, true + } + + if foundRedirect { + redirected, newPath, err := i.handleRedirectsFileRules(w, r, immutableContentPath, contentPath, redirectRules) if err != nil { - err = fmt.Errorf("trouble processing _redirects file at %q: %w", redirectsFile.String(), err) + err = fmt.Errorf("trouble processing _redirects file at %q: %w", redirectsPath, err) webError(w, err, http.StatusInternalServerError) - return nil, nil, false, true + return ImmutablePath{}, false, true } if redirected { - return nil, nil, false, true + return ImmutablePath{}, false, true } // 200 is treated as a rewrite, so update the path and continue if newPath != "" { // Reassign contentPath and resolvedPath since the URL was rewritten - contentPath = ipath.New(newPath) - resolvedPath, err = i.api.ResolvePath(r.Context(), contentPath) + p := ipath.New(newPath) + imPath, err := NewImmutablePath(p) if err != nil { + err = fmt.Errorf("could not use _redirects file to %q: %w", p, err) webError(w, err, http.StatusInternalServerError) - return nil, nil, false, true + return ImmutablePath{}, false, true } - - return resolvedPath, contentPath, true, true + return imPath, true, true } } + // No matching rule, paths remain the same, continue regular processing - return resolvedPath, contentPath, true, false + return maybeResolvedImPath, true, false } -func (i *handler) handleRedirectsFileRules(w http.ResponseWriter, r *http.Request, contentPath ipath.Path, redirectRules []redirects.Rule) (redirected bool, newContentPath string, err error) { +func (i *handler) handleRedirectsFileRules(w http.ResponseWriter, r *http.Request, immutableContentPath ImmutablePath, cPath ipath.Path, redirectRules []redirects.Rule) (redirected bool, newContentPath string, err error) { // Attempt to match a rule to the URL path, and perform the corresponding redirect or rewrite - pathParts := strings.Split(contentPath.String(), "/") + pathParts := strings.Split(immutableContentPath.String(), "/") if len(pathParts) > 3 { // All paths should start with /ipfs/cid/, so get the path after that urlPath := "/" + strings.Join(pathParts[3:], "/") @@ -101,8 +115,22 @@ func (i *handler) handleRedirectsFileRules(w http.ResponseWriter, r *http.Reques // Or 4xx if rule.Status == 404 || rule.Status == 410 || rule.Status == 451 { toPath := rootPath + rule.To - content4xxPath := ipath.New(toPath) - err := i.serve4xx(w, r, content4xxPath, rule.Status) + imContent4xxPath, err := NewImmutablePath(ipath.New(toPath)) + if err != nil { + return true, toPath, err + } + + // While we have the immutable path which is enough to fetch the data we need to track mutability for + // headers. + contentPathParts := strings.Split(cPath.String(), "/") + if len(contentPathParts) <= 3 { + // Match behavior as with the immutable path + return false, "", nil + } + // All paths should start with /ip(f|n)s//, so get the path after that + contentRootPath := strings.Join(contentPathParts[:3], "/") + content4xxPath := ipath.New(contentRootPath + rule.To) + err = i.serve4xx(w, r, imContent4xxPath, content4xxPath, rule.Status) return true, toPath, err } @@ -118,44 +146,33 @@ func (i *handler) handleRedirectsFileRules(w http.ResponseWriter, r *http.Reques return false, "", nil } -func (i *handler) getRedirectRules(r *http.Request, redirectsFilePath ipath.Resolved) ([]redirects.Rule, error) { - // Convert the path into a file node - node, err := i.api.GetUnixFsNode(r.Context(), redirectsFilePath) +// getRedirectRules fetches the _redirects file corresponding to a given path and returns the rules +// Returns whether _redirects was found, the rules (if they exist) and if there was an error (other than a missing _redirects) +// If there is an error returns (false, nil, err) +func (i *handler) getRedirectRules(r *http.Request, redirectsPath ImmutablePath) (bool, []redirects.Rule, error) { + // Check for _redirects file. + // Any path resolution failures are ignored and we just assume there's no _redirects file. + // Note that ignoring these errors also ensures that the use of the empty CID (bafkqaaa) in tests doesn't fail. + _, redirectsFileGetResp, err := i.api.Get(r.Context(), redirectsPath) if err != nil { - return nil, fmt.Errorf("could not get _redirects: %w", err) + if isErrNotFound(err) { + return false, nil, nil + } + return false, nil, err } - defer node.Close() - // Convert the node into a file - f, ok := node.(files.File) - if !ok { - return nil, fmt.Errorf("could not parse _redirects: %w", err) + if redirectsFileGetResp.bytes == nil { + return false, nil, fmt.Errorf(" _redirects is not a file") } + f := redirectsFileGetResp.bytes + defer f.Close() // Parse redirect rules from file redirectRules, err := redirects.Parse(f) if err != nil { - return nil, fmt.Errorf("could not parse _redirects: %w", err) - } - - return redirectRules, nil -} - -// Returns a resolved path to the _redirects file located in the root CID path of the requested path -func (i *handler) getRedirectsFile(r *http.Request, contentPath ipath.Path, logger *zap.SugaredLogger) ipath.Resolved { - // contentPath is the full ipfs path to the requested resource, - // regardless of whether path or subdomain resolution is used. - rootPath := getRootPath(contentPath) - - // Check for _redirects file. - // Any path resolution failures are ignored and we just assume there's no _redirects file. - // Note that ignoring these errors also ensures that the use of the empty CID (bafkqaaa) in tests doesn't fail. - path := ipath.Join(rootPath, "_redirects") - resolvedPath, err := i.api.ResolvePath(r.Context(), path) - if err != nil { - return nil + return false, nil, fmt.Errorf("could not parse _redirects: %w", err) } - return resolvedPath + return true, redirectRules, nil } // Returns the root CID Path for the given path @@ -164,24 +181,21 @@ func getRootPath(path ipath.Path) ipath.Path { return ipath.New(gopath.Join("/", path.Namespace(), parts[2])) } -func (i *handler) serve4xx(w http.ResponseWriter, r *http.Request, content4xxPath ipath.Path, status int) error { - resolved4xxPath, err := i.api.ResolvePath(r.Context(), content4xxPath) +func (i *handler) serve4xx(w http.ResponseWriter, r *http.Request, content4xxPathImPath ImmutablePath, content4xxPath ipath.Path, status int) error { + pathMetadata, getresp, err := i.api.Get(r.Context(), content4xxPathImPath) if err != nil { return err } - node, err := i.api.GetUnixFsNode(r.Context(), resolved4xxPath) - if err != nil { - return err - } - defer node.Close() - - f, ok := node.(files.File) - if !ok { + if getresp.bytes == nil { return fmt.Errorf("could not convert node for %d page to file", status) } + content4xxFile := getresp.bytes + defer content4xxFile.Close() - size, err := f.Size() + content4xxCid := pathMetadata.LastSegment.Cid() + + size, err := content4xxFile.Size() if err != nil { return fmt.Errorf("could not get size of %d page", status) } @@ -189,9 +203,9 @@ func (i *handler) serve4xx(w http.ResponseWriter, r *http.Request, content4xxPat log.Debugf("using _redirects: custom %d file at %q", status, content4xxPath) w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) - addCacheControlHeaders(w, r, content4xxPath, resolved4xxPath.Cid()) + addCacheControlHeaders(w, r, content4xxPath, content4xxCid) w.WriteHeader(status) - _, err = io.CopyN(w, f, size) + _, err = io.CopyN(w, content4xxFile, size) return err } @@ -206,51 +220,36 @@ func hasOriginIsolation(r *http.Request) bool { return false } -func isUnixfsResponseFormat(responseFormat string) bool { - // The implicit response format is UnixFS - return responseFormat == "" -} - // Deprecated: legacy ipfs-404.html files are superseded by _redirects file // This is provided only for backward-compatibility, until websites migrate // to 404s managed via _redirects file (https://github.com/ipfs/specs/pull/290) -func (i *handler) serveLegacy404IfPresent(w http.ResponseWriter, r *http.Request, contentPath ipath.Path) bool { - resolved404Path, ctype, err := i.searchUpTreeFor404(r, contentPath) +func (i *handler) serveLegacy404IfPresent(w http.ResponseWriter, r *http.Request, imPath ImmutablePath) bool { + resolved404File, ctype, err := i.searchUpTreeFor404(r, imPath) if err != nil { return false } + defer resolved404File.Close() - dr, err := i.api.GetUnixFsNode(r.Context(), resolved404Path) + size, err := resolved404File.Size() if err != nil { return false } - defer dr.Close() - f, ok := dr.(files.File) - if !ok { - return false - } - - size, err := f.Size() - if err != nil { - return false - } - - log.Debugw("using pretty 404 file", "path", contentPath) + log.Debugw("using pretty 404 file", "path", imPath) w.Header().Set("Content-Type", ctype) w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) w.WriteHeader(http.StatusNotFound) - _, err = io.CopyN(w, f, size) + _, err = io.CopyN(w, resolved404File, size) return err == nil } -func (i *handler) searchUpTreeFor404(r *http.Request, contentPath ipath.Path) (ipath.Resolved, string, error) { +func (i *handler) searchUpTreeFor404(r *http.Request, imPath ImmutablePath) (files.File, string, error) { filename404, ctype, err := preferred404Filename(r.Header.Values("Accept")) if err != nil { return nil, "", err } - pathComponents := strings.Split(contentPath.String(), "/") + pathComponents := strings.Split(imPath.String(), "/") for idx := len(pathComponents); idx >= 3; idx-- { pretty404 := gopath.Join(append(pathComponents[0:idx], filename404)...) @@ -258,11 +257,19 @@ func (i *handler) searchUpTreeFor404(r *http.Request, contentPath ipath.Path) (i if parsed404Path.IsValid() != nil { break } - resolvedPath, err := i.api.ResolvePath(r.Context(), parsed404Path) + imparsed404Path, err := NewImmutablePath(parsed404Path) + if err != nil { + break + } + + _, getResp, err := i.api.Get(r.Context(), imparsed404Path) if err != nil { continue } - return resolvedPath, ctype, nil + if getResp.bytes == nil { + return nil, "", fmt.Errorf("found a pretty 404 but it was not a file") + } + return getResp.bytes, ctype, nil } return nil, "", fmt.Errorf("no pretty 404 in any parent folder") diff --git a/gateway/handler_unixfs_dir.go b/gateway/handler_unixfs_dir.go index 40ea6ae0b..6c5317246 100644 --- a/gateway/handler_unixfs_dir.go +++ b/gateway/handler_unixfs_dir.go @@ -14,7 +14,6 @@ import ( "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/gateway/assets" path "github.com/ipfs/boxo/path" - "github.com/ipfs/boxo/path/resolver" cid "github.com/ipfs/go-cid" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -24,7 +23,7 @@ import ( // serveDirectory returns the best representation of UnixFS directory // // It will return index.html if present, or generate directory listing otherwise. -func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, dir files.Directory, begin time.Time, logger *zap.SugaredLogger) bool { +func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, isHeadRequest bool, directoryMetadata *directoryMetadata, begin time.Time, logger *zap.SugaredLogger) bool { ctx, span := spanTrace(ctx, "ServeDirectory", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) defer span.End() @@ -61,31 +60,50 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * // Check if directory has index.html, if so, serveFile idxPath := ipath.Join(contentPath, "index.html") - idxResolvedPath, err := i.api.ResolvePath(ctx, idxPath) - switch err.(type) { - case nil: - idx, err := i.api.GetUnixFsNode(ctx, idxResolvedPath) - if err != nil { - webError(w, err, http.StatusInternalServerError) - return false - } + imIndexPath, err := NewImmutablePath(ipath.Join(resolvedPath, "index.html")) + if err != nil { + webError(w, err, http.StatusInternalServerError) + return false + } - f, ok := idx.(files.File) - if !ok { - webError(w, files.ErrNotReader, http.StatusInternalServerError) - return false + // TODO: could/should this all be skipped to have HEAD requests just return html content type and save the complexity? If so can we skip the above code as well? + var idxFile files.File + if isHeadRequest { + var idx files.Node + _, idx, err = i.api.Head(ctx, imIndexPath) + if err == nil { + f, ok := idx.(files.File) + if !ok { + webError(w, fmt.Errorf("%q could not be read: %w", imIndexPath, files.ErrNotReader), http.StatusUnprocessableEntity) + return false + } + idxFile = f + } + } else { + var getResp *GetResponse + _, getResp, err = i.api.Get(ctx, imIndexPath) + if err == nil { + if getResp.bytes == nil { + webError(w, fmt.Errorf("%q could not be read: %w", imIndexPath, files.ErrNotReader), http.StatusUnprocessableEntity) + return false + } + idxFile = getResp.bytes } + } + if err == nil { logger.Debugw("serving index.html file", "path", idxPath) // write to request - success := i.serveFile(ctx, w, r, resolvedPath, idxPath, f, begin) + success := i.serveFile(ctx, w, r, resolvedPath, idxPath, idxFile, "text/html", begin) if success { i.unixfsDirIndexGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) } return success - case resolver.ErrNoLink: + } + + if isErrNotFound(err) { logger.Debugw("no index.html; noop", "path", idxPath) - default: + } else if err != nil { webError(w, err, http.StatusInternalServerError) return false } @@ -104,7 +122,7 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * // type instead of relying on autodetection (which may fail). w.Header().Set("Content-Type", "text/html") - // Generated dir index requires custom Etag (output may change between go-ipfs versions) + // Generated dir index requires custom Etag (output may change between go-libipfs versions) dirEtag := getDirListingEtag(resolvedPath.Cid()) w.Header().Set("Etag", dirEtag) @@ -113,24 +131,22 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * return true } - results, err := i.api.LsUnixFsDir(ctx, resolvedPath) - if err != nil { - webError(w, err, http.StatusInternalServerError) - return false - } - - dirListing := make([]assets.DirectoryItem, 0, len(results)) - for link := range results { - if link.Err != nil { - webError(w, link.Err, http.StatusInternalServerError) + var dirListing []assets.DirectoryItem + for l := range directoryMetadata.entries { + if l.Err != nil { + webError(w, l.Err, http.StatusInternalServerError) return false } - hash := link.Cid.String() + name := l.Link.Name + sz := l.Link.Size + linkCid := l.Link.Cid + + hash := linkCid.String() di := assets.DirectoryItem{ - Size: humanize.Bytes(uint64(link.Size)), - Name: link.Name, - Path: gopath.Join(originalURLPath, link.Name), + Size: humanize.Bytes(sz), + Name: name, + Path: gopath.Join(originalURLPath, name), Hash: hash, ShortHash: assets.ShortHash(hash), } @@ -161,11 +177,7 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * } } - size := "?" - if s, err := dir.Size(); err == nil { - // Size may not be defined/supported. Continue anyways. - size = humanize.Bytes(uint64(s)) - } + size := humanize.Bytes(directoryMetadata.dagSize) hash := resolvedPath.Cid().String() diff --git a/gateway/handler_unixfs_file.go b/gateway/handler_unixfs_file.go index 980885cb2..d5609abd2 100644 --- a/gateway/handler_unixfs_file.go +++ b/gateway/handler_unixfs_file.go @@ -19,7 +19,7 @@ import ( // serveFile returns data behind a file along with HTTP headers based on // the file itself, its CID and the contentPath used for accessing it. -func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, file files.File, begin time.Time) bool { +func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, file files.File, fileContentType string, begin time.Time) bool { _, span := spanTrace(ctx, "ServeFile", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) defer span.End() @@ -60,6 +60,9 @@ func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http. ctype = "inode/symlink" } else { ctype = mime.TypeByExtension(gopath.Ext(name)) + if ctype == "" { + ctype = fileContentType + } if ctype == "" { // uses https://github.com/gabriel-vasile/mimetype library to determine the content type. // Fixes https://github.com/ipfs/kubo/issues/7252 diff --git a/gateway/hostname.go b/gateway/hostname.go index 97e8bd41e..bb0d4da25 100644 --- a/gateway/hostname.go +++ b/gateway/hostname.go @@ -57,7 +57,7 @@ type Specification struct { // noDNSLink configures the gateway to _not_ perform DNS TXT record lookups in // response to requests with values in `Host` HTTP header. This flag can be overridden // per FQDN in publicGateways. -func WithHostname(next http.Handler, api API, publicGateways map[string]*Specification, noDNSLink bool) http.HandlerFunc { +func WithHostname(next http.Handler, api IPFSBackend, publicGateways map[string]*Specification, noDNSLink bool) http.HandlerFunc { gateways := prepareHostnameGateways(publicGateways) return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -268,7 +268,7 @@ func isDomainNameAndNotPeerID(hostname string) bool { } // hasDNSLinkRecord returns if a DNS TXT record exists for the provided host. -func hasDNSLinkRecord(ctx context.Context, api API, host string) bool { +func hasDNSLinkRecord(ctx context.Context, api IPFSBackend, host string) bool { dnslinkName := stripPort(host) if !isDomainNameAndNotPeerID(dnslinkName) { @@ -355,7 +355,7 @@ func toDNSLinkFQDN(dnsLabel string) (fqdn string) { } // Converts a hostname/path to a subdomain-based URL, if applicable. -func toSubdomainURL(hostname, path string, r *http.Request, inlineDNSLink bool, api API) (redirURL string, err error) { +func toSubdomainURL(hostname, path string, r *http.Request, inlineDNSLink bool, api IPFSBackend) (redirURL string, err error) { var scheme, ns, rootID, rest string query := r.URL.RawQuery diff --git a/go.mod b/go.mod index b11c795f9..d7c4faf91 100644 --- a/go.mod +++ b/go.mod @@ -40,9 +40,11 @@ require ( github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd github.com/jbenet/goprocess v0.1.4 github.com/libp2p/go-buffer-pool v0.1.0 + github.com/libp2p/go-doh-resolver v0.4.0 github.com/libp2p/go-libp2p v0.26.3 github.com/libp2p/go-libp2p-kad-dht v0.21.1 github.com/libp2p/go-libp2p-record v0.2.0 + github.com/libp2p/go-libp2p-routing-helpers v0.4.0 github.com/libp2p/go-libp2p-testing v0.12.0 github.com/libp2p/go-msgio v0.3.0 github.com/miekg/dns v1.1.50 diff --git a/go.sum b/go.sum index 3bdea04b2..08ecc37f7 100644 --- a/go.sum +++ b/go.sum @@ -425,6 +425,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= +github.com/libp2p/go-doh-resolver v0.4.0 h1:gUBa1f1XsPwtpE1du0O+nnZCUqtG7oYi7Bb+0S7FQqw= +github.com/libp2p/go-doh-resolver v0.4.0/go.mod h1:v1/jwsFusgsWIGX/c6vCRrnJ60x7bhTiq/fs2qt0cAg= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= github.com/libp2p/go-libp2p v0.26.3 h1:6g/psubqwdaBqNNoidbRKSTBEYgaOuKBhHl8Q5tO+PM= @@ -437,6 +439,8 @@ github.com/libp2p/go-libp2p-kbucket v0.5.0 h1:g/7tVm8ACHDxH29BGrpsQlnNeu+6OF1A9b github.com/libp2p/go-libp2p-kbucket v0.5.0/go.mod h1:zGzGCpQd78b5BNTDGHNDLaTt9aDK/A02xeZp9QeFC4U= github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= +github.com/libp2p/go-libp2p-routing-helpers v0.4.0 h1:b7y4aixQ7AwbqYfcOQ6wTw8DQvuRZeTAA0Od3YYN5yc= +github.com/libp2p/go-libp2p-routing-helpers v0.4.0/go.mod h1:dYEAgkVhqho3/YKxfOEGdFMIcWfAFNlZX8iAIihYA2E= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= @@ -504,6 +508,7 @@ github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU= github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= +github.com/multiformats/go-multiaddr-dns v0.3.0/go.mod h1:mNzQ4eTGDg0ll1N9jKPOUogZPoJ30W8a7zk66FQPpdQ= github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E=