diff --git a/test/cli/fixtures/README.md b/test/cli/fixtures/README.md new file mode 100644 index 00000000000..0da73382cfb --- /dev/null +++ b/test/cli/fixtures/README.md @@ -0,0 +1,72 @@ +# Dataset Description / Sources + +TestGatewayHAMTDirectory.car generated with: + +```bash +ipfs version +# ipfs version 0.19.0 + +export HAMT_DIR=bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm +export IPFS_PATH=$(mktemp -d) + +# Init and start daemon, ensure we have an empty repository. +ipfs init --empty-repo +ipfs daemon &> /dev/null & +export IPFS_PID=$! + +# Retrieve the directory listing, forcing the daemon to download all required DAGs. Kill daemon. +curl -o dir.html http://127.0.0.1:8080/ipfs/$HAMT_DIR/ +kill $IPFS_PID + +# Get the list with all the downloaded refs and sanity check. +ipfs refs local > required_refs +cat required_refs | wc -l +# 962 + +# Get the list of all the files CIDs inside the directory and sanity check. +cat dir.html| pup '#content tbody .ipfs-hash attr{href}' | sed 's/\/ipfs\///g;s/\?filename=.*//g' > files_refs +cat files_refs | wc -l +# 10100 + +# Make and export our fixture. +ipfs files mkdir --cid-version 1 /fixtures +cat required_refs | xargs -I {} ipfs files cp /ipfs/{} /fixtures/{} +cat files_refs | ipfs files write --create /fixtures/files_refs +export FIXTURE_CID=$(ipfs files stat --hash /fixtures/) +echo $FIXTURE_CID +# bafybeig3yoibxe56aolixqa4zk55gp5sug3qgaztkakpndzk2b2ynobd4i +ipfs dag export $FIXTURE_CID > TestGatewayHAMTDirectory.car +``` + +TestGatewayMultiRange.car generated with: + + +```sh +ipfs version +# ipfs version 0.19.0 + +export FILE_CID=bafybeibkzwf3ffl44yfej6ak44i7aly7rb4udhz5taskreec7qemmw5jiu +export IPFS_PATH=$(mktemp -d) + +# Init and start daemon, ensure we have an empty repository. +ipfs init --empty-repo +ipfs daemon &> /dev/null & +export IPFS_PID=$! + +# Get a specific byte range from the file. +curl http://127.0.0.1:8080/ipfs/$FILE_CID -i -H "Range: bytes=2000-2002, 40000000000-40000000002" +kill $IPFS_PID + +# Get the list with all the downloaded refs and sanity check. +ipfs refs local > required_refs +cat required_refs | wc -l +# 48 + +# Make and export our fixture. +ipfs files mkdir --cid-version 1 /fixtures +cat required_refs | xargs -I {} ipfs files cp /ipfs/{} /fixtures/{} +export FIXTURE_CID=$(ipfs files stat --hash /fixtures/) +echo $FIXTURE_CID +# bafybeihqs4hdx64a6wmrclp3a2pwxkd5prwdos45bdftpegls5ktzspi7a +ipfs dag export $FIXTURE_CID > TestGatewayMultiRange.car +``` diff --git a/test/sharness/t0115-gateway-dir-listing/hamt-refs.car b/test/cli/fixtures/TestGatewayHAMTDirectory.car similarity index 56% rename from test/sharness/t0115-gateway-dir-listing/hamt-refs.car rename to test/cli/fixtures/TestGatewayHAMTDirectory.car index 58069310cc4..2cb03c6c879 100644 Binary files a/test/sharness/t0115-gateway-dir-listing/hamt-refs.car and b/test/cli/fixtures/TestGatewayHAMTDirectory.car differ diff --git a/test/sharness/t0110-gateway/hamt-refs.car b/test/cli/fixtures/TestGatewayMultiRange.car similarity index 82% rename from test/sharness/t0110-gateway/hamt-refs.car rename to test/cli/fixtures/TestGatewayMultiRange.car index 8946047593c..b9c89df94dc 100644 Binary files a/test/sharness/t0110-gateway/hamt-refs.car and b/test/cli/fixtures/TestGatewayMultiRange.car differ diff --git a/test/cli/gateway_range_test.go b/test/cli/gateway_range_test.go new file mode 100644 index 00000000000..e8433067415 --- /dev/null +++ b/test/cli/gateway_range_test.go @@ -0,0 +1,116 @@ +package cli + +import ( + "fmt" + "io" + "math/rand" + "net/http" + "os" + "strings" + "testing" + "time" + + "github.com/ipfs/kubo/test/cli/harness" + "github.com/stretchr/testify/assert" +) + +func TestGatewayHAMTDirectory(t *testing.T) { + t.Parallel() + + const ( + // The CID of the HAMT-sharded directory that has 10k items + hamtCid = "bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm" + + // fixtureCid is the CID of root of the DAG that is a subset of hamtCid DAG + // representing the minimal set of blocks necessary for directory listing. + // It also includes a "files_refs" file with the list of the references + // we do NOT needs to fetch (files inside the directory) + fixtureCid = "bafybeig3yoibxe56aolixqa4zk55gp5sug3qgaztkakpndzk2b2ynobd4i" + ) + + // Start node + h := harness.NewT(t) + node := h.NewNode().Init("--empty-repo", "--profile=test").StartDaemon("--offline") + client := node.GatewayClient() + + // Import fixtures + r, err := os.Open("./fixtures/TestGatewayHAMTDirectory.car") + assert.Nil(t, err) + defer r.Close() + cid := node.IPFSDagImport(r) + assert.Equal(t, fixtureCid, cid) + + t.Run("Fetch HAMT directory succeeds with minimal refs", func(t *testing.T) { + t.Parallel() + resp := client.Get(fmt.Sprintf("/ipfs/%s/", hamtCid)) + assert.Equal(t, http.StatusOK, resp.StatusCode) + }) + + t.Run("Non-minimal refs are not present in the repository", func(t *testing.T) { + t.Parallel() + + // Fetch list with refs of files that should NOT be available locally. + resp := client.Get(fmt.Sprintf("/ipfs/%s/files_refs", fixtureCid)) + assert.Equal(t, http.StatusOK, resp.StatusCode) + files := strings.Split(strings.TrimSpace(resp.Body), "\n") + assert.Len(t, files, 10100) + + // Shuffle the files list and try fetching the first 200. + rand.Seed(time.Now().UnixNano()) + rand.Shuffle(len(files), func(i, j int) { files[i], files[j] = files[j], files[i] }) + for _, cid := range files[:200] { + resp = client.Get(fmt.Sprintf("/ipfs/%s", cid)) + assert.Equal(t, http.StatusNotFound, resp.StatusCode) + } + }) +} + +func TestGatewayMultiRange(t *testing.T) { + t.Parallel() + + const ( + // fileCid is the CID of the large HAMT-sharded file. + fileCid = "bafybeibkzwf3ffl44yfej6ak44i7aly7rb4udhz5taskreec7qemmw5jiu" + + // fixtureCid is the CID of root of the DAG that is a subset of fileCid DAG + // representing the minimal set of blocks necessary for a simple byte range request. + fixtureCid = "bafybeihqs4hdx64a6wmrclp3a2pwxkd5prwdos45bdftpegls5ktzspi7a" + ) + + // Start node + h := harness.NewT(t) + node := h.NewNode().Init("--empty-repo", "--profile=test").StartDaemon("--offline") + client := node.GatewayClient() + + // Import fixtures + r, err := os.Open("./fixtures/TestGatewayMultiRange.car") + assert.Nil(t, err) + defer r.Close() + cid := node.IPFSDagImport(r) + assert.Equal(t, fixtureCid, cid) + + t.Run("Succeeds to fetch range of blocks we have", func(t *testing.T) { + t.Parallel() + + resp := client.Get(fmt.Sprintf("/ipfs/%s", fileCid), func(r *http.Request) { + r.Header.Set("Range", "bytes=2000-2002, 40000000000-40000000002") + }) + assert.Equal(t, http.StatusPartialContent, resp.StatusCode) + assert.Contains(t, resp.Body, "Content-Type: application/octet-stream") + assert.Contains(t, resp.Body, "Content-Range: bytes 2000-2002/87186935127") + assert.Contains(t, resp.Body, "Content-Range: bytes 40000000000-40000000002/87186935127") + }) + + t.Run("Fail to fetch range of blocks we do not have", func(t *testing.T) { + t.Parallel() + + req, err := http.NewRequest(http.MethodGet, client.BuildURL(fmt.Sprintf("/ipfs/%s", fileCid)), nil) + assert.Nil(t, err) + req.Header.Set("Range", "bytes=1000-1100, 87186935125-87186935127") + httpResp, err := client.Client.Do(req) + assert.Nil(t, err) + assert.Equal(t, http.StatusPartialContent, httpResp.StatusCode) + _, err = io.ReadAll(httpResp.Body) + assert.Equal(t, err, io.ErrUnexpectedEOF) + }) +} diff --git a/test/cli/harness/ipfs.go b/test/cli/harness/ipfs.go index 6ae7bdf947d..01addb92d64 100644 --- a/test/cli/harness/ipfs.go +++ b/test/cli/harness/ipfs.go @@ -78,3 +78,20 @@ func (n *Node) IPFSAdd(content io.Reader, args ...string) string { log.Debugf("add result: %q", out) return out } + +func (n *Node) IPFSDagImport(content io.Reader, args ...string) string { + log.Debugf("node %d dag import with args: %v", n.ID, args) + fullArgs := []string{"dag", "import"} + fullArgs = append(fullArgs, args...) + res := n.Runner.MustRun(RunRequest{ + Path: n.IPFSBin, + Args: fullArgs, + CmdOpts: []CmdOpt{RunWithStdin(content)}, + }) + out := strings.TrimSpace(res.Stdout.String()) + out = strings.TrimPrefix(out, "Pinned root") + out = strings.TrimSuffix(out, "success") + out = strings.TrimSpace(out) + log.Debugf("add result: %q", out) + return out +} diff --git a/test/sharness/t0110-gateway.sh b/test/sharness/t0110-gateway.sh deleted file mode 100755 index c84ac6a28d4..00000000000 --- a/test/sharness/t0110-gateway.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -test_description="Miscellaneous Gateway Tests" - -. lib/test-lib.sh - -## ============================================================================ -## HAMT-sharded file range test. We must execute this test first as we want to -## start from an empty repository and count the exact number of refs we want. -## ============================================================================ - -test_expect_success "ipfs init" ' - export IPFS_PATH="$(pwd)/.ipfs-for-hamt" && - ipfs init --empty-repo --profile=test > /dev/null -' - -test_launch_ipfs_daemon_without_network - -# HAMT_CID is a HAMT file. -HAMT_CID=bafybeibkzwf3ffl44yfej6ak44i7aly7rb4udhz5taskreec7qemmw5jiu - -# HAMT_REFS_CID is root of a DAG that is a subset of HAMT_CID DAG -# representing minimal set of block necessary for the range request. -HAMT_REFS_CID=bafybeigcvrf7fvk7i3fdhxgk6saqdpw6spujwfxxkq5cshy5kxdjc674ua -HAMT_REFS_COUNT=48 - -test_expect_success "hamt: import fixture with necessary refs" ' - ipfs dag import ../t0110-gateway/hamt-refs.car && - ipfs refs local | wc -l | tr -d " " > refs_count_actual && - echo $HAMT_REFS_COUNT > refs_count_expected && - test_cmp refs_count_expected refs_count_actual -' - -# we want to confirm that the code responsible for the range requests in regular -# files works with a minimal set of blocks, and does not fetch the entire thing -# (regression test) -test_expect_success "hamt: fetch directory from gateway in offline mode" ' - curl --max-time 30 -sD - -H "Range: bytes=2000-2002, 40000000000-40000000002" http://127.0.0.1:$GWAY_PORT/ipfs/$HAMT_CID/ > hamt_range_request && - ipfs refs local | wc -l | tr -d " " > refs_count_actual && - echo $HAMT_REFS_COUNT > refs_count_expected && - test_cmp refs_count_expected refs_count_actual && - test_should_contain "Content-Range: bytes 2000-2002/87186935127" hamt_range_request && - test_should_contain "Content-Type: application/octet-stream" hamt_range_request && - test_should_contain "Content-Range: bytes 40000000000-40000000002/87186935127" hamt_range_request -' - -test_kill_ipfs_daemon - -test_done diff --git a/test/sharness/t0110-gateway/README.md b/test/sharness/t0110-gateway/README.md deleted file mode 100644 index d959ec87902..00000000000 --- a/test/sharness/t0110-gateway/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# Dataset description/sources - -- hamt-refs.car - - raw CARv1 - -generated with: - -```sh -# using ipfs version 0.18.1 -export IPFS_PATH=$(mktemp -d) -ipfs init --empty-repo -ipfs daemon & -curl http://127.0.0.1:8080/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze/wikipedia_en_all_maxi_2021-02.zim -i -H "Range: bytes=2000-2002, 40000000000-40000000002" -killall ipfs -ipfs refs local >> refs -ipfs files mkdir --cid-version 1 /Test -cat refs | xargs -I {} ipfs files cp /ipfs/{} /Test/{} -ipfs files stat /Test -# Grab CID: bafybeigcvrf7fvk7i3fdhxgk6saqdpw6spujwfxxkq5cshy5kxdjc674ua -ipfs dag export bafybeigcvrf7fvk7i3fdhxgk6saqdpw6spujwfxxkq5cshy5kxdjc674ua > ./refs.car -``` diff --git a/test/sharness/t0115-gateway-dir-listing.sh b/test/sharness/t0115-gateway-dir-listing.sh index 7805d521f02..cf95bf4b05e 100755 --- a/test/sharness/t0115-gateway-dir-listing.sh +++ b/test/sharness/t0115-gateway-dir-listing.sh @@ -8,61 +8,21 @@ test_description="Test directory listing (dir-index-html) on the HTTP gateway" . lib/test-lib.sh ## ============================================================================ -## HAMT-sharded directory test. We must execute this test first as we want to -## start from an empty repository and count the exact number of refs we want. +## Start IPFS Node and prepare test CIDs ## ============================================================================ test_expect_success "ipfs init" ' - export IPFS_PATH="$(pwd)/.ipfs-for-hamt" && - ipfs init --empty-repo --profile=test > /dev/null + export IPFS_PATH="$(pwd)/.ipfs" && + ipfs init --profile=test > /dev/null ' test_launch_ipfs_daemon_without_network -# HAMT_CID is a hamt-sharded-directory that has 10k of items -HAMT_CID=bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm - -# HAMT_REFS_CID is root of a DAG that is a subset of HAMT_CID DAG -# representing minimal set of block necessary for directory listing -# without fetching ANY root blocks of child items. -HAMT_REFS_CID=bafybeicv4utmj46mgbpamvw2k6zg4qjs5yvspfnlxz2y6iuey5smjfcn4a -HAMT_REFS_COUNT=963 - -test_expect_success "hamt: import fixture with necessary refs" ' - ipfs dag import ../t0115-gateway-dir-listing/hamt-refs.car && - ipfs refs local | wc -l | tr -d " " > refs_count_actual && - echo $HAMT_REFS_COUNT > refs_count_expected && - test_cmp refs_count_expected refs_count_actual -' - -# we want to confirm that code responsible for directory listing works -# with minimal set of blocks, and does not fetch entire thing (regression test) -test_expect_success "hamt: fetch directory from gateway in offline mode" ' - curl --max-time 30 -sD - http://127.0.0.1:$GWAY_PORT/ipfs/$HAMT_CID/ > hamt_list_response && - ipfs refs local | wc -l | tr -d " " > refs_count_actual && - echo $HAMT_REFS_COUNT > refs_count_expected && - test_cmp refs_count_expected refs_count_actual -' - -test_kill_ipfs_daemon - -## ============================================================================ -## Start IPFS Node and prepare test CIDs -## ============================================================================ - -test_expect_success "ipfs init" ' - export IPFS_PATH="$(pwd)/.ipfs" && - ipfs init --empty-repo --profile=test > /dev/null -' -# # Import test case # See the static fixtures in ./t0115-gateway-dir-listing/ -test_expect_success "add remaining test fixtures" ' +test_expect_success "Add the test directory" ' ipfs dag import ../t0115-gateway-dir-listing/fixtures.car ' - -test_launch_ipfs_daemon_without_network - DIR_CID=bafybeig6ka5mlwkl4subqhaiatalkcleo4jgnr3hqwvpmsqfca27cijp3i # ./rootDir/ FILE_CID=bafkreialihlqnf5uwo4byh4n3cmwlntwqzxxs2fg5vanqdi3d7tb2l5xkm # ./rootDir/ą/ę/file-źł.txt FILE_SIZE=34 diff --git a/test/sharness/t0115-gateway-dir-listing/README.md b/test/sharness/t0115-gateway-dir-listing/README.md index e2d1abb29a6..937438bcd65 100644 --- a/test/sharness/t0115-gateway-dir-listing/README.md +++ b/test/sharness/t0115-gateway-dir-listing/README.md @@ -2,12 +2,8 @@ - fixtures.car - raw CARv1 -- hamt-refs.car - - raw CARv1 containing all the necessary blocks to present a directory listing - for `bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm`, which is - a directory containing over 10k items. -fixtures.car generated with: +generated with: ```sh # using ipfs version 0.18.1 @@ -34,20 +30,3 @@ ipfs dag export ${DIR_CID} > ./fixtures.car # FILE_CID=bafkreialihlqnf5uwo4byh4n3cmwlntwqzxxs2fg5vanqdi3d7tb2l5xkm # ./rootDir/ą/ę/file-źł.txt # FILE_SIZE=34 ``` - -hamt-refs.car generated with: - -```sh -# using ipfs version 0.18.1 -export IPFS_PATH=$(mktemp -d) -ipfs init --empty-repo -ipfs daemon & -curl http://127.0.0.1:8080/ipfs/bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm/ -killall ipfs -ipfs refs local >> refs -ipfs files mkdir --cid-version 1 /Test -cat refs | xargs -I {} ipfs files cp /ipfs/{} /Test/{} -ipfs files stat /Test -# Grab CID: bafybeicv4utmj46mgbpamvw2k6zg4qjs5yvspfnlxz2y6iuey5smjfcn4a -ipfs dag export bafybeicv4utmj46mgbpamvw2k6zg4qjs5yvspfnlxz2y6iuey5smjfcn4a > ./refs.car -```