diff --git a/.github/workflows/sharness.yml b/.github/workflows/sharness.yml index 7b392bcd992..90c622c3bd1 100644 --- a/.github/workflows/sharness.yml +++ b/.github/workflows/sharness.yml @@ -29,24 +29,10 @@ jobs: path: kubo - name: Install missing tools run: sudo apt install -y socat net-tools fish libxml2-utils - - name: Checkout IPFS Pinning Service API - uses: actions/checkout@v3 - with: - repository: ipfs-shipyard/rb-pinning-service-api - ref: 773c3adbb421c551d2d89288abac3e01e1f7c3a8 - path: rb-pinning-service-api - # TODO: check if docker compose (not docker-compose) is available on default gh runners - - name: Start IPFS Pinning Service API - run: | - (for i in {1..3}; do docker compose pull && break || sleep 5; done) && - docker compose up -d - working-directory: rb-pinning-service-api - name: Restore Go Cache uses: protocol/cache-go-action@v1 with: name: ${{ github.job }} - - name: Find IPFS Pinning Service API address - run: echo "TEST_DOCKER_HOST=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')" >> $GITHUB_ENV - uses: actions/cache@v3 with: path: test/sharness/lib/dependencies diff --git a/cmd/ipfs/pinmfs.go b/cmd/ipfs/pinmfs.go index f36b0a8c53f..c2c0cb8b7f7 100644 --- a/cmd/ipfs/pinmfs.go +++ b/cmd/ipfs/pinmfs.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "os" "time" "github.com/libp2p/go-libp2p/core/host" @@ -31,7 +32,19 @@ func (x lastPin) IsValid() bool { return x != lastPin{} } -const daemonConfigPollInterval = time.Minute / 2 +var daemonConfigPollInterval = time.Minute / 2 + +func init() { + // this environment variable is solely for testing, use at your own risk + if pollDurStr := os.Getenv("MFS_PIN_POLL_INTERVAL"); pollDurStr != "" { + d, err := time.ParseDuration(pollDurStr) + if err != nil { + mfslog.Error("error parsing MFS_PIN_POLL_INTERVAL, using default:", err) + } + daemonConfigPollInterval = d + } +} + const defaultRepinInterval = 5 * time.Minute type pinMFSContext interface { diff --git a/go.mod b/go.mod index 3d8585be087..3648783d532 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/jbenet/go-temp-err-catcher v0.1.0 github.com/jbenet/goprocess v0.1.4 + github.com/julienschmidt/httprouter v1.3.0 github.com/libp2p/go-doh-resolver v0.4.0 github.com/libp2p/go-libp2p v0.26.4 github.com/libp2p/go-libp2p-http v0.4.0 @@ -67,6 +68,8 @@ require ( github.com/prometheus/client_golang v1.14.0 github.com/stretchr/testify v1.8.2 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 + github.com/tidwall/gjson v1.14.4 + github.com/tidwall/sjson v1.2.5 github.com/whyrusleeping/go-sysinfo v0.0.0-20190219211824-4a357d4b90b1 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 go.opencensus.io v0.24.0 @@ -198,6 +201,8 @@ require ( github.com/samber/lo v1.36.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect diff --git a/go.sum b/go.sum index fc8dd18e8a0..95daffbf755 100644 --- a/go.sum +++ b/go.sum @@ -495,6 +495,7 @@ github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVY github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -858,6 +859,15 @@ github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cb github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e h1:T5PdfK/M1xyrHwynxMIVMWLS7f/qHwfslZphxtGnw7s= github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= diff --git a/test/cli/harness/node.go b/test/cli/harness/node.go index f740ab1b19f..4d9ed965c23 100644 --- a/test/cli/harness/node.go +++ b/test/cli/harness/node.go @@ -76,6 +76,23 @@ func (n *Node) WriteBytes(filename string, b []byte) { } } +// ReadFile reads the specific file. If it is relative, it is relative the node's root dir. +func (n *Node) ReadFile(filename string) string { + f := filename + if !filepath.IsAbs(filename) { + f = filepath.Join(n.Dir, filename) + } + b, err := os.ReadFile(f) + if err != nil { + panic(err) + } + return string(b) +} + +func (n *Node) ConfigFile() string { + return filepath.Join(n.Dir, "config") +} + func (n *Node) ReadConfig() *config.Config { cfg, err := serial.Load(filepath.Join(n.Dir, "config")) if err != nil { diff --git a/test/cli/must.go b/test/cli/must.go new file mode 100644 index 00000000000..e125984666d --- /dev/null +++ b/test/cli/must.go @@ -0,0 +1,8 @@ +package cli + +func MustVal[V any](val V, err error) V { + if err != nil { + panic(err) + } + return val +} diff --git a/test/cli/pinning_remote_test.go b/test/cli/pinning_remote_test.go new file mode 100644 index 00000000000..fede942baae --- /dev/null +++ b/test/cli/pinning_remote_test.go @@ -0,0 +1,446 @@ +package cli + +import ( + "errors" + "fmt" + "net" + "net/http" + "testing" + "time" + + "github.com/google/uuid" + "github.com/ipfs/kubo/test/cli/harness" + "github.com/ipfs/kubo/test/cli/testutils" + "github.com/ipfs/kubo/test/cli/testutils/pinningservice" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" +) + +func runPinningService(t *testing.T, authToken string) (*pinningservice.PinningService, string) { + svc := pinningservice.New() + router := pinningservice.NewRouter(authToken, svc) + server := &http.Server{Handler: router} + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + go func() { + err := server.Serve(listener) + if err != nil && !errors.Is(err, net.ErrClosed) && !errors.Is(err, http.ErrServerClosed) { + t.Logf("Serve error: %s", err) + } + }() + t.Cleanup(func() { listener.Close() }) + + return svc, fmt.Sprintf("http://%s/api/v1", listener.Addr().String()) +} + +func TestRemotePinning(t *testing.T) { + t.Parallel() + authToken := "testauthtoken" + + t.Run("MFS pinning", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.Runner.Env["MFS_PIN_POLL_INTERVAL"] = "10ms" + + _, svcURL := runPinningService(t, authToken) + node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken) + node.IPFS("config", "--json", "Pinning.RemoteServices.svc.Policies.MFS.RepinInterval", `"1s"`) + node.IPFS("config", "--json", "Pinning.RemoteServices.svc.Policies.MFS.PinName", `"test_pin"`) + node.IPFS("config", "--json", "Pinning.RemoteServices.svc.Policies.MFS.Enable", "true") + + node.StartDaemon() + + node.IPFS("files", "cp", "/ipfs/bafkqaaa", "/mfs-pinning-test-"+uuid.NewString()) + node.IPFS("files", "flush") + res := node.IPFS("files", "stat", "/", "--enc=json") + hash := gjson.Get(res.Stdout.String(), "Hash").Str + + assert.Eventually(t, + func() bool { + res = node.IPFS("pin", "remote", "ls", + "--service=svc", + "--name=test_pin", + "--status=queued,pinning,pinned,failed", + "--enc=json", + ) + pinnedHash := gjson.Get(res.Stdout.String(), "Cid").Str + return hash == pinnedHash + }, + 10*time.Second, + 10*time.Millisecond, + ) + + t.Run("MFS root is repinned on CID change", func(t *testing.T) { + node.IPFS("files", "cp", "/ipfs/bafkqaaa", "/mfs-pinning-repin-test-"+uuid.NewString()) + node.IPFS("files", "flush") + res = node.IPFS("files", "stat", "/", "--enc=json") + hash := gjson.Get(res.Stdout.String(), "Hash").Str + assert.Eventually(t, + func() bool { + res := node.IPFS("pin", "remote", "ls", + "--service=svc", + "--name=test_pin", + "--status=queued,pinning,pinned,failed", + "--enc=json", + ) + pinnedHash := gjson.Get(res.Stdout.String(), "Cid").Str + return hash == pinnedHash + }, + 10*time.Second, + 10*time.Millisecond, + ) + }) + }) + + // Pinning.RemoteServices includes API.Key, so we give it the same treatment + // as Identity,PrivKey to prevent exposing it on the network + t.Run("access token security", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.IPFS("pin", "remote", "service", "add", "1", "http://example1.com", "testkey") + res := node.RunIPFS("config", "Pinning") + assert.Equal(t, 1, res.ExitCode()) + assert.Contains(t, res.Stderr.String(), "cannot show or change pinning services credentials") + assert.NotContains(t, res.Stdout.String(), "testkey") + + res = node.RunIPFS("config", "Pinning.RemoteServices.1.API.Key") + assert.Equal(t, 1, res.ExitCode()) + assert.Contains(t, res.Stderr.String(), "cannot show or change pinning services credentials") + assert.NotContains(t, res.Stdout.String(), "testkey") + + configShow := node.RunIPFS("config", "show").Stdout.String() + assert.NotContains(t, configShow, "testkey") + + t.Run("re-injecting config with 'ipfs config replace' preserves the API keys", func(t *testing.T) { + node.WriteBytes("config-show", []byte(configShow)) + node.IPFS("config", "replace", "config-show") + assert.Contains(t, node.ReadFile(node.ConfigFile()), "testkey") + }) + + t.Run("injecting config with 'ipfs config replace' with API keys returns an error", func(t *testing.T) { + // remove Identity.PrivKey to ensure error is triggered by Pinning.RemoteServices + configJSON := MustVal(sjson.Delete(configShow, "Identity.PrivKey")) + configJSON = MustVal(sjson.Set(configJSON, "Pinning.RemoteServices.1.API.Key", "testkey")) + node.WriteBytes("new-config", []byte(configJSON)) + res := node.RunIPFS("config", "replace", "new-config") + assert.Equal(t, 1, res.ExitCode()) + assert.Contains(t, res.Stderr.String(), "cannot change remote pinning services api info with `config replace`") + }) + }) + + t.Run("pin remote service ls --stat", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + _, svcURL := runPinningService(t, authToken) + + node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken) + node.IPFS("pin", "remote", "service", "add", "invalid-svc", svcURL+"/invalidpath", authToken) + + res := node.IPFS("pin", "remote", "service", "ls", "--stat") + assert.Contains(t, res.Stdout.String(), " 0/0/0/0") + + stats := node.IPFS("pin", "remote", "service", "ls", "--stat", "--enc=json").Stdout.String() + assert.Equal(t, "valid", gjson.Get(stats, `RemoteServices.#(Service == "svc").Stat.Status`).Str) + assert.Equal(t, "invalid", gjson.Get(stats, `RemoteServices.#(Service == "invalid-svc").Stat.Status`).Str) + + // no --stat returns no stat obj + t.Run("no --stat returns no stat obj", func(t *testing.T) { + res := node.IPFS("pin", "remote", "service", "ls", "--enc=json") + assert.False(t, gjson.Get(res.Stdout.String(), `RemoteServices.#(Service == "svc").Stat`).Exists()) + }) + }) + + t.Run("adding service with invalid URL fails", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + + res := node.RunIPFS("pin", "remote", "service", "add", "svc", "invalid-service.example.com", "key") + assert.Equal(t, 1, res.ExitCode()) + assert.Contains(t, res.Stderr.String(), "service endpoint must be a valid HTTP URL") + + res = node.RunIPFS("pin", "remote", "service", "add", "svc", "xyz://invalid-service.example.com", "key") + assert.Equal(t, 1, res.ExitCode()) + assert.Contains(t, res.Stderr.String(), "service endpoint must be a valid HTTP URL") + }) + + t.Run("unauthorized pinning service calls fail", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + _, svcURL := runPinningService(t, authToken) + + node.IPFS("pin", "remote", "service", "add", "svc", svcURL, "othertoken") + + res := node.RunIPFS("pin", "remote", "ls", "--service=svc") + assert.Equal(t, 1, res.ExitCode()) + assert.Contains(t, res.Stderr.String(), "access denied") + }) + + t.Run("pinning service calls fail when there is a wrong path", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + _, svcURL := runPinningService(t, authToken) + node.IPFS("pin", "remote", "service", "add", "svc", svcURL+"/invalid-path", authToken) + + res := node.RunIPFS("pin", "remote", "ls", "--service=svc") + assert.Equal(t, 1, res.ExitCode()) + assert.Contains(t, res.Stderr.String(), "404") + }) + + t.Run("pinning service calls fail when DNS resolution fails", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + node.IPFS("pin", "remote", "service", "add", "svc", "https://invalid-service.example.com", authToken) + + res := node.RunIPFS("pin", "remote", "ls", "--service=svc") + assert.Equal(t, 1, res.ExitCode()) + assert.Contains(t, res.Stderr.String(), "no such host") + }) + + t.Run("pin remote service rm", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + node.IPFS("pin", "remote", "service", "add", "svc", "https://example.com", authToken) + node.IPFS("pin", "remote", "service", "rm", "svc") + res := node.IPFS("pin", "remote", "service", "ls") + assert.NotContains(t, res.Stdout.String(), "svc") + }) + + t.Run("remote pinning", func(t *testing.T) { + t.Parallel() + + verifyStatus := func(node *harness.Node, name, hash, status string) { + resJSON := node.IPFS("pin", "remote", "ls", + "--service=svc", + "--enc=json", + "--name="+name, + "--status="+status, + ).Stdout.String() + + assert.Equal(t, status, gjson.Get(resJSON, "Status").Str) + assert.Equal(t, hash, gjson.Get(resJSON, "Cid").Str) + assert.Equal(t, name, gjson.Get(resJSON, "Name").Str) + } + + t.Run("'ipfs pin remote add --background=true'", func(t *testing.T) { + node := harness.NewT(t).NewNode().Init().StartDaemon() + svc, svcURL := runPinningService(t, authToken) + node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken) + + // retain a ptr to the pin that's in the DB so we can directly mutate its status + // to simulate async work + pinCh := make(chan *pinningservice.PinStatus, 1) + svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) { + pinCh <- pin + } + + hash := node.IPFSAddStr("foo") + node.IPFS("pin", "remote", "add", + "--background=true", + "--service=svc", + "--name=pin1", + hash, + ) + + pin := <-pinCh + + transitionStatus := func(status string) { + pin.M.Lock() + pin.Status = status + pin.M.Unlock() + } + + verifyStatus(node, "pin1", hash, "queued") + + transitionStatus("pinning") + verifyStatus(node, "pin1", hash, "pinning") + + transitionStatus("pinned") + verifyStatus(node, "pin1", hash, "pinned") + + transitionStatus("failed") + verifyStatus(node, "pin1", hash, "failed") + }) + + t.Run("'ipfs pin remote add --background=false'", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + svc, svcURL := runPinningService(t, authToken) + node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken) + + svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) { + pin.M.Lock() + defer pin.M.Unlock() + pin.Status = "pinned" + } + hash := node.IPFSAddStr("foo") + node.IPFS("pin", "remote", "add", + "--background=false", + "--service=svc", + "--name=pin2", + hash, + ) + verifyStatus(node, "pin2", hash, "pinned") + }) + + t.Run("'ipfs pin remote ls' with multiple statuses", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + svc, svcURL := runPinningService(t, authToken) + node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken) + + hash := node.IPFSAddStr("foo") + desiredStatuses := map[string]string{ + "pin-queued": "queued", + "pin-pinning": "pinning", + "pin-pinned": "pinned", + "pin-failed": "failed", + } + var pins []*pinningservice.PinStatus + svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) { + pin.M.Lock() + defer pin.M.Unlock() + pins = append(pins, pin) + // this must be "pinned" for the 'pin remote add' command to return + // after 'pin remote add', we change the status to its real status + pin.Status = "pinned" + } + + for pinName := range desiredStatuses { + node.IPFS("pin", "remote", "add", + "--service=svc", + "--name="+pinName, + hash, + ) + } + for _, pin := range pins { + pin.M.Lock() + pin.Status = desiredStatuses[pin.Pin.Name] + pin.M.Unlock() + } + + res := node.IPFS("pin", "remote", "ls", + "--service=svc", + "--status=queued,pinning,pinned,failed", + "--enc=json", + ) + actualStatuses := map[string]string{} + for _, line := range res.Stdout.Lines() { + name := gjson.Get(line, "Name").Str + status := gjson.Get(line, "Status").Str + // drop statuses of other pins we didn't add + if _, ok := desiredStatuses[name]; ok { + actualStatuses[name] = status + } + } + assert.Equal(t, desiredStatuses, actualStatuses) + }) + + t.Run("'ipfs pin remote ls' by CID", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + svc, svcURL := runPinningService(t, authToken) + node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken) + + transitionedCh := make(chan struct{}, 1) + svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) { + pin.M.Lock() + defer pin.M.Unlock() + pin.Status = "pinned" + transitionedCh <- struct{}{} + } + hash := node.IPFSAddStr(string(testutils.RandomBytes(1000))) + node.IPFS("pin", "remote", "add", "--background=false", "--service=svc", hash) + <-transitionedCh + res := node.IPFS("pin", "remote", "ls", "--service=svc", "--cid="+hash, "--enc=json").Stdout.String() + assert.Contains(t, res, hash) + }) + + t.Run("'ipfs pin remote rm --name' without --force when multiple pins match", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + svc, svcURL := runPinningService(t, authToken) + node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken) + + svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) { + pin.M.Lock() + defer pin.M.Unlock() + pin.Status = "pinned" + } + hash := node.IPFSAddStr(string(testutils.RandomBytes(1000))) + node.IPFS("pin", "remote", "add", "--service=svc", "--name=force-test-name", hash) + node.IPFS("pin", "remote", "add", "--service=svc", "--name=force-test-name", hash) + + t.Run("fails", func(t *testing.T) { + res := node.RunIPFS("pin", "remote", "rm", "--service=svc", "--name=force-test-name") + assert.Equal(t, 1, res.ExitCode()) + assert.Contains(t, res.Stderr.String(), "Error: multiple remote pins are matching this query, add --force to confirm the bulk removal") + }) + + t.Run("matching pins are not removed", func(t *testing.T) { + lines := node.IPFS("pin", "remote", "ls", "--service=svc", "--name=force-test-name").Stdout.Lines() + assert.Contains(t, lines[0], "force-test-name") + assert.Contains(t, lines[1], "force-test-name") + }) + }) + + t.Run("'ipfs pin remote rm --name --force' remove multiple pins", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + svc, svcURL := runPinningService(t, authToken) + node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken) + + svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) { + pin.M.Lock() + defer pin.M.Unlock() + pin.Status = "pinned" + } + hash := node.IPFSAddStr(string(testutils.RandomBytes(1000))) + node.IPFS("pin", "remote", "add", "--service=svc", "--name=force-test-name", hash) + node.IPFS("pin", "remote", "add", "--service=svc", "--name=force-test-name", hash) + + node.IPFS("pin", "remote", "rm", "--service=svc", "--name=force-test-name", "--force") + out := node.IPFS("pin", "remote", "ls", "--service=svc", "--name=force-test-name").Stdout.Trimmed() + assert.Empty(t, out) + }) + + t.Run("'ipfs pin remote rm --force' removes all pins", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + svc, svcURL := runPinningService(t, authToken) + node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken) + + svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) { + pin.M.Lock() + defer pin.M.Unlock() + pin.Status = "pinned" + } + for i := 0; i < 4; i++ { + hash := node.IPFSAddStr(string(testutils.RandomBytes(1000))) + name := fmt.Sprintf("--name=%d", i) + node.IPFS("pin", "remote", "add", "--service=svc", "--name="+name, hash) + } + + lines := node.IPFS("pin", "remote", "ls", "--service=svc").Stdout.Lines() + assert.Len(t, lines, 4) + + node.IPFS("pin", "remote", "rm", "--service=svc", "--force") + + lines = node.IPFS("pin", "remote", "ls", "--service=svc").Stdout.Lines() + assert.Len(t, lines, 0) + }) + }) + + t.Run("'ipfs pin remote add' shows a warning message when offline", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + _, svcURL := runPinningService(t, authToken) + node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken) + + hash := node.IPFSAddStr(string(testutils.RandomBytes(1000))) + res := node.IPFS("pin", "remote", "add", "--service=svc", "--background", hash) + warningMsg := "WARNING: the local node is offline and remote pinning may fail if there is no other provider for this CID" + assert.Contains(t, res.Stdout.String(), warningMsg) + }) +} diff --git a/test/cli/testutils/pinningservice/pinning.go b/test/cli/testutils/pinningservice/pinning.go new file mode 100644 index 00000000000..6bfd4ed4ece --- /dev/null +++ b/test/cli/testutils/pinningservice/pinning.go @@ -0,0 +1,401 @@ +package pinningservice + +import ( + "encoding/json" + "fmt" + "net/http" + "reflect" + "strconv" + "strings" + "sync" + "time" + + "github.com/google/uuid" + "github.com/julienschmidt/httprouter" +) + +func NewRouter(authToken string, svc *PinningService) http.Handler { + router := httprouter.New() + router.GET("/api/v1/pins", svc.listPins) + router.POST("/api/v1/pins", svc.addPin) + router.GET("/api/v1/pins/:requestID", svc.getPin) + router.POST("/api/v1/pins/:requestID", svc.replacePin) + router.DELETE("/api/v1/pins/:requestID", svc.removePin) + + handler := authHandler(authToken, router) + + return handler +} + +func authHandler(authToken string, delegate http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + authz := r.Header.Get("Authorization") + if !strings.HasPrefix(authz, "Bearer ") { + errResp(w, "invalid authorization token, must start with 'Bearer '", "", http.StatusBadRequest) + return + } + + token := strings.TrimPrefix(authz, "Bearer ") + if token != authToken { + errResp(w, "access denied", "", http.StatusUnauthorized) + return + } + + delegate.ServeHTTP(w, r) + }) +} + +func New() *PinningService { + return &PinningService{ + PinAdded: func(*AddPinRequest, *PinStatus) {}, + } +} + +// PinningService is a basic pinning service that implements the Remote Pinning API, for testing Kubo's integration with remote pinning services. +// Pins are not persisted, they are just kept in-memory, and this provides callbacks for controlling the behavior of the pinning service. +type PinningService struct { + m sync.Mutex + // PinAdded is a callback that is invoked after a new pin is added via the API. + PinAdded func(*AddPinRequest, *PinStatus) + pins []*PinStatus +} + +type Pin struct { + CID string `json:"cid"` + Name string `json:"name"` + Origins []string `json:"origins"` + Meta map[string]interface{} `json:"meta"` +} + +type PinStatus struct { + M sync.Mutex + RequestID string + Status string + Created time.Time + Pin Pin + Delegates []string + Info map[string]interface{} +} + +func (p *PinStatus) MarshalJSON() ([]byte, error) { + type pinStatusJSON struct { + RequestID string `json:"requestid"` + Status string `json:"status"` + Created time.Time `json:"created"` + Pin Pin `json:"pin"` + Delegates []string `json:"delegates"` + Info map[string]interface{} `json:"info"` + } + // lock the pin before marshaling it to protect against data races while marshaling + p.M.Lock() + pinJSON := pinStatusJSON{ + RequestID: p.RequestID, + Status: p.Status, + Created: p.Created, + Pin: p.Pin, + Delegates: p.Delegates, + Info: p.Info, + } + p.M.Unlock() + return json.Marshal(pinJSON) +} + +func (p *PinStatus) Clone() PinStatus { + return PinStatus{ + RequestID: p.RequestID, + Status: p.Status, + Created: p.Created, + Pin: p.Pin, + Delegates: p.Delegates, + Info: p.Info, + } +} + +const ( + matchExact = "exact" + matchIExact = "iexact" + matchPartial = "partial" + matchIPartial = "ipartial" + + statusQueued = "queued" + statusPinning = "pinning" + statusPinned = "pinned" + statusFailed = "failed" + + timeLayout = "2006-01-02T15:04:05.999Z" +) + +func errResp(w http.ResponseWriter, reason, details string, statusCode int) { + type errorObj struct { + Reason string `json:"reason"` + Details string `json:"details"` + } + type errorResp struct { + Error errorObj `json:"error"` + } + resp := errorResp{ + Error: errorObj{ + Reason: reason, + Details: details, + }, + } + writeJSON(w, resp, statusCode) +} + +func writeJSON(w http.ResponseWriter, val any, statusCode int) { + b, err := json.Marshal(val) + if err != nil { + w.Header().Set("Content-Type", "text/plain") + errResp(w, fmt.Sprintf("marshaling response: %s", err), "", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + _, _ = w.Write(b) +} + +type AddPinRequest struct { + CID string `json:"cid"` + Name string `json:"name"` + Origins []string `json:"origins"` + Meta map[string]interface{} `json:"meta"` +} + +func (p *PinningService) addPin(writer http.ResponseWriter, req *http.Request, params httprouter.Params) { + var addReq AddPinRequest + err := json.NewDecoder(req.Body).Decode(&addReq) + if err != nil { + errResp(writer, fmt.Sprintf("unmarshaling req: %s", err), "", http.StatusBadRequest) + return + } + + pin := &PinStatus{ + RequestID: uuid.NewString(), + Status: statusQueued, + Created: time.Now(), + Pin: Pin(addReq), + } + + p.m.Lock() + p.pins = append(p.pins, pin) + p.m.Unlock() + + writeJSON(writer, &pin, http.StatusAccepted) + p.PinAdded(&addReq, pin) +} + +type ListPinsResponse struct { + Count int `json:"count"` + Results []*PinStatus `json:"results"` +} + +func (p *PinningService) listPins(writer http.ResponseWriter, req *http.Request, params httprouter.Params) { + q := req.URL.Query() + + cidStr := q.Get("cid") + name := q.Get("name") + match := q.Get("match") + status := q.Get("status") + beforeStr := q.Get("before") + afterStr := q.Get("after") + limitStr := q.Get("limit") + metaStr := q.Get("meta") + + if limitStr == "" { + limitStr = "10" + } + limit, err := strconv.Atoi(limitStr) + if err != nil { + errResp(writer, fmt.Sprintf("parsing limit: %s", err), "", http.StatusBadRequest) + return + } + + var cids []string + if cidStr != "" { + cids = strings.Split(cidStr, ",") + } + + var statuses []string + if status != "" { + statuses = strings.Split(status, ",") + } + + p.m.Lock() + defer p.m.Unlock() + var pins []*PinStatus + for _, pinStatus := range p.pins { + // clone it so we can immediately release the lock + pinStatus.M.Lock() + clonedPS := pinStatus.Clone() + pinStatus.M.Unlock() + + // cid + var matchesCID bool + if len(cids) == 0 { + matchesCID = true + } else { + for _, cid := range cids { + if cid == clonedPS.Pin.CID { + matchesCID = true + } + } + } + if !matchesCID { + continue + } + + // name + if match == "" { + match = matchExact + } + if name != "" { + switch match { + case matchExact: + if name != clonedPS.Pin.Name { + continue + } + case matchIExact: + if !strings.EqualFold(name, clonedPS.Pin.Name) { + continue + } + case matchPartial: + if !strings.Contains(clonedPS.Pin.Name, name) { + continue + } + case matchIPartial: + if !strings.Contains(strings.ToLower(clonedPS.Pin.Name), strings.ToLower(name)) { + continue + } + default: + errResp(writer, fmt.Sprintf("unknown match %q", match), "", http.StatusBadRequest) + return + } + } + + // status + var matchesStatus bool + if len(statuses) == 0 { + statuses = []string{statusPinned} + } + for _, status := range statuses { + if status == clonedPS.Status { + matchesStatus = true + } + } + if !matchesStatus { + continue + } + + // before + if beforeStr != "" { + before, err := time.Parse(timeLayout, beforeStr) + if err != nil { + errResp(writer, fmt.Sprintf("parsing before: %s", err), "", http.StatusBadRequest) + return + } + if !clonedPS.Created.Before(before) { + continue + } + } + + // after + if afterStr != "" { + after, err := time.Parse(timeLayout, afterStr) + if err != nil { + errResp(writer, fmt.Sprintf("parsing before: %s", err), "", http.StatusBadRequest) + return + } + if !clonedPS.Created.After(after) { + continue + } + } + + // meta + if metaStr != "" { + meta := map[string]interface{}{} + err := json.Unmarshal([]byte(metaStr), &meta) + if err != nil { + errResp(writer, fmt.Sprintf("parsing meta: %s", err), "", http.StatusBadRequest) + return + } + var matchesMeta bool + for k, v := range meta { + pinV, contains := clonedPS.Pin.Meta[k] + if !contains || !reflect.DeepEqual(pinV, v) { + matchesMeta = false + break + } + } + if !matchesMeta { + continue + } + } + + // add the original pin status, not the cloned one + pins = append(pins, pinStatus) + + if len(pins) == limit { + break + } + } + + out := ListPinsResponse{ + Count: len(pins), + Results: pins, + } + writeJSON(writer, out, http.StatusOK) +} + +func (p *PinningService) getPin(writer http.ResponseWriter, req *http.Request, params httprouter.Params) { + requestID := params.ByName("requestID") + p.m.Lock() + defer p.m.Unlock() + for _, pin := range p.pins { + if pin.RequestID == requestID { + writeJSON(writer, pin, http.StatusOK) + return + } + } + errResp(writer, "", "", http.StatusNotFound) +} + +func (p *PinningService) replacePin(writer http.ResponseWriter, req *http.Request, params httprouter.Params) { + requestID := params.ByName("requestID") + + var replaceReq Pin + err := json.NewDecoder(req.Body).Decode(&replaceReq) + if err != nil { + errResp(writer, fmt.Sprintf("decoding request: %s", err), "", http.StatusBadRequest) + return + } + + p.m.Lock() + defer p.m.Unlock() + for _, pin := range p.pins { + if pin.RequestID == requestID { + pin.M.Lock() + pin.Pin = replaceReq + pin.M.Unlock() + writer.WriteHeader(http.StatusAccepted) + return + } + } + errResp(writer, "", "", http.StatusNotFound) +} + +func (p *PinningService) removePin(writer http.ResponseWriter, req *http.Request, params httprouter.Params) { + requestID := params.ByName("requestID") + + p.m.Lock() + defer p.m.Unlock() + + for i, pin := range p.pins { + if pin.RequestID == requestID { + p.pins = append(p.pins[0:i], p.pins[i+1:]...) + writer.WriteHeader(http.StatusAccepted) + return + } + } + + errResp(writer, "", "", http.StatusNotFound) +} diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh deleted file mode 100755 index 2566c06d899..00000000000 --- a/test/sharness/t0700-remotepin.sh +++ /dev/null @@ -1,332 +0,0 @@ -#!/usr/bin/env bash - -test_description="Test ipfs remote pinning operations" - -. lib/test-lib.sh - -if [ -z ${TEST_DOCKER_HOST+x} ]; then - # TODO: set up instead of skipping? - skip_all='Skipping pinning service integration tests: missing TEST_DOCKER_HOST, remote pinning service not available' - test_done -fi - -# daemon running in online mode to ensure Pin.origins/PinStatus.delegates work -test_init_ipfs -test_launch_ipfs_daemon - -# create user on pinning service -TEST_PIN_SVC="http://${TEST_DOCKER_HOST}:5000/api/v1" -TEST_PIN_SVC_KEY=$(curl -s -X POST "$TEST_PIN_SVC/users" -d email="go-ipfs-sharness@ipfs.example.com" | jq --raw-output .access_token) - -# pin remote service add|ls|rm - -# confirm empty service list response has proper json struct -# https://github.com/ipfs/go-ipfs/pull/7829 -test_expect_success "test 'ipfs pin remote service ls' JSON on empty list" ' - ipfs pin remote service ls --stat --enc=json | tee empty_ls_out && - echo "{\"RemoteServices\":[]}" > exp_ls_out && - test_cmp exp_ls_out empty_ls_out -' - -# add valid and invalid services -test_expect_success "creating test user on remote pinning service" ' - echo CI host IP address ${TEST_PIN_SVC} && - ipfs pin remote service add test_pin_svc ${TEST_PIN_SVC} ${TEST_PIN_SVC_KEY} && - ipfs pin remote service add test_invalid_key_svc ${TEST_PIN_SVC} fake_api_key && - ipfs pin remote service add test_invalid_url_path_svc ${TEST_PIN_SVC}/invalid-path fake_api_key && - ipfs pin remote service add test_invalid_url_dns_svc https://invalid-service.example.com fake_api_key && - ipfs pin remote service add test_pin_mfs_svc ${TEST_PIN_SVC} ${TEST_PIN_SVC_KEY} -' - -# add a service with a invalid endpoint -test_expect_success "adding remote service with invalid endpoint" ' - test_expect_code 1 ipfs pin remote service add test_endpoint_no_protocol invalid-service.example.com fake_api_key && - test_expect_code 1 ipfs pin remote service add test_endpoint_bad_protocol xyz://invalid-service.example.com fake_api_key -' - -test_expect_success "test 'ipfs pin remote service ls'" ' - ipfs pin remote service ls | tee ls_out && - grep -q test_pin_svc ls_out && - grep -q test_invalid_key_svc ls_out && - grep -q test_invalid_url_path_svc ls_out && - grep -q test_invalid_url_dns_svc ls_out -' - -test_expect_success "test enabling mfs pinning" ' - ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.RepinInterval \"10s\" && - ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.PinName \"mfs_test_pin\" && - ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.Enable true && - ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.RepinInterval > repin_interval && - ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.PinName > pin_name && - ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.Enable > enable && - echo 10s > expected_repin_interval && - echo mfs_test_pin > expected_pin_name && - echo true > expected_enable && - test_cmp repin_interval expected_repin_interval && - test_cmp pin_name expected_pin_name && - test_cmp enable expected_enable -' - -# expect PIN to be created -test_expect_success "verify MFS root is being pinned" ' - ipfs files cp /ipfs/bafkqaaa /mfs-pinning-test-$(date +%s.%N) && - ipfs files flush && - sleep 31 && - ipfs files stat / --enc=json | jq -r .Hash > mfs_cid && - ipfs pin remote ls --service=test_pin_mfs_svc --name=mfs_test_pin --status=queued,pinning,pinned,failed --enc=json | tee ls_out | jq -r .Cid > pin_cid && - cat mfs_cid ls_out && - test_cmp mfs_cid pin_cid -' - -# expect existing PIN to be replaced -test_expect_success "verify MFS root is being repinned on CID change" ' - ipfs files cp /ipfs/bafkqaaa /mfs-pinning-repin-test-$(date +%s.%N) && - ipfs files flush && - sleep 31 && - ipfs files stat / --enc=json | jq -r .Hash > mfs_cid && - ipfs pin remote ls --service=test_pin_mfs_svc --name=mfs_test_pin --status=queued,pinning,pinned,failed --enc=json | tee ls_out | jq -r .Cid > pin_cid && - cat mfs_cid ls_out && - test_cmp mfs_cid pin_cid -' - -# SECURITY of access tokens in API.Key fields: -# Pinning.RemoteServices includes API.Key, and we give it the same treatment -# as Identity.PrivKey to prevent exposing it on the network - -test_expect_success "'ipfs config Pinning' fails" ' - test_expect_code 1 ipfs config Pinning 2>&1 > config_out -' -test_expect_success "output does not include API.Key" ' - test_expect_code 1 grep -q Key config_out -' - -test_expect_success "'ipfs config Pinning.RemoteServices.test_pin_svc.API.Key' fails" ' - test_expect_code 1 ipfs config Pinning.RemoteServices.test_pin_svc.API.Key 2> config_out -' - -test_expect_success "output includes meaningful error" ' - echo "Error: cannot show or change pinning services credentials" > config_exp && - test_cmp config_exp config_out -' - -test_expect_success "'ipfs config Pinning.RemoteServices.test_pin_svc' fails" ' - test_expect_code 1 ipfs config Pinning.RemoteServices.test_pin_svc 2> config_out -' -test_expect_success "output includes meaningful error" ' - test_cmp config_exp config_out -' - -test_expect_success "'ipfs config show' does not include Pinning.RemoteServices[*].API.Key" ' - ipfs config show | tee show_config | jq -r .Pinning.RemoteServices > remote_services && - test_expect_code 1 grep \"Key\" remote_services && - test_expect_code 1 grep fake_api_key show_config && - test_expect_code 1 grep "$TEST_PIN_SVC_KEY" show_config -' - -test_expect_success "'ipfs config replace' injects Pinning.RemoteServices[*].API.Key back" ' - test_expect_code 1 grep fake_api_key show_config && - test_expect_code 1 grep "$TEST_PIN_SVC_KEY" show_config && - ipfs config replace show_config && - test_expect_code 0 grep fake_api_key "$IPFS_PATH/config" && - test_expect_code 0 grep "$TEST_PIN_SVC_KEY" "$IPFS_PATH/config" -' - -# note: we remove Identity.PrivKey to ensure error is triggered by Pinning.RemoteServices -test_expect_success "'ipfs config replace' with Pinning.RemoteServices[*].API.Key errors out" ' - jq -M "del(.Identity.PrivKey)" "$IPFS_PATH/config" | jq ".Pinning += { RemoteServices: {\"myservice\": {\"API\": {\"Endpoint\": \"https://example.com/psa\", \"Key\": \"mysecret\"}}}}" > new_config && - test_expect_code 1 ipfs config replace - < new_config 2> replace_out -' -test_expect_success "output includes meaningful error" " - echo \"Error: cannot add or remove remote pinning services with 'config replace'\" > replace_expected && - test_cmp replace_out replace_expected -" - -# /SECURITY - -test_expect_success "pin remote service ls --stat' returns numbers for a valid service" ' - ipfs pin remote service ls --stat | grep -E "^test_pin_svc.+[0-9]+/[0-9]+/[0-9]+/[0-9]+$" -' - -test_expect_success "pin remote service ls --enc=json --stat' returns valid status" " - ipfs pin remote service ls --stat --enc=json | jq --raw-output '.RemoteServices[] | select(.Service == \"test_pin_svc\") | .Stat.Status' | tee stat_out && - echo valid > stat_expected && - test_cmp stat_out stat_expected -" - -test_expect_success "pin remote service ls --stat' returns invalid status for invalid service" ' - ipfs pin remote service ls --stat | grep -E "^test_invalid_url_path_svc.+invalid$" -' - -test_expect_success "pin remote service ls --enc=json --stat' returns invalid status" " - ipfs pin remote service ls --stat --enc=json | jq --raw-output '.RemoteServices[] | select(.Service == \"test_invalid_url_path_svc\") | .Stat.Status' | tee stat_out && - echo invalid > stat_expected && - test_cmp stat_out stat_expected -" - -test_expect_success "pin remote service ls --enc=json' (without --stat) returns no Stat object" " - ipfs pin remote service ls --enc=json | jq --raw-output '.RemoteServices[] | select(.Service == \"test_invalid_url_path_svc\") | .Stat' | tee stat_out && - echo null > stat_expected && - test_cmp stat_out stat_expected -" - -test_expect_success "check connection to the test pinning service" ' - ipfs pin remote ls --service=test_pin_svc --enc=json -' - -test_expect_success "unauthorized pinning service calls fail" ' - test_expect_code 1 ipfs pin remote ls --service=test_invalid_key_svc -' - -test_expect_success "misconfigured pinning service calls fail (wrong path)" ' - test_expect_code 1 ipfs pin remote ls --service=test_invalid_url_path_svc -' - -test_expect_success "misconfigured pinning service calls fail (dns error)" ' - test_expect_code 1 ipfs pin remote ls --service=test_invalid_url_dns_svc -' - -# pin remote service rm - -test_expect_success "remove pinning service" ' - ipfs pin remote service rm test_invalid_key_svc && - ipfs pin remote service rm test_invalid_url_path_svc && - ipfs pin remote service rm test_invalid_url_dns_svc -' - -test_expect_success "verify pinning service removal works" ' - ipfs pin remote service ls | tee ls_out && - test_expect_code 1 grep test_invalid_key_svc ls_out && - test_expect_code 1 grep test_invalid_url_path_svc ls_out && - test_expect_code 1 grep test_invalid_url_dns_svc ls_out -' - -# pin remote add - -# we leverage the fact that inlined CID can be pinned instantly on the remote service -# (https://github.com/ipfs-shipyard/rb-pinning-service-api/issues/8) -# below test ensures that assumption is correct (before we proceed to actual tests) -test_expect_success "verify that default add (implicit --background=false) works with data inlined in CID" ' - ipfs pin remote add --service=test_pin_svc --name=inlined_null bafkqaaa && - ipfs pin remote ls --service=test_pin_svc --enc=json --name=inlined_null --status=pinned | jq --raw-output .Status | tee ls_out && - grep -q "pinned" ls_out -' - -test_remote_pins() { - BASE=$1 - if [ -n "$BASE" ]; then - BASE_ARGS="--cid-base=$BASE" - fi - - # note: HAS_MISSING is not inlined nor imported to IPFS on purpose, to reliably test 'queued' state - test_expect_success "create some hashes using base $BASE" ' - export HASH_A=$(echo -n "A @ $(date +%s.%N)" | ipfs add $BASE_ARGS -q --inline --inline-limit 1000 --pin=false) && - export HASH_B=$(echo -n "B @ $(date +%s.%N)" | ipfs add $BASE_ARGS -q --inline --inline-limit 1000 --pin=false) && - export HASH_C=$(echo -n "C @ $(date +%s.%N)" | ipfs add $BASE_ARGS -q --inline --inline-limit 1000 --pin=false) && - export HASH_MISSING=$(echo "MISSING FROM IPFS @ $(date +%s.%N)" | ipfs add $BASE_ARGS -q --only-hash) && - echo "A: $HASH_A" && - echo "B: $HASH_B" && - echo "C: $HASH_C" && - echo "M: $HASH_MISSING" - ' - - test_expect_success "'ipfs pin remote add --background=true'" ' - ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_a $HASH_A - ' - - test_expect_success "verify background add worked (instantly pinned variant)" ' - ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_a | tee ls_out && - test_expect_code 0 grep -q name_a ls_out && - test_expect_code 0 grep -q $HASH_A ls_out - ' - - test_expect_success "'ipfs pin remote add --background=true' with CID that is not available" ' - test_expect_code 0 ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_m $HASH_MISSING - ' - - test_expect_success "verify background add worked (queued variant)" ' - ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_m --status=queued,pinning | tee ls_out && - test_expect_code 0 grep -q name_m ls_out && - test_expect_code 0 grep -q $HASH_MISSING ls_out - ' - - test_expect_success "'ipfs pin remote add --background=false'" ' - test_expect_code 0 ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_B - ' - - test_expect_success "verify foreground add worked" ' - ipfs pin remote ls --service=test_pin_svc --enc=json $ID_B | tee ls_out && - test_expect_code 0 grep -q name_b ls_out && - test_expect_code 0 grep -q pinned ls_out && - test_expect_code 0 grep -q $HASH_B ls_out - ' - - test_expect_success "'ipfs pin remote ls' for existing pins by multiple statuses" ' - ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinning,pinned,failed | tee ls_out && - test_expect_code 0 grep -q $HASH_A ls_out && - test_expect_code 0 grep -q $HASH_B ls_out && - test_expect_code 0 grep -q $HASH_MISSING ls_out - ' - - test_expect_success "'ipfs pin remote ls' for existing pins by CID" ' - ipfs pin remote ls --service=test_pin_svc --enc=json --cid=$HASH_B | tee ls_out && - test_expect_code 0 grep -q $HASH_B ls_out - ' - - test_expect_success "'ipfs pin remote ls' for existing pins by name" ' - ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_a | tee ls_out && - test_expect_code 0 grep -q $HASH_A ls_out - ' - - test_expect_success "'ipfs pin remote ls' for ongoing pins by status" ' - ipfs pin remote ls --service=test_pin_svc --status=queued,pinning | tee ls_out && - test_expect_code 0 grep -q $HASH_MISSING ls_out - ' - - # --force is required only when more than a single match is found, - # so we add second pin with the same name (but different CID) to simulate that scenario - test_expect_success "'ipfs pin remote rm --name' fails without --force when matching multiple pins" ' - test_expect_code 0 ipfs pin remote add --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_C && - test_expect_code 1 ipfs pin remote rm --service=test_pin_svc --name=name_b 2> rm_out && - echo "Error: multiple remote pins are matching this query, add --force to confirm the bulk removal" > rm_expected && - test_cmp rm_out rm_expected - ' - - test_expect_success "'ipfs pin remote rm --name' without --force did not remove matching pins" ' - ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_b | jq --raw-output .Cid | tee ls_out && - test_expect_code 0 grep -q $HASH_B ls_out && - test_expect_code 0 grep -q $HASH_C ls_out - ' - - test_expect_success "'ipfs pin remote rm --name' with --force removes all matching pins" ' - test_expect_code 0 ipfs pin remote rm --service=test_pin_svc --name=name_b --force && - ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_b | jq --raw-output .Cid | tee ls_out && - test_expect_code 1 grep -q $HASH_B ls_out && - test_expect_code 1 grep -q $HASH_C ls_out - ' - - test_expect_success "'ipfs pin remote rm --force' removes all pinned items" ' - ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinning,pinned,failed | jq --raw-output .Cid | tee ls_out && - test_expect_code 0 grep -q $HASH_A ls_out && - test_expect_code 0 grep -q $HASH_MISSING ls_out && - ipfs pin remote rm --service=test_pin_svc --status=queued,pinning,pinned,failed --force && - ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinning,pinned,failed | jq --raw-output .Cid | tee ls_out && - test_expect_code 1 grep -q $HASH_A ls_out && - test_expect_code 1 grep -q $HASH_MISSING ls_out - ' - -} - -test_remote_pins "" - -test_kill_ipfs_daemon - -WARNINGMESSAGE="WARNING: the local node is offline and remote pinning may fail if there is no other provider for this CID" - -test_expect_success "'ipfs pin remote add' shows the warning message while offline" ' - test_expect_code 0 ipfs pin remote add --service=test_pin_svc --background $BASE_ARGS --name=name_a $HASH_A > actual && - test_expect_code 0 grep -q "$WARNINGMESSAGE" actual -' - -test_done - -# vim: ts=2 sw=2 sts=2 et: