diff --git a/server/http/storage_nip_96_test.go b/server/http/storage_nip_96_test.go index 8ce80c8..bf637d8 100644 --- a/server/http/storage_nip_96_test.go +++ b/server/http/storage_nip_96_test.go @@ -57,6 +57,8 @@ func TestNIP96(t *testing.T) { user2, user2PubKey := model.GenerateKeyPair() var tagsToBroadcast nostr.Tags var contentToBroadcast string + var outdatedTags nostr.Tags + var outdatedContent string t.Run("create on-behalf attestations", func(t *testing.T) { var ev model.Event ev.Kind = model.CustomIONKindAttestation @@ -70,24 +72,42 @@ func TestNIP96(t *testing.T) { }) t.Run("files are uploaded, response is ok", func(t *testing.T) { var responses chan *nip96.UploadResponse - responses = make(chan *nip96.UploadResponse, 2) + responses = make(chan *nip96.UploadResponse, 100) upload(t, ctx, user1, masterPubKey, ".testdata/image2.png", "profile.png", "ice profile pic", func(resp *nip96.UploadResponse) { responses <- resp }) upload(t, ctx, user1, masterPubKey, ".testdata/image.jpg", "ice.jpg", "ice logo", func(resp *nip96.UploadResponse) { responses <- resp }) - upload(t, ctx, user2, masterPubKey, ".testdata/image2.png", "profile.png", "ice profile pic", func(resp *nip96.UploadResponse) {}) - upload(t, ctx, user2, masterPubKey, ".testdata/image.jpg", "ice.jpg", "ice logo", func(resp *nip96.UploadResponse) {}) - upload(t, ctx, user2, masterPubKey, ".testdata/text.txt", "text.txt", "text file", func(resp *nip96.UploadResponse) {}) - upload(t, ctx, master, "", ".testdata/text-master.txt", "master.txt", "master's file", func(resp *nip96.UploadResponse) {}) + upload(t, ctx, master, "", ".testdata/text-master.txt", "master.txt", "master's file", func(resp *nip96.UploadResponse) { responses <- resp }) + upload(t, ctx, user1, masterPubKey, ".testdata/text.txt", "text.txt", "text file", func(resp *nip96.UploadResponse) { responses <- resp }) close(responses) + i := 0 for resp := range responses { + if i == 0 { + outdatedTags = resp.Nip94Event.Tags + outdatedContent = resp.Nip94Event.Content + } verifyFile(t, resp.Nip94Event.Content, resp.Nip94Event.Tags) tagsToBroadcast = resp.Nip94Event.Tags contentToBroadcast = resp.Nip94Event.Content + i += 1 } }) + var outdatedNip94EventToSign *model.Event + t.Run("nip-94 event is accepted on the same relay it was uploaded to = no-op", func(t *testing.T) { + outdatedTags = outdatedTags.AppendUnique(model.Tag{"b", masterPubKey}) + outdatedNip94EventToSign = &model.Event{Event: nostr.Event{ + CreatedAt: nostr.Timestamp(time.Now().Unix()), + Kind: nostr.KindFileMetadata, + Tags: outdatedTags, + Content: outdatedContent, + }} + require.NoError(t, outdatedNip94EventToSign.SignWithAlg(user1, model.SignAlgEDDSA, model.KeyAlgCurve25519)) + require.NoError(t, query.AcceptEvents(ctx, outdatedNip94EventToSign)) + require.NoError(t, storage.AcceptEvents(ctx, outdatedNip94EventToSign)) + time.Sleep(3 * time.Second) + }) const newStorageRoot = "./../../.test-uploads2" var nip94EventToSign *model.Event t.Run("nip-94 event is broadcasted, it causes download to other node", func(t *testing.T) { @@ -153,20 +173,20 @@ func TestNIP96(t *testing.T) { }) t.Run("delete file owned by user 1 on behave of usr 1 (normally)", func(t *testing.T) { fileHash := "" - if xTag := nip94EventToSign.Tags.GetFirst([]string{"x"}); xTag != nil && len(*xTag) > 1 { + if xTag := outdatedNip94EventToSign.Tags.GetFirst([]string{"x"}); xTag != nil && len(*xTag) > 1 { fileHash = xTag.Value() } else { t.Fatalf("malformed x tag in nip94 event %v", nip94EventToSign.ID) } status := deleteFile(t, ctx, user1, fileHash, masterPubKey) require.Equal(t, http.StatusOK, status) - fileName := nip94.ParseFileMetadata(nostr.Event{Tags: expectedResponse(nip94EventToSign.Content).Nip94Event.Tags}).Summary + fileName := nip94.ParseFileMetadata(nostr.Event{Tags: expectedResponse(outdatedNip94EventToSign.Content).Nip94Event.Tags}).Summary require.NoFileExists(t, filepath.Join(storageRoot, masterPubKey, fileName)) deletionEventToSign := &model.Event{Event: nostr.Event{ CreatedAt: nostr.Timestamp(time.Now().Unix()), Kind: nostr.KindDeletion, Tags: nostr.Tags{ - nostr.Tag{"e", nip94EventToSign.ID}, + nostr.Tag{"e", outdatedNip94EventToSign.ID}, nostr.Tag{"k", strconv.FormatInt(int64(nostr.KindFileMetadata), 10)}, nostr.Tag{"b", masterPubKey}, }, @@ -175,8 +195,8 @@ func TestNIP96(t *testing.T) { require.NoError(t, storage.AcceptEvents(ctx, deletionEventToSign)) require.NoFileExists(t, filepath.Join(newStorageRoot, masterPubKey, fileName)) }) - t.Run("delete file owned by user 2 on behave of usr 1 (attestation)", func(t *testing.T) { - status := deleteFile(t, ctx, user1, "982d9e3eb996f559e633f4d194def3761d909f5a3b647d1a851fead67c32c9d1", masterPubKey) + t.Run("delete file owned by user 1 on behave of usr 2 (attestation)", func(t *testing.T) { + status := deleteFile(t, ctx, user2, "982d9e3eb996f559e633f4d194def3761d909f5a3b647d1a851fead67c32c9d1", masterPubKey) require.Equal(t, http.StatusOK, status) fileName := "text.txt" require.NoFileExists(t, filepath.Join(storageRoot, masterPubKey, fileName)) @@ -189,7 +209,7 @@ func TestNIP96(t *testing.T) { nostr.Tag{"b", masterPubKey}, }, }} - require.NoError(t, deletionEventToSign.SignWithAlg(user1, model.SignAlgEDDSA, model.KeyAlgCurve25519)) + require.NoError(t, deletionEventToSign.SignWithAlg(user2, model.SignAlgEDDSA, model.KeyAlgCurve25519)) require.NoError(t, storage.AcceptEvents(ctx, deletionEventToSign)) require.NoFileExists(t, filepath.Join(newStorageRoot, masterPubKey, fileName)) }) diff --git a/storage/download.go b/storage/download.go index cec0eee..0c3b2e3 100644 --- a/storage/download.go +++ b/storage/download.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "encoding/json" "log" + "strconv" "strings" "time" @@ -42,6 +43,7 @@ func (c *client) DownloadUrl(masterPubkey string, fileHash string) (string, erro } func acceptNewBag(ctx context.Context, event *model.Event) error { + log.Printf("[STORAGE] INFO: ACCEPT NIP-94 with new files for user %v: %v", event.GetMasterPublicKey(), event.String()) infohash := "" var err error if iTag := event.Tags.GetFirst([]string{"i"}); iTag != nil && len(*iTag) > 1 { @@ -50,24 +52,29 @@ func acceptNewBag(ctx context.Context, event *model.Event) error { return errors.Newf("malformed i tag %v", iTag) } - bootstrap := "" spl := strings.Split(infohash, ":") - if len(spl) == 2 { - infohash = spl[0] - bootstrap = spl[1] + if len(spl) != 3 { + return errors.Newf("malformed i tag %v, cannot detect bootstrap and createdAt", infohash) } - if err = globalClient.newBagIDPromoted(ctx, event.GetMasterPublicKey(), infohash, &bootstrap); err != nil { + infohash = spl[0] + bootstrap := spl[1] + createdAt, cErr := strconv.ParseInt(spl[2], 10, 64) + if cErr != nil { + return errors.Wrapf(err, "malformed i tag %v, cannot createdAt", infohash) + } + + if err = globalClient.newBagIDPromoted(ctx, event.GetMasterPublicKey(), infohash, &bootstrap, createdAt); err != nil { return errors.Wrapf(err, "failed to promote new bag ID %v for user %v", infohash, event.PubKey) } return nil } -func (c *client) newBagIDPromoted(ctx context.Context, user, bagID string, bootstap *string) error { +func (c *client) newBagIDPromoted(ctx context.Context, user, bagID string, bootstap *string, newCreatedAt int64) error { existingBagForUser, err := c.bagByUser(user) if err != nil { return errors.Wrapf(err, "failed to find existing bag for user %s", user) } - if existingBagForUser != nil && hex.EncodeToString(existingBagForUser.BagID) != bagID { + if existingBagForUser != nil && hex.EncodeToString(existingBagForUser.BagID) != bagID && existingBagForUser.CreatedAt.UnixNano() < newCreatedAt { log.Printf("[STORAGE] INFO: GOT NIP-94 with new files for user %v, replacing %v with %v", user, hex.EncodeToString(existingBagForUser.BagID), bagID) existingBagForUser.Stop() if err = c.progressStorage.RemoveTorrent(existingBagForUser, false); err != nil { @@ -199,6 +206,14 @@ func (c *client) saveTorrent(tr *storage.Torrent, userPubKey *string, bs *string return errors.Wrapf(err, "failed to save bootstrap node for bag %v", hex.EncodeToString(tr.BagID)) } } + if tr.Header != nil && len(tr.Header.Data) > 0 { + k := make([]byte, 3+32) + copy(k, "th:") + copy(k[3:], tr.BagID) + if err := c.db.Put(k, []byte(tr.Header.Data), nil); err != nil { + return errors.Wrapf(err, "failed to save header for bag %v", hex.EncodeToString(tr.BagID)) + } + } return nil } diff --git a/storage/global.go b/storage/global.go index 6217724..220dfe9 100644 --- a/storage/global.go +++ b/storage/global.go @@ -99,7 +99,7 @@ func acceptDeletion(ctx context.Context, event *model.Event) error { if fileEvent.Kind != nostr.KindFileMetadata { return errors.Errorf("event mismatch: event %v is %v not file metadata (%v)", fileEvent.ID, fileEvent.Kind, nostr.KindFileMetadata) } - if fileEvent.PubKey != event.PubKey { + if fileEvent.GetMasterPublicKey() != event.GetMasterPublicKey() { return errors.Errorf("user mismatch: event %v is signed by %v not %v", fileEvent.ID, fileEvent.PubKey, event.PubKey) } originalEvent = fileEvent @@ -108,6 +108,7 @@ func acceptDeletion(ctx context.Context, event *model.Event) error { if originalEvent == nil { return nil } + log.Printf("[STORAGE] INFO: ACCEPT FILE DELETION OF NIP-94 for user %v: %v, original event %v", event.GetMasterPublicKey(), event.String(), originalEvent.String()) fileHash := "" if xTag := originalEvent.Tags.GetFirst([]string{"x"}); xTag != nil && len(*xTag) > 1 { fileHash = xTag.Value() diff --git a/storage/storage.go b/storage/storage.go index ae32c97..d24be5a 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -90,10 +90,20 @@ var ( func (c *client) fileMeta(bag *storage.Torrent) (*headerData, error) { var desc headerData + var hData []byte if bag.Header == nil { - return nil, errors.Errorf("No header fetched yet for %v", hex.EncodeToString(bag.BagID)) + var err error + hData, err = c.latestHeaderForBag(bag.BagID) + if err != nil { + return nil, errors.Wrapf(err, "No header fetched yet for %v and failed to get stored", hex.EncodeToString(bag.BagID)) + } + if hData == nil { + return nil, errors.Errorf("No header fetched yet for %v", hex.EncodeToString(bag.BagID)) + } + } else { + hData = bag.Header.Data } - hData := bag.Header.Data + if len(hData) == 0 { hData = []byte("{}") } @@ -142,6 +152,16 @@ func (c *client) bootstrapForBag(bagID []byte) (string, error) { } return string(bs), nil } +func (c *client) latestHeaderForBag(bagID []byte) ([]byte, error) { + k := make([]byte, 3+32) + copy(k, "th:") + copy(k[3:], bagID) + th, err := c.db.Get(k, nil) + if err != nil && !errors.Is(err, leveldb.ErrNotFound) { + return nil, errors.Wrapf(err, "failed to read stored header for %v, will wait downloading header", hex.EncodeToString(bagID)) + } + return th, nil +} func (c *client) BuildUserPath(userPubKey string, contentType string) (userStorage string, uploadPath string) { spl := strings.Split(contentType, "/") diff --git a/storage/upload.go b/storage/upload.go index ef8940c..cad766b 100644 --- a/storage/upload.go +++ b/storage/upload.go @@ -12,6 +12,7 @@ import ( "log" "net/url" "path/filepath" + "strconv" "sync" "time" @@ -29,9 +30,17 @@ func (c *client) StartUpload(ctx context.Context, userPubKey, masterPubKey, rela if err != nil { return "", "", false, errors.Wrapf(err, "failed to find existing bag for user %s", masterPubKey) } + var existingHDData []byte var existingHD headerData if existingBagForUser != nil { - if len(existingBagForUser.Header.Data) > 0 { + if existingBagForUser.Header != nil && len(existingBagForUser.Header.Data) > 0 { + existingHDData = existingBagForUser.Header.Data + } else { + if existingHDData, err = c.latestHeaderForBag(existingBagForUser.BagID); err != nil { + return "", "", false, errors.Wrapf(err, "failed to get header for bag %v", hex.EncodeToString(existingBagForUser.BagID)) + } + } + if len(existingHDData) > 0 { if err = json.Unmarshal(existingBagForUser.Header.Data, &existingHD); err != nil { return "", "", false, errors.Wrapf(err, "corrupted header metadata for bag %v", hex.EncodeToString(existingBagForUser.BagID)) } @@ -80,7 +89,7 @@ func (c *client) StartUpload(ctx context.Context, userPubKey, masterPubKey, rela return "", "", false, errors.Wrapf(err, "failed to build url for %v (bag %v)", relativePathToFileForUrl, bagID) } - return bagID + ":" + bootstrap, url, existed, err + return bagID + ":" + bootstrap + ":" + strconv.FormatInt(bag.CreatedAt.UnixNano(), 10), url, existed, err } func (c *client) upload(ctx context.Context, user, master, relativePath, hash string, fileMeta *FileMetaInput, headerMetadata *headerData) (torrent *storage.Torrent, bootstrap []*Bootstrap, err error) {