feat(NIP-96-4e73e94,nip-98-ae0fd96): storage management endpoints

.github/workflows/CICD.yaml

@@ -158,6 +158,8 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: ${{ steps.findLatestGoVersion.outputs.latestGoVersion }}
+      - name: Install libvips and ffmpeg for meta extraction
+        run: sudo apt-get install -y libvips-dev ffmpeg
       - name: Build all
         run: make build-all@ci/cd
       - name: Slack Notification For Failure/Cancellation
@@ -193,6 +195,8 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: ${{ steps.findLatestGoVersion.outputs.latestGoVersion }}
+      - name: Install libvips and ffmpeg for meta extraction
+        run: sudo apt-get install -y libvips-dev ffmpeg
       - name: Test ${{ matrix.package }}
 #TODO enable coverage
         run: |
@@ -233,6 +237,8 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: ${{ steps.findLatestGoVersion.outputs.latestGoVersion }}
+      - name: Install libvips and ffmpeg for meta extraction
+        run: sudo apt-get install -y libvips-dev ffmpeg
       - name: Benchmark ${{ matrix.package }}
         run: |
           cd ${{ matrix.package }}

README.md

@@ -1,5 +1,23 @@
 # subzero
+## Starting
+```bash
+subzero --port=9998 --cert=./cmd/subzero/.testdata/localhost.crt --key=./cmd/subzero/.testdata/localhost.key --adnl-external-ip=127.0.0.1 --adnl-port=11512 --storage-root=./../.uploads --adnl-node-key=<hex> [--global-config-url=file://path/to/global.json]
+```
+Parameters:
+* port - port to start https/ws server for user iteraction
+* cert - tls cert for https / ws server
+* key - tls key for https / ws server
+* adnl-external-ip - node's external address (needed to serve storage uploads), other nodes connect to <adnl-external-ip>:<adnl-port>
+* adnl-port - port to start adnl / storage server
+* storage-root - root storage to store files `<storage-root>/<user's key>/files_here.ext`
+* adnl-node-key - adnl key for the node in hex form (length: 64 bytes, 128 in hex), i.e `6cc91d96a67bcae7a7a4df91f9c04469f652cf007b33460c60c0649f1777df5703bec10efbd4520126e53d0d70552f873ba843d54352d59fa28989bdf3925a7d` = random if not specified
+```go
+_, key ,_ := ed25519.GenerateKey(nil)
+fmt.Println(hex.EncodeToString(key))
+```
+* global-config-url - url (supports file:// schema) of global config (to fetch initial DHT nodes for storage), by default = mainnet url
 
+## NIPs
 NIPs | latest commit hash implemented | comments
 --- | --- | --- 
 [01](https://github.com/nostr-protocol/nips/blob/master/01.md) | [9971db3](https://github.com/nostr-protocol/nips/commit/9971db355164815c986251f8f89d1c7c70ec9e53)
@@ -28,4 +46,5 @@ NIPs | latest commit hash implemented | comments
 [90](https://github.com/nostr-protocol/nips/blob/master/90.md) | |
 [92](https://github.com/nostr-protocol/nips/blob/master/92.md) | |
 [94](https://github.com/nostr-protocol/nips/blob/master/94.md) | |
-[96](https://github.com/nostr-protocol/nips/blob/master/96.md) | |
+[96](https://github.com/nostr-protocol/nips/blob/master/96.md) | [4e73e94d417f16fa3451e58ef921cb3b512c6f8e](https://github.com/ice-blockchain/subzero/commit/130bac5adedf6563fe8d8e869f7e46b4cfb414e0)|
+[98](https://github.com/nostr-protocol/nips/blob/master/98.md) | [ae0fd96907d0767f07fb54ca1de9f197c600cb27](https://github.com/ice-blockchain/subzero/commit/130bac5adedf6563fe8d8e869f7e46b4cfb414e0)|

cmd/subzero/.testdata/regenerate.sh

@@ -10,7 +10,8 @@ go run filippo.io/mkcert -ecdsa -install
 cp $(go run filippo.io/mkcert -CAROOT)/rootCA.pem ca.pem
 
 # Generate a new certificate for localhost
-go run filippo.io/mkcert -ecdsa -days 13 -cert-file "$CRT" -key-file "$KEY" localhost $(hostname) 127.0.0.1 ::1
+NETWORK_IP=$(ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1' | tail -1)
+go run filippo.io/mkcert -ecdsa -days 13 -cert-file "$CRT" -key-file "$KEY" localhost $(hostname) 127.0.0.1 ::1 $NETWORK_IP
 
 # Compute the sha256 fingerprint of the certificate for WebTransport
 rm fingerprint.base64 || true

cmd/subzero/subzero.go

@@ -4,7 +4,9 @@ package main
 
 import (
 	"context"
+	"crypto/ed25519"
 	"log"
+	"net"
 
 	"github.com/cockroachdb/errors"
 	"github.com/spf13/cobra"
@@ -14,6 +16,7 @@ import (
 	"github.com/ice-blockchain/subzero/model"
 	"github.com/ice-blockchain/subzero/server"
 	wsserver "github.com/ice-blockchain/subzero/server/ws"
+	"github.com/ice-blockchain/subzero/storage"
 )
 
 var (
@@ -22,6 +25,12 @@ var (
 	cert               string
 	key                string
 	databasePath       string
+	externalIP         string
+	adnlPort           uint16
+	storageRootDir     string
+	globalConfigUrl    string
+	adnlNodeKey        []byte
+	debug              bool
 	subzero            = &cobra.Command{
 		Use:   "subzero",
 		Short: "subzero",
@@ -35,7 +44,7 @@ var (
 				log.Print("using database at ", databasePath)
 			}
 			query.MustInit(databasePath)
-
+			storage.MustInit(ctx, adnlNodeKey, globalConfigUrl, storageRootDir, net.ParseIP(externalIP), int(adnlPort), debug)
 			server.ListenAndServe(ctx, cancel, &server.Config{
 				CertPath:                cert,
 				KeyPath:                 key,
@@ -50,6 +59,18 @@ var (
 		subzero.Flags().StringVar(&key, "key", "", "path to tls certificate for the http/ws server (TLS)")
 		subzero.Flags().Uint16Var(&port, "port", 0, "port to communicate with clients (http/websocket)")
 		subzero.Flags().IntVar(&minLeadingZeroBits, "minLeadingZeroBits", 0, "min leading zero bits according NIP-13")
+		subzero.Flags().StringVar(&externalIP, "adnl-external-ip", "", "external ip for storage service")
+		subzero.Flags().Uint16Var(&adnlPort, "adnl-port", 0, "port to open adnl-gateway for storage service")
+		subzero.Flags().StringVar(&storageRootDir, "storage-root", "./.uploads", "root storage directory")
+		subzero.Flags().StringVar(&globalConfigUrl, "global-config-url", storage.DefaultConfigUrl, "global config for ION storage")
+		subzero.Flags().BytesHexVar(&adnlNodeKey, "adnl-node-key", func() []byte {
+			_, nodeKey, err := ed25519.GenerateKey(nil)
+			if err != nil {
+				log.Panic(errors.Wrapf(err, "failed to generate node key"))
+			}
+			return nodeKey
+		}(), "adnl node key in hex")
+		subzero.Flags().BoolVar(&debug, "debug", false, "enable debugging info")
 		if err := subzero.MarkFlagRequired("cert"); err != nil {
 			log.Print(err)
 		}
@@ -59,6 +80,15 @@ var (
 		if err := subzero.MarkFlagRequired("port"); err != nil {
 			log.Print(err)
 		}
+		if err := subzero.MarkFlagRequired("adnl-external-ip"); err != nil {
+			log.Print(err)
+		}
+		if err := subzero.MarkFlagRequired("adnl-port"); err != nil {
+			log.Print(err)
+		}
+		if err := subzero.MarkFlagRequired("storage-root"); err != nil {
+			log.Print(err)
+		}
 	}
 )
 
@@ -71,7 +101,9 @@ func init() {
 		if err := query.AcceptEvent(ctx, event); err != nil {
 			return errors.Wrapf(err, "failed to query.AcceptEvent(%#v)", event)
 		}
-
+		if sErr := storage.AcceptEvent(ctx, event); sErr != nil {
+			return errors.Wrapf(sErr, "failed to process NIP-94 event")
+		}
 		return nil
 	})
 	wsserver.RegisterWSSubscriptionListener(query.GetStoredEvents)

database/query/query.go

@@ -25,7 +25,6 @@ const (
 var (
 	ErrUnexpectedRowsAffected      = errors.New("unexpected rows affected")
 	ErrTargetReactionEventNotFound = errors.New("target reaction event not found")
-	ErrOnBehalfAccessDenied        = errors.New("on-behalf access denied")
 	ErrAttestationUpdateRejected   = errors.New("attestation update rejected")
 	errEventIteratorInterrupted    = errors.New("interrupted")
 )
@@ -219,7 +218,7 @@ func (db *dbClient) handleError(err error) error {
 	if errors.As(err, &sqlError) && sqlError.Code == sqlite3.ErrConstraint {
 		switch sqlError.Error() {
 		case "onbehalf permission denied":
-			err = ErrOnBehalfAccessDenied
+			err = model.ErrOnBehalfAccessDenied
 		case "attestation list update must be linear":
 			err = ErrAttestationUpdateRejected
 		}

database/query/query_test.go

@@ -975,7 +975,7 @@ func TestQueryEventAttestation(t *testing.T) {
 			ev.Tags = model.Tags{{model.CustomIONTagOnBehalfOf, nostr.GeneratePrivateKey()}}
 			require.NoError(t, ev.Sign(active))
 			t.Logf("event %+v", ev)
-			require.ErrorIs(t, db.AcceptEvent(context.TODO(), &ev), ErrOnBehalfAccessDenied)
+			require.ErrorIs(t, db.AcceptEvent(context.TODO(), &ev), model.ErrOnBehalfAccessDenied)
 		})
 		t.Run("Count", func(t *testing.T) {
 			count, err := db.CountEvents(context.TODO(), helperNewFilterSubscription(func(apply *model.Filter) {
@@ -1090,7 +1090,7 @@ func TestEventDeleteWithAttestation(t *testing.T) {
 			ev.Tags = append(ev.Tags, model.Tag{model.TagAttestationName, hackerPublic, "", model.CustomIONAttestationKindActive + ":" + strconv.Itoa(int(now-1))})
 			ev.Tags = append(ev.Tags, model.Tag{model.CustomIONTagOnBehalfOf, masterPublic})
 			require.NoError(t, ev.Sign(user2Private))
-			require.ErrorIs(t, db.AcceptEvent(context.TODO(), &ev), ErrOnBehalfAccessDenied)
+			require.ErrorIs(t, db.AcceptEvent(context.TODO(), &ev), model.ErrOnBehalfAccessDenied)
 		})
 	})
 	t.Run("DeleteEvents", func(t *testing.T) {

go.mod

@@ -2,12 +2,17 @@ module github.com/ice-blockchain/subzero
 
 go 1.23.2
 
-replace filippo.io/mkcert => github.com/kixelated/mkcert v1.4.4-days
-
-replace github.com/quic-go/quic-go v0.48.0 => github.com/quic-go/quic-go v0.47.0
+replace (
+	filippo.io/mkcert => github.com/kixelated/mkcert v1.4.4-days
+	github.com/quic-go/quic-go v0.48.0 => github.com/quic-go/quic-go v0.47.0
+	github.com/xssnick/tonutils-storage => github.com/ice-cronus/tonutils-storage v0.0.0-20241007090031-eb90cffa7912
+)
 
 require (
 	github.com/cockroachdb/errors v1.11.3
+	github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8
+	github.com/davidbyttow/govips/v2 v2.15.0
+	github.com/fsnotify/fsnotify v1.7.0
 	github.com/gin-gonic/gin v1.10.0
 	github.com/gobwas/httphead v0.1.0
 	github.com/gobwas/ws v1.4.0
@@ -21,22 +26,33 @@ require (
 	github.com/nbd-wtf/go-nostr v0.39.1
 	github.com/quic-go/quic-go v0.48.0
 	github.com/quic-go/webtransport-go v0.8.0
+	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
 	github.com/schollz/progressbar/v3 v3.16.1
 	github.com/spf13/cobra v1.8.1
 	github.com/stretchr/testify v1.9.0
+	github.com/syndtr/goleveldb v1.0.0
+	github.com/u2takey/ffmpeg-go v0.5.0
+	github.com/xssnick/tonutils-go v1.10.2
+	github.com/xssnick/tonutils-storage v0.6.5
 	go.uber.org/goleak v1.3.0
+	golang.org/x/net v0.30.0
 	pgregory.net/rand v1.0.2
 )
 
 require (
+	atomicgo.dev/cursor v0.2.0 // indirect
+	atomicgo.dev/keyboard v0.2.9 // indirect
+	atomicgo.dev/schedule v0.1.0 // indirect
+	github.com/aws/aws-sdk-go v1.55.5 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect
 	github.com/bytedance/sonic v1.12.3 // indirect
-	github.com/bytedance/sonic/loader v0.2.0 // indirect
+	github.com/bytedance/sonic/loader v0.2.1 // indirect
 	github.com/cloudwego/base64x v0.1.4 // indirect
 	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
 	github.com/cockroachdb/redact v1.1.5 // indirect
+	github.com/containerd/console v1.0.4 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
@@ -50,40 +66,51 @@ require (
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/pprof v0.0.0-20241017200806-017d972448fc // indirect
+	github.com/gookit/color v1.5.4 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kevinms/leakybucket-go v0.0.0-20200115003610-082473db97ca // indirect
 	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
+	github.com/lithammer/fuzzysearch v1.1.8 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a // indirect
 	github.com/onsi/ginkgo/v2 v2.20.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/pterm/pterm v0.12.79 // indirect
 	github.com/puzpuzpuz/xsync/v3 v3.4.0 // indirect
 	github.com/quic-go/qpack v0.5.1 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/rogpeppe/go-internal v1.13.1 // indirect
+	github.com/sigurn/crc16 v0.0.0-20240131213347-83fcde1e29d1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tidwall/gjson v1.18.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/u2takey/go-utils v0.3.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
+	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	go.uber.org/mock v0.5.0 // indirect
 	golang.org/x/arch v0.11.0 // indirect
 	golang.org/x/crypto v0.28.0 // indirect
 	golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect
+	golang.org/x/image v0.21.0 // indirect
 	golang.org/x/mod v0.21.0 // indirect
-	golang.org/x/net v0.30.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
 	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/term v0.25.0 // indirect