diff --git a/.github/workflows/CICD.yaml b/.github/workflows/CICD.yaml index 985d852..f321389 100644 --- a/.github/workflows/CICD.yaml +++ b/.github/workflows/CICD.yaml @@ -158,6 +158,8 @@ jobs: uses: actions/setup-go@v5 with: go-version: ${{ steps.findLatestGoVersion.outputs.latestGoVersion }} + - name: Install libvips and ffmpeg for meta extraction + run: sudo apt-get install -y libvips-dev ffmpeg - name: Build all run: make build-all@ci/cd - name: Slack Notification For Failure/Cancellation @@ -193,6 +195,8 @@ jobs: uses: actions/setup-go@v5 with: go-version: ${{ steps.findLatestGoVersion.outputs.latestGoVersion }} + - name: Install libvips and ffmpeg for meta extraction + run: sudo apt-get install -y libvips-dev ffmpeg - name: Test ${{ matrix.package }} #TODO enable coverage run: | @@ -233,6 +237,8 @@ jobs: uses: actions/setup-go@v5 with: go-version: ${{ steps.findLatestGoVersion.outputs.latestGoVersion }} + - name: Install libvips and ffmpeg for meta extraction + run: sudo apt-get install -y libvips-dev ffmpeg - name: Benchmark ${{ matrix.package }} run: | cd ${{ matrix.package }} diff --git a/README.md b/README.md index 687793f..6751d9c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,23 @@ # subzero +## Starting +```bash +subzero --port=9998 --cert=./cmd/subzero/.testdata/localhost.crt --key=./cmd/subzero/.testdata/localhost.key --adnl-external-ip=127.0.0.1 --adnl-port=11512 --storage-root=./../.uploads --adnl-node-key= [--global-config-url=file://path/to/global.json] +``` +Parameters: +* port - port to start https/ws server for user iteraction +* cert - tls cert for https / ws server +* key - tls key for https / ws server +* adnl-external-ip - node's external address (needed to serve storage uploads), other nodes connect to : +* adnl-port - port to start adnl / storage server +* storage-root - root storage to store files `//files_here.ext` +* adnl-node-key - adnl key for the node in hex form (length: 64 bytes, 128 in hex), i.e `6cc91d96a67bcae7a7a4df91f9c04469f652cf007b33460c60c0649f1777df5703bec10efbd4520126e53d0d70552f873ba843d54352d59fa28989bdf3925a7d` = random if not specified +```go +_, key ,_ := ed25519.GenerateKey(nil) +fmt.Println(hex.EncodeToString(key)) +``` +* global-config-url - url (supports file:// schema) of global config (to fetch initial DHT nodes for storage), by default = mainnet url +## NIPs NIPs | latest commit hash implemented | comments --- | --- | --- [01](https://github.com/nostr-protocol/nips/blob/master/01.md) | [9971db3](https://github.com/nostr-protocol/nips/commit/9971db355164815c986251f8f89d1c7c70ec9e53) @@ -28,4 +46,5 @@ NIPs | latest commit hash implemented | comments [90](https://github.com/nostr-protocol/nips/blob/master/90.md) | | [92](https://github.com/nostr-protocol/nips/blob/master/92.md) | | [94](https://github.com/nostr-protocol/nips/blob/master/94.md) | | -[96](https://github.com/nostr-protocol/nips/blob/master/96.md) | | \ No newline at end of file +[96](https://github.com/nostr-protocol/nips/blob/master/96.md) | [4e73e94d417f16fa3451e58ef921cb3b512c6f8e](https://github.com/ice-blockchain/subzero/commit/130bac5adedf6563fe8d8e869f7e46b4cfb414e0)| +[98](https://github.com/nostr-protocol/nips/blob/master/98.md) | [ae0fd96907d0767f07fb54ca1de9f197c600cb27](https://github.com/ice-blockchain/subzero/commit/130bac5adedf6563fe8d8e869f7e46b4cfb414e0)| \ No newline at end of file diff --git a/cmd/subzero/.testdata/regenerate.sh b/cmd/subzero/.testdata/regenerate.sh index 0a3b478..c2ede30 100755 --- a/cmd/subzero/.testdata/regenerate.sh +++ b/cmd/subzero/.testdata/regenerate.sh @@ -10,7 +10,8 @@ go run filippo.io/mkcert -ecdsa -install cp $(go run filippo.io/mkcert -CAROOT)/rootCA.pem ca.pem # Generate a new certificate for localhost -go run filippo.io/mkcert -ecdsa -days 13 -cert-file "$CRT" -key-file "$KEY" localhost $(hostname) 127.0.0.1 ::1 +NETWORK_IP=$(ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1' | tail -1) +go run filippo.io/mkcert -ecdsa -days 13 -cert-file "$CRT" -key-file "$KEY" localhost $(hostname) 127.0.0.1 ::1 $NETWORK_IP # Compute the sha256 fingerprint of the certificate for WebTransport rm fingerprint.base64 || true diff --git a/cmd/subzero/subzero.go b/cmd/subzero/subzero.go index 0602aaf..f64a0e9 100644 --- a/cmd/subzero/subzero.go +++ b/cmd/subzero/subzero.go @@ -4,7 +4,9 @@ package main import ( "context" + "crypto/ed25519" "log" + "net" "github.com/cockroachdb/errors" "github.com/spf13/cobra" @@ -14,6 +16,7 @@ import ( "github.com/ice-blockchain/subzero/model" "github.com/ice-blockchain/subzero/server" wsserver "github.com/ice-blockchain/subzero/server/ws" + "github.com/ice-blockchain/subzero/storage" ) var ( @@ -22,6 +25,12 @@ var ( cert string key string databasePath string + externalIP string + adnlPort uint16 + storageRootDir string + globalConfigUrl string + adnlNodeKey []byte + debug bool subzero = &cobra.Command{ Use: "subzero", Short: "subzero", @@ -35,7 +44,7 @@ var ( log.Print("using database at ", databasePath) } query.MustInit(databasePath) - + storage.MustInit(ctx, adnlNodeKey, globalConfigUrl, storageRootDir, net.ParseIP(externalIP), int(adnlPort), debug) server.ListenAndServe(ctx, cancel, &server.Config{ CertPath: cert, KeyPath: key, @@ -50,6 +59,18 @@ var ( subzero.Flags().StringVar(&key, "key", "", "path to tls certificate for the http/ws server (TLS)") subzero.Flags().Uint16Var(&port, "port", 0, "port to communicate with clients (http/websocket)") subzero.Flags().IntVar(&minLeadingZeroBits, "minLeadingZeroBits", 0, "min leading zero bits according NIP-13") + subzero.Flags().StringVar(&externalIP, "adnl-external-ip", "", "external ip for storage service") + subzero.Flags().Uint16Var(&adnlPort, "adnl-port", 0, "port to open adnl-gateway for storage service") + subzero.Flags().StringVar(&storageRootDir, "storage-root", "./.uploads", "root storage directory") + subzero.Flags().StringVar(&globalConfigUrl, "global-config-url", storage.DefaultConfigUrl, "global config for ION storage") + subzero.Flags().BytesHexVar(&adnlNodeKey, "adnl-node-key", func() []byte { + _, nodeKey, err := ed25519.GenerateKey(nil) + if err != nil { + log.Panic(errors.Wrapf(err, "failed to generate node key")) + } + return nodeKey + }(), "adnl node key in hex") + subzero.Flags().BoolVar(&debug, "debug", false, "enable debugging info") if err := subzero.MarkFlagRequired("cert"); err != nil { log.Print(err) } @@ -59,6 +80,15 @@ var ( if err := subzero.MarkFlagRequired("port"); err != nil { log.Print(err) } + if err := subzero.MarkFlagRequired("adnl-external-ip"); err != nil { + log.Print(err) + } + if err := subzero.MarkFlagRequired("adnl-port"); err != nil { + log.Print(err) + } + if err := subzero.MarkFlagRequired("storage-root"); err != nil { + log.Print(err) + } } ) @@ -71,7 +101,9 @@ func init() { if err := query.AcceptEvent(ctx, event); err != nil { return errors.Wrapf(err, "failed to query.AcceptEvent(%#v)", event) } - + if sErr := storage.AcceptEvent(ctx, event); sErr != nil { + return errors.Wrapf(sErr, "failed to process NIP-94 event") + } return nil }) wsserver.RegisterWSSubscriptionListener(query.GetStoredEvents) diff --git a/database/query/query.go b/database/query/query.go index 906a07d..34c4706 100644 --- a/database/query/query.go +++ b/database/query/query.go @@ -25,7 +25,6 @@ const ( var ( ErrUnexpectedRowsAffected = errors.New("unexpected rows affected") ErrTargetReactionEventNotFound = errors.New("target reaction event not found") - ErrOnBehalfAccessDenied = errors.New("on-behalf access denied") ErrAttestationUpdateRejected = errors.New("attestation update rejected") errEventIteratorInterrupted = errors.New("interrupted") ) @@ -219,7 +218,7 @@ func (db *dbClient) handleError(err error) error { if errors.As(err, &sqlError) && sqlError.Code == sqlite3.ErrConstraint { switch sqlError.Error() { case "onbehalf permission denied": - err = ErrOnBehalfAccessDenied + err = model.ErrOnBehalfAccessDenied case "attestation list update must be linear": err = ErrAttestationUpdateRejected } diff --git a/database/query/query_test.go b/database/query/query_test.go index ca42f46..e0caa7e 100644 --- a/database/query/query_test.go +++ b/database/query/query_test.go @@ -975,7 +975,7 @@ func TestQueryEventAttestation(t *testing.T) { ev.Tags = model.Tags{{model.CustomIONTagOnBehalfOf, nostr.GeneratePrivateKey()}} require.NoError(t, ev.Sign(active)) t.Logf("event %+v", ev) - require.ErrorIs(t, db.AcceptEvent(context.TODO(), &ev), ErrOnBehalfAccessDenied) + require.ErrorIs(t, db.AcceptEvent(context.TODO(), &ev), model.ErrOnBehalfAccessDenied) }) t.Run("Count", func(t *testing.T) { count, err := db.CountEvents(context.TODO(), helperNewFilterSubscription(func(apply *model.Filter) { @@ -1090,7 +1090,7 @@ func TestEventDeleteWithAttestation(t *testing.T) { ev.Tags = append(ev.Tags, model.Tag{model.TagAttestationName, hackerPublic, "", model.CustomIONAttestationKindActive + ":" + strconv.Itoa(int(now-1))}) ev.Tags = append(ev.Tags, model.Tag{model.CustomIONTagOnBehalfOf, masterPublic}) require.NoError(t, ev.Sign(user2Private)) - require.ErrorIs(t, db.AcceptEvent(context.TODO(), &ev), ErrOnBehalfAccessDenied) + require.ErrorIs(t, db.AcceptEvent(context.TODO(), &ev), model.ErrOnBehalfAccessDenied) }) }) t.Run("DeleteEvents", func(t *testing.T) { diff --git a/go.mod b/go.mod index 4fff9cf..90ef315 100644 --- a/go.mod +++ b/go.mod @@ -2,12 +2,17 @@ module github.com/ice-blockchain/subzero go 1.23.2 -replace filippo.io/mkcert => github.com/kixelated/mkcert v1.4.4-days - -replace github.com/quic-go/quic-go v0.48.0 => github.com/quic-go/quic-go v0.47.0 +replace ( + filippo.io/mkcert => github.com/kixelated/mkcert v1.4.4-days + github.com/quic-go/quic-go v0.48.0 => github.com/quic-go/quic-go v0.47.0 + github.com/xssnick/tonutils-storage => github.com/ice-cronus/tonutils-storage v0.0.0-20241007090031-eb90cffa7912 +) require ( github.com/cockroachdb/errors v1.11.3 + github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 + github.com/davidbyttow/govips/v2 v2.15.0 + github.com/fsnotify/fsnotify v1.7.0 github.com/gin-gonic/gin v1.10.0 github.com/gobwas/httphead v0.1.0 github.com/gobwas/ws v1.4.0 @@ -21,22 +26,33 @@ require ( github.com/nbd-wtf/go-nostr v0.39.1 github.com/quic-go/quic-go v0.48.0 github.com/quic-go/webtransport-go v0.8.0 + github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 github.com/schollz/progressbar/v3 v3.16.1 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 + github.com/syndtr/goleveldb v1.0.0 + github.com/u2takey/ffmpeg-go v0.5.0 + github.com/xssnick/tonutils-go v1.10.2 + github.com/xssnick/tonutils-storage v0.6.5 go.uber.org/goleak v1.3.0 + golang.org/x/net v0.30.0 pgregory.net/rand v1.0.2 ) require ( + atomicgo.dev/cursor v0.2.0 // indirect + atomicgo.dev/keyboard v0.2.9 // indirect + atomicgo.dev/schedule v0.1.0 // indirect + github.com/aws/aws-sdk-go v1.55.5 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect github.com/bytedance/sonic v1.12.3 // indirect - github.com/bytedance/sonic/loader v0.2.0 // indirect + github.com/bytedance/sonic/loader v0.2.1 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect + github.com/containerd/console v1.0.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect @@ -50,40 +66,51 @@ require ( github.com/gobwas/pool v0.2.1 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/pprof v0.0.0-20241017200806-017d972448fc // indirect + github.com/gookit/color v1.5.4 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kevinms/leakybucket-go v0.0.0-20200115003610-082473db97ca // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect + github.com/lithammer/fuzzysearch v1.1.8 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a // indirect github.com/onsi/ginkgo/v2 v2.20.2 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/pterm/pterm v0.12.79 // indirect github.com/puzpuzpuz/xsync/v3 v3.4.0 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.13.1 // indirect + github.com/sigurn/crc16 v0.0.0-20240131213347-83fcde1e29d1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/u2takey/go-utils v0.3.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.uber.org/mock v0.5.0 // indirect golang.org/x/arch v0.11.0 // indirect golang.org/x/crypto v0.28.0 // indirect golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect + golang.org/x/image v0.21.0 // indirect golang.org/x/mod v0.21.0 // indirect - golang.org/x/net v0.30.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/term v0.25.0 // indirect diff --git a/go.sum b/go.sum index 26e7d7a..b4df5d4 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,26 @@ +atomicgo.dev/assert v0.0.2 h1:FiKeMiZSgRrZsPo9qn/7vmr7mCsh5SZyXY4YGYiYwrg= +atomicgo.dev/assert v0.0.2/go.mod h1:ut4NcI3QDdJtlmAxQULOmA13Gz6e2DWbSAS8RUOmNYQ= +atomicgo.dev/cursor v0.2.0 h1:H6XN5alUJ52FZZUkI7AlJbUc1aW38GWZalpYRPpoPOw= +atomicgo.dev/cursor v0.2.0/go.mod h1:Lr4ZJB3U7DfPPOkbH7/6TOtJ4vFGHlgj1nc+n900IpU= +atomicgo.dev/keyboard v0.2.9 h1:tOsIid3nlPLZ3lwgG8KZMp/SFmr7P0ssEN5JUsm78K8= +atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtExQ= +atomicgo.dev/schedule v0.1.0 h1:nTthAbhZS5YZmgYbb2+DH8uQIZcTlIrd4eYr3UQxEjs= +atomicgo.dev/schedule v0.1.0/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs= +github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8= +github.com/MarvinJWendt/testza v0.2.8/go.mod h1:nwIcjmr0Zz+Rcwfh3/4UhBp7ePKVhuBExvZqnKYWlII= +github.com/MarvinJWendt/testza v0.2.10/go.mod h1:pd+VWsoGUiFtq+hRKSU1Bktnn+DMCSrDrXDpX2bG66k= +github.com/MarvinJWendt/testza v0.2.12/go.mod h1:JOIegYyV7rX+7VZ9r77L/eH6CfJHHzXjB69adAhzZkI= +github.com/MarvinJWendt/testza v0.3.0/go.mod h1:eFcL4I0idjtIx8P9C6KkAuLgATNKpX4/2oUqKc6bF2c= +github.com/MarvinJWendt/testza v0.4.2/go.mod h1:mSdhXiKH8sg/gQehJ63bINcCKp7RtYewEjXsvsVUPbE= +github.com/MarvinJWendt/testza v0.5.2 h1:53KDo64C1z/h/d/stCYCPY69bt/OSwjq5KpFNwi+zB4= +github.com/MarvinJWendt/testza v0.5.2/go.mod h1:xu53QFE5sCdjtMCKk8YMQ2MnymimEctc4n3EjyIYvEY= +github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= +github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= +github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/btcsuite/btcd/btcec/v2 v2.3.4 h1:3EJjcN70HCu/mwqlUsGK8GcNVyLVxFDlWurTXGPFfiQ= github.com/btcsuite/btcd/btcec/v2 v2.3.4/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 h1:59Kx4K6lzOW5w6nFlA0v5+lk/6sjybR934QNHSJZPTQ= @@ -7,8 +28,8 @@ github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0/go.mod h1:7SFka0XMvUgj3hfZtyd github.com/bytedance/sonic v1.12.3 h1:W2MGa7RCU1QTeYRTPE3+88mVC0yXmsRQRChiyVocVjU= github.com/bytedance/sonic v1.12.3/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= -github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.1 h1:1GgorWTqf12TA8mma4DDSbaQigE2wOgQo7iCjjJv3+E= +github.com/bytedance/sonic/loader v0.2.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= @@ -21,18 +42,29 @@ github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZe github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= +github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro= +github.com/containerd/console v1.0.4/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= +github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davidbyttow/govips/v2 v2.15.0 h1:h3lF+rQElBzGXbQSSPqmE3XGySPhcQo2x3t5l/dZ+pU= +github.com/davidbyttow/govips/v2 v2.15.0/go.mod h1:3OQCHj0nf5Mnrplh5VlNvmx3IhJXyxbAoTJZPflUjmM= github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc= github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc= github.com/getsentry/sentry-go v0.29.1 h1:DyZuChN8Hz3ARxGVV8ePaNXh1dQ7d76AiB117xcREwA= @@ -43,6 +75,7 @@ github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -65,50 +98,81 @@ github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20241017200806-017d972448fc h1:NGyrhhFhwvRAZg02jnYVg3GBQy0qGBKmFQJwaPmpmxs= github.com/google/pprof v0.0.0-20241017200806-017d972448fc/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ= +github.com/gookit/color v1.5.0/go.mod h1:43aQb+Zerm/BWh2GnrgOQm7ffz7tvQXEKV6BFMl7wAo= +github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= +github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ice-blockchain/go/src v0.0.0-20240529122316-8d9458949bdd h1:SY8V7ujSDgEYEdtMk26DEaiq8qvCTEZ7XlojacrWZj4= github.com/ice-blockchain/go/src v0.0.0-20240529122316-8d9458949bdd/go.mod h1:pVzkmaIwkLaJ5cKPHPtuKyxaXiTsa/V7kbbgLmsr4Hw= +github.com/ice-cronus/tonutils-storage v0.0.0-20241007090031-eb90cffa7912 h1:nhY1xWbrbwbAfjyLd8QFgeatVzBEXp4S9KnRb0xgkX0= +github.com/ice-cronus/tonutils-storage v0.0.0-20241007090031-eb90cffa7912/go.mod h1:kDD8rSR3Fcp7tTNZ1C/eHxq/JSYpXBJdHU3amOZoTcg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jamiealquiza/tachymeter v2.0.0+incompatible h1:mGiF1DGo8l6vnGT8FXNNcIXht/YmjzfraiUprXYwJ6g= github.com/jamiealquiza/tachymeter v2.0.0+incompatible/go.mod h1:Ayf6zPZKEnLsc3winWEXJRkTBhdHo58HODAu1oFJkYU= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kevinms/leakybucket-go v0.0.0-20200115003610-082473db97ca h1:qNtd6alRqd3qOdPrKXMZImV192ngQ0WSh1briEO33Tk= +github.com/kevinms/leakybucket-go v0.0.0-20200115003610-082473db97ca/go.mod h1:ph+C5vpnCcQvKBwJwKLTK3JLNGnBXYlG7m7JjoC/zYA= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= +github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4= +github.com/lithammer/fuzzysearch v1.1.8/go.mod h1:IdqeyBClc3FFqSzYq/MXESsS4S0FsZ5ajtkr5xPLts4= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= @@ -119,26 +183,46 @@ github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mxschmitt/golang-combinations v1.2.0 h1:V5E7MncIK8Yr1SL/SpdqMuSquFsfoIs5auI7Y3n8z14= github.com/mxschmitt/golang-combinations v1.2.0/go.mod h1:RCm5eR03B+JrBOMRDLsKZWShluXdrHu+qwhPEJ0miBM= github.com/nbd-wtf/go-nostr v0.39.1 h1:sz9GxRzMxofil9w5l0eyNs+3nsyQlHU3BZf0Cc79vpA= github.com/nbd-wtf/go-nostr v0.39.1/go.mod h1:UTFitPKazxHbGlFMh0QvB8MpOyuTViBU4Zf05Z1aUdk= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a h1:dlRvE5fWabOchtH7znfiFCcOvmIYgOeAS5ifBXBlh9Q= +github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a/go.mod h1:hVoHR2EVESiICEMbg137etN/Lx+lSrHPTD39Z/uE+2s= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4= github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= +github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pterm/pterm v0.12.27/go.mod h1:PhQ89w4i95rhgE+xedAoqous6K9X+r6aSOI2eFF7DZI= +github.com/pterm/pterm v0.12.29/go.mod h1:WI3qxgvoQFFGKGjGnJR849gU0TsEOvKn5Q8LlY1U7lg= +github.com/pterm/pterm v0.12.30/go.mod h1:MOqLIyMOgmTDz9yorcYbcw+HsgoZo3BQfg2wtl3HEFE= +github.com/pterm/pterm v0.12.31/go.mod h1:32ZAWZVXD7ZfG0s8qqHXePte42kdz8ECtRyEejaWgXU= +github.com/pterm/pterm v0.12.33/go.mod h1:x+h2uL+n7CP/rel9+bImHD5lF3nM9vJj80k9ybiiTTE= +github.com/pterm/pterm v0.12.36/go.mod h1:NjiL09hFhT/vWjQHSj1athJpx6H8cjpHXNAK5bUw8T8= +github.com/pterm/pterm v0.12.40/go.mod h1:ffwPLwlbXxP+rxT0GsgDTzS3y3rmpAO1NMjUkGTYf8s= +github.com/pterm/pterm v0.12.79 h1:lH3yrYMhdpeqX9y5Ep1u7DejyHy7NSQg9qrBjF9dFT4= +github.com/pterm/pterm v0.12.79/go.mod h1:1v/gzOF1N0FsjbgTHZ1wVycRkKiatFvJSJC4IGaQAAo= github.com/puzpuzpuz/xsync/v3 v3.4.0 h1:DuVBAdXuGFHv8adVXjWWZ63pJq+NRXOWVXlKDBZ+mJ4= github.com/puzpuzpuz/xsync/v3 v3.4.0/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= @@ -147,6 +231,9 @@ github.com/quic-go/quic-go v0.47.0 h1:yXs3v7r2bm1wmPTYNLKAAJTHMYkPEsfYJmTazXrCZ7 github.com/quic-go/quic-go v0.47.0/go.mod h1:3bCapYsJvXGZcipOHuu7plYtaV6tnF+z7wIFsU0WK9E= github.com/quic-go/webtransport-go v0.8.0 h1:HxSrwun11U+LlmwpgM1kEqIqH90IT4N8auv/cD7QFJg= github.com/quic-go/webtransport-go v0.8.0/go.mod h1:N99tjprW432Ut5ONql/aUhSLT0YVSlwHohQsuac9WaM= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= @@ -155,6 +242,11 @@ github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWN github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/schollz/progressbar/v3 v3.16.1 h1:RnF1neWZFzLCoGx8yp1yF7SDl4AzNDI5y4I0aUJRrZQ= github.com/schollz/progressbar/v3 v3.16.1/go.mod h1:I2ILR76gz5VXqYMIY/LdLecvMHDPVcQm3W/MSKi1TME= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sigurn/crc16 v0.0.0-20240131213347-83fcde1e29d1 h1:NVK+OqnavpyFmUiKfUMHrpvbCi2VFoWTrcpI7aDaJ2I= +github.com/sigurn/crc16 v0.0.0-20240131213347-83fcde1e29d1/go.mod h1:9/etS5gpQq9BJsJMWg1wpLbfuSnkm8dPF6FdW2JXVhA= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -163,12 +255,17 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -178,59 +275,122 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/u2takey/ffmpeg-go v0.5.0 h1:r7d86XuL7uLWJ5mzSeQ03uvjfIhiJYvsRAJFCW4uklU= +github.com/u2takey/ffmpeg-go v0.5.0/go.mod h1:ruZWkvC1FEiUNjmROowOAps3ZcWxEiOpFoHCvk97kGc= +github.com/u2takey/go-utils v0.3.1 h1:TaQTgmEZZeDHQFYfd+AdUT1cT4QJgJn/XVPELhHw4ys= +github.com/u2takey/go-utils v0.3.1/go.mod h1:6e+v5vEZ/6gu12w/DC2ixZdZtCrNokVxD0JUklcqdCs= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= +github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/xssnick/tonutils-go v1.10.2 h1:1wgnQPrzbOt+5PtuNrlMSUyh1/y0pvWRi0zeRNRLEbw= +github.com/xssnick/tonutils-go v1.10.2/go.mod h1:p1l1Bxdv9sz6x2jfbuGQUGJn6g5cqg7xsTp8rBHFoJY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4= golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.10.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0= +golang.org/x/image v0.21.0 h1:c5qV36ajHpdj4Qi0GnE0jUc/yuo33OLFaa0d+crTD5s= +golang.org/x/image v0.21.0/go.mod h1:vUbsLavqK/W303ZroQQVKQ+Af3Yl6Uz1Ppu5J/cLz78= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -240,9 +400,23 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= @@ -250,3 +424,4 @@ pgregory.net/rand v1.0.2 h1:ASEbkvwOmY/UPF2evJPBJ8XZg71xdKWYdByqKapI7Vw= pgregory.net/rand v1.0.2/go.mod h1:EyNx8APnDE3Svi8sWgUZ5lOiz60cNZUPPBTyzOUpPl4= pgregory.net/rapid v0.4.8 h1:d+5SGZWUbJPbl3ss6tmPFqnNeQR6VDOFly+eTjwPiEw= pgregory.net/rapid v0.4.8/go.mod h1:Z5PbWqjvWR1I3UGjvboUuan4fe4ZYEYNLNQLExzCoUs= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/model/model.go b/model/model.go index 9cc8b93..24ee645 100644 --- a/model/model.go +++ b/model/model.go @@ -33,8 +33,9 @@ type ( ) var ( - ErrDuplicate = errors.New("duplicate") - ErrUnsupportedAlg = errors.New("unsupported signature/key algorithm combination") + ErrDuplicate = errors.New("duplicate") + ErrUnsupportedAlg = errors.New("unsupported signature/key algorithm combination") + ErrOnBehalfAccessDenied = errors.New("on-behalf access denied") ) const ( diff --git a/server/http/.testdata/image.jpg b/server/http/.testdata/image.jpg new file mode 100644 index 0000000..75dff0b Binary files /dev/null and b/server/http/.testdata/image.jpg differ diff --git a/server/http/.testdata/image2.png b/server/http/.testdata/image2.png new file mode 100644 index 0000000..2e27484 Binary files /dev/null and b/server/http/.testdata/image2.png differ diff --git a/server/http/.testdata/text-master.txt b/server/http/.testdata/text-master.txt new file mode 100644 index 0000000..8b25206 --- /dev/null +++ b/server/http/.testdata/text-master.txt @@ -0,0 +1 @@ +master \ No newline at end of file diff --git a/server/http/.testdata/text.txt b/server/http/.testdata/text.txt new file mode 100644 index 0000000..f3a3485 --- /dev/null +++ b/server/http/.testdata/text.txt @@ -0,0 +1 @@ +text \ No newline at end of file diff --git a/server/http/auth_nip98.go b/server/http/auth_nip98.go new file mode 100644 index 0000000..5adcdce --- /dev/null +++ b/server/http/auth_nip98.go @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: ice License 1.0 + +package http + +import ( + "context" + "encoding/base64" + "net/url" + "strings" + "time" + + "github.com/cockroachdb/errors" + "github.com/gin-gonic/gin" + + "github.com/ice-blockchain/subzero/database/query" + "github.com/ice-blockchain/subzero/model" +) + +type ( + Token interface { + PubKey() string + MasterPubKey() string + ExpectedHash() string + ValidateAttestation(ctx context.Context, kind int, now time.Time) error + } + AuthClient interface { + VerifyToken(gCtx *gin.Context, token string, now time.Time) (Token, error) + } + + nostrToken struct { + ev model.Event + expectedHash string + } + authNostr struct { + } +) + +const ( + tokenExpirationWindow = 15 * time.Minute + nostrHttpAuthKind = 27235 +) + +var ( + ErrTokenExpired = errors.New("expired token") + ErrTokenInvalid = errors.New("invalid token") +) + +func NewAuth() AuthClient { + return &authNostr{} +} + +func getAuthHeader(gCtx *gin.Context) string { + return strings.TrimPrefix(gCtx.GetHeader("Authorization"), "Nostr ") +} + +func (a *authNostr) VerifyToken(gCtx *gin.Context, token string, now time.Time) (Token, error) { + bToken, err := base64.StdEncoding.DecodeString(token) + if err != nil { + return nil, errors.Wrap(err, "failed to unmarshal auth token: malformed base64") + } + var event model.Event + if err = event.UnmarshalJSON(bToken); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal auth token: malformed event json") + } + var ok bool + if ok, err = event.CheckSignature(); err != nil { + return nil, errors.Wrapf(err, "invalid token signature") + } else if !ok { + return nil, errors.Wrapf(ErrTokenInvalid, "invalid token signature") + } + if event.Kind != nostrHttpAuthKind { + return nil, errors.Wrapf(ErrTokenInvalid, "invalid token event kind %v", event.Kind) + } + if event.CreatedAt.Time().After(now) || (event.CreatedAt.Time().Before(now) && now.Sub(event.CreatedAt.Time()) > tokenExpirationWindow) { + return nil, ErrTokenExpired + } + if urlTag := event.Tags.GetFirst([]string{"u"}); urlTag != nil && len(*urlTag) > 1 { + var urlValue *url.URL + urlValue, err = url.Parse(urlTag.Value()) + if err != nil { + return nil, errors.Wrapf(ErrTokenInvalid, "failed to parse url tag with %v", urlTag.Value()) + } + fullReqUrl := (&url.URL{ + Scheme: "https", + Host: gCtx.Request.Host, + Path: gCtx.Request.URL.Path, + RawQuery: gCtx.Request.URL.RawQuery, + Fragment: gCtx.Request.URL.Fragment, + }) + if urlValue.String() != fullReqUrl.String() { + return nil, errors.Wrapf(ErrTokenInvalid, "url mismatch token>%v url>%v", urlValue, fullReqUrl) + } + } else { + return nil, errors.Wrapf(ErrTokenInvalid, "malformed u tag %v", urlTag) + } + if methodTag := event.Tags.GetFirst([]string{"method"}); methodTag != nil && len(*methodTag) > 1 { + method := methodTag.Value() + if method != gCtx.Request.Method { + return nil, errors.Wrapf(ErrTokenInvalid, "method mismatch token>%v url>%v", method, gCtx.Request.Method) + } + } else { + return nil, errors.Wrapf(ErrTokenInvalid, "malformed method tag %v", methodTag) + } + expectedHash := "" + if payloadTag := event.Tags.GetFirst([]string{"payload"}); payloadTag != nil && len(*payloadTag) > 1 { + expectedHash = payloadTag.Value() + } + return &nostrToken{ev: event, expectedHash: expectedHash}, nil +} +func (t *nostrToken) PubKey() string { + return t.ev.PubKey +} +func (t *nostrToken) MasterPubKey() string { + return t.ev.GetMasterPublicKey() +} +func (t *nostrToken) ExpectedHash() string { + return t.expectedHash +} + +func (t *nostrToken) ValidateAttestation(ctx context.Context, kind int, now time.Time) error { + if t.ev.PubKey == t.MasterPubKey() { + return nil + } + attestationEventIt := query.GetStoredEvents(ctx, &model.Subscription{model.Filters{model.Filter{ + Kinds: []int{model.CustomIONKindAttestation}, + Tags: model.TagMap{ + "p": []string{t.PubKey()}, + }, + }, + }}) + var allowed bool + for attestation, err := range attestationEventIt { + if err != nil { + return errors.Wrapf(err, "failed to get attestation event") + } + allowed, err = model.OnBehalfIsAccessAllowed(attestation.Tags, t.ev.PubKey, kind, now.Unix()) + if err != nil { + return errors.Wrapf(err, "failed to parse attestation event") + } + break + } + if !allowed { + return model.ErrOnBehalfAccessDenied + } + return nil +} diff --git a/server/http/nip11.go b/server/http/nip11.go index 0234418..1b41cb0 100644 --- a/server/http/nip11.go +++ b/server/http/nip11.go @@ -40,7 +40,7 @@ func (n *nip11handler) info() nip11.RelayInformationDocument { Description: "subzero", PubKey: "~", Contact: "~", - SupportedNIPs: []int{1, 2, 9, 10, 11, 13, 18, 23, 24, 25, 32, 40, 45, 50, 51, 56, 58, 65}, + SupportedNIPs: []int{1, 2, 9, 10, 11, 13, 18, 23, 24, 25, 32, 40, 45, 50, 51, 56, 58, 65, 96, 98}, Software: "subzero", } } diff --git a/server/http/nip11_test.go b/server/http/nip11_test.go index a1b10f2..193512a 100644 --- a/server/http/nip11_test.go +++ b/server/http/nip11_test.go @@ -10,10 +10,13 @@ import ( "testing" "time" + "github.com/gin-gonic/gin" "github.com/nbd-wtf/go-nostr/nip11" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/net/http2" + "github.com/ice-blockchain/subzero/database/query" wsserver "github.com/ice-blockchain/subzero/server/ws" "github.com/ice-blockchain/subzero/server/ws/fixture" ) @@ -22,6 +25,7 @@ const ( testDeadline = 30 * time.Second certPath = "%v/../ws/fixture/.testdata/localhost.crt" keyPath = "%v/../ws/fixture/.testdata/localhost.key" + storageRoot = "../../.test-uploads" ) var pubsubServer *fixture.MockService @@ -29,18 +33,31 @@ var pubsubServer *fixture.MockService func TestMain(m *testing.M) { serverCtx, serverCancel := context.WithTimeout(context.Background(), 10*time.Minute) defer serverCancel() + query.MustInit() + initServer(serverCtx, serverCancel, 9997, storageRoot) + http.DefaultClient.Transport = &http2.Transport{TLSClientConfig: fixture.LocalhostTLS()} + code := m.Run() + serverCancel() + os.Exit(code) +} + +func initServer(serverCtx context.Context, serverCancel context.CancelFunc, port uint16, storageRoot string) { wd, _ := os.Getwd() certFilePath := fmt.Sprintf(certPath, wd) keyFilePath := fmt.Sprintf(keyPath, wd) + initStorage(serverCtx, storageRoot) + uploader := NewUploadHandler(serverCtx) pubsubServer = fixture.NewTestServer(serverCtx, serverCancel, &wsserver.Config{ CertPath: certFilePath, KeyPath: keyFilePath, - Port: 9997, - }, nil, NewNIP11Handler()) - http.DefaultClient.Transport = &http.Transport{TLSClientConfig: fixture.LocalhostTLS()} + Port: port, + }, nil, NewNIP11Handler(), map[string]gin.HandlerFunc{ + "POST /files": uploader.Upload(), + "GET /files": uploader.ListFiles(), + "GET /files/:file": uploader.Download(), + "DELETE /files/:file": uploader.Delete(), + }) time.Sleep(100 * time.Millisecond) - m.Run() - serverCancel() } func TestNIP11(t *testing.T) { diff --git a/server/http/nip96.json b/server/http/nip96.json new file mode 100644 index 0000000..e0ce224 --- /dev/null +++ b/server/http/nip96.json @@ -0,0 +1,15 @@ +{ + "api_url": "/files", + "download_url": "/files", + "delegated_to_url": "", + "content_types": ["image/webp", "video/mpeg4", "application/x-brotli"], + "plans": { + "free": { + "name": "Free Tier", + "is_nip98_required": true, + "max_byte_size": 104857600, + "file_expiration": [0, 0], + "media_transformations": {} + } + } +} diff --git a/server/http/storage_nip96.go b/server/http/storage_nip96.go new file mode 100644 index 0000000..c5409a5 --- /dev/null +++ b/server/http/storage_nip96.go @@ -0,0 +1,332 @@ +// SPDX-License-Identifier: ice License 1.0 + +package http + +import ( + "context" + "crypto/sha256" + _ "embed" + "encoding/hex" + "fmt" + "io" + "log" + "mime/multipart" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/cockroachdb/errors" + gomime "github.com/cubewise-code/go-mime" + "github.com/gin-gonic/gin" + "github.com/gin-gonic/gin/binding" + "github.com/nbd-wtf/go-nostr" + + "github.com/ice-blockchain/subzero/storage" +) + +type ( + Uploader interface { + Upload() gin.HandlerFunc + NIP96Info() gin.HandlerFunc + Download() gin.HandlerFunc + Delete() gin.HandlerFunc + ListFiles() gin.HandlerFunc + } +) + +//go:embed nip96.json +var nip96Info string + +type storageHandler struct { + storageClient storage.StorageClient + auth AuthClient +} + +const mediaEndpointTimeout = 60 * time.Second +const maxUploadSize = 100 * 1024 * 1024 +const mediaTypeAvatar = "avatar" +const mediaTypeBanner = "banner" + +type ( + fileUpload struct { + File *multipart.FileHeader `form:"file" formMultipart:"file" swaggerignore:"true"` + // Optional. + Caption string `form:"caption" formMultipart:"caption"` + // Optional. + Expiration string `form:"expiration" formMultipart:"expiration" swaggerignore:"expiration"` + Size uint64 `form:"size" formMultipart:"size"` + Alt string `form:"alt" formMultipart:"alt"` + MediaType string `form:"media_type" formMultipart:"media_type"` + ContentType string `form:"content_type" formMultipart:"content_type"` + NoTransform string `form:"no_transform" formMultipart:"no_transform"` + } + fileUploadResponse struct { + Status string `json:"status"` + Message string `json:"message"` + ProcessingURL string `json:"processing_url"` + Nip94Event struct { + Tags nostr.Tags `json:"tags"` + Content string `json:"content"` + } `json:"nip94_event"` + } + listedFiles struct { + Total uint32 `json:"total"` + Page uint32 `json:"page"` + Files []struct { + Tags nostr.Tags `json:"tags"` + Content string `json:"content"` + CreatedAt uint64 `json:"created_at"` + } + } +) + +func (storageHandler) NIP96Info() gin.HandlerFunc { + return func(c *gin.Context) { + c.Data(http.StatusOK, "application/json", []byte(nip96Info)) + } +} + +func (s *storageHandler) Upload() gin.HandlerFunc { + return func(gCtx *gin.Context) { + now := time.Now() + ctx, cancel := context.WithTimeout(gCtx, mediaEndpointTimeout) + defer cancel() + authHeader := getAuthHeader(gCtx) + token, authErr := s.auth.VerifyToken(gCtx, authHeader, now) + if authErr != nil { + log.Printf("ERROR: endpoint authentification failed: %v", errors.Wrap(authErr, "endpoint authentification failed")) + gCtx.JSON(http.StatusUnauthorized, uploadErr("Unauthorized")) + return + } + attestationValid := token.ValidateAttestation(ctx, nostr.KindFileMetadata, now) + if attestationValid != nil { + log.Printf("ERROR: on-behalf attestation failed: %v", errors.Wrap(attestationValid, "endpoint authentification failed")) + gCtx.JSON(http.StatusForbidden, uploadErr("Forbidden: on-behalf attestation failed")) + return + } + + var upload fileUpload + if err := gCtx.ShouldBindWith(&upload, binding.FormMultipart); err != nil { + log.Printf("ERROR: failed to bind multipart form: %v", errors.Wrap(err, "failed to bind multipart form")) + gCtx.JSON(http.StatusBadRequest, uploadErr("invalid multipart data")) + return + } + if upload.Size > maxUploadSize { + gCtx.JSON(http.StatusRequestEntityTooLarge, uploadErr("file too large")) + return + } + if upload.File == nil { + gCtx.JSON(http.StatusBadRequest, uploadErr("file required")) + return + } + if upload.MediaType != "" && upload.MediaType != mediaTypeAvatar && upload.MediaType != mediaTypeBanner { + gCtx.JSON(http.StatusBadRequest, uploadErr(fmt.Sprintf("unsupported media type %v", upload.MediaType))) + return + } + if upload.ContentType == "" { + upload.ContentType = gomime.TypeByExtension(filepath.Ext(upload.File.Filename)) + } + + storagePath, _ := s.storageClient.BuildUserPath(token.MasterPubKey(), upload.ContentType) + uploadingFilePath := filepath.Join(storagePath, upload.File.Filename) + relativePath := upload.File.Filename + if err := os.MkdirAll(filepath.Dir(uploadingFilePath), 0o755); err != nil { + log.Printf("ERROR: %v", errors.Wrap(err, "failed to open temp file while processing upload")) + gCtx.JSON(http.StatusInternalServerError, uploadErr("failed to open temporary file")) + return + } + fileUploadTo, err := os.Create(uploadingFilePath) + if err != nil { + log.Printf("ERROR: %v", errors.Wrap(err, "failed to open temp file while processing upload")) + gCtx.JSON(http.StatusInternalServerError, uploadErr("failed to open temporary file")) + return + } + defer fileUploadTo.Close() + mpFile, err := upload.File.Open() + if err != nil { + log.Printf("ERROR: %v", errors.Wrap(err, "failed to open upload file")) + gCtx.JSON(http.StatusInternalServerError, uploadErr("failed to open upload file")) + return + } + defer mpFile.Close() + hashCalc := sha256.New() + if _, err = io.Copy(fileUploadTo, io.TeeReader(mpFile, hashCalc)); err != nil { + log.Printf("ERROR: %v", errors.Wrap(err, "failed to copy temp file while processing upload")) + gCtx.JSON(http.StatusBadRequest, uploadErr("failed to store temporary file")) + return + } + if err = fileUploadTo.Sync(); err != nil { + log.Printf("ERROR: %v", errors.Wrap(err, "failed to copy temp file while processing upload")) + gCtx.JSON(http.StatusBadRequest, uploadErr("failed to store temporary file")) + return + } + hash := hashCalc.Sum(nil) + hashHex := hex.EncodeToString(hash) + if hashHex != token.ExpectedHash() { + log.Printf("ERROR: endpoint authentification failed: %v", errors.Errorf("payload hash mismatch actual>%v token>%v", hashHex, token.ExpectedHash())) + gCtx.JSON(http.StatusForbidden, uploadErr("Unauthorized")) + os.Remove(uploadingFilePath) + return + } + bagID, url, existed, err := s.storageClient.StartUpload(ctx, token.PubKey(), token.MasterPubKey(), relativePath, hex.EncodeToString(hash), &storage.FileMetaInput{ + Hash: hash, + Caption: upload.Caption, + Alt: upload.Alt, + CreatedAt: uint64(now.UnixNano()), + }) + + if err != nil { + log.Printf("ERROR: failed to upload file: %v", errors.Wrap(err, "failed to upload file to ion storage")) + gCtx.JSON(http.StatusInternalServerError, uploadErr("oops, error occured!")) + os.Remove(uploadingFilePath) + return + } + resStatus := http.StatusCreated + if existed { + resStatus = http.StatusOK + } + gCtx.JSON(resStatus, fileUploadResponse{ + Status: "success", + Message: "Upload successful.", + Nip94Event: struct { + Tags nostr.Tags `json:"tags"` + Content string `json:"content"` + }{ + Tags: nostr.Tags{ + nostr.Tag{"url", url}, + nostr.Tag{"ox", hashHex}, + nostr.Tag{"x", hashHex}, + nostr.Tag{"m", upload.ContentType}, + nostr.Tag{"i", bagID}, + nostr.Tag{"alt", upload.Alt}, + nostr.Tag{"size", strconv.FormatUint(uint64(upload.File.Size), 10)}, + }, + Content: upload.Caption, + }, + }) + return + } +} +func (s *storageHandler) Download() gin.HandlerFunc { + return func(gCtx *gin.Context) { + now := time.Now() + authHeader := getAuthHeader(gCtx) + token, authErr := s.auth.VerifyToken(gCtx, authHeader, now) + if authErr != nil { + log.Printf("ERROR: endpoint authentification failed: %v", errors.Wrap(authErr, "endpoint authentification failed")) + gCtx.JSON(http.StatusUnauthorized, uploadErr("Unauthorized")) + return + } + file := gCtx.Param("file") + if strings.TrimSpace(file) == "" { + gCtx.JSON(http.StatusBadRequest, uploadErr("filename is required")) + return + } + url, err := s.storageClient.DownloadUrl(token.MasterPubKey(), file) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + gCtx.Status(http.StatusNotFound) + return + } + log.Printf("ERROR: %v", errors.Wrap(err, "failed to build download url")) + gCtx.JSON(http.StatusInternalServerError, uploadErr("oops, error occured!")) + return + } + gCtx.Redirect(http.StatusFound, url) + } +} +func (s *storageHandler) Delete() gin.HandlerFunc { + return func(gCtx *gin.Context) { + now := time.Now() + ctx, cancel := context.WithTimeout(gCtx, mediaEndpointTimeout) + defer cancel() + authHeader := getAuthHeader(gCtx) + token, authErr := s.auth.VerifyToken(gCtx, authHeader, now) + if authErr != nil { + log.Printf("ERROR: endpoint authentification failed: %v", errors.Wrap(authErr, "endpoint authentification failed")) + gCtx.JSON(http.StatusUnauthorized, uploadErr("Unauthorized")) + return + } + attestationValid := token.ValidateAttestation(ctx, nostr.KindFileMetadata, now) + if attestationValid != nil { + log.Printf("ERROR: on-behalf attestation failed: %v", errors.Wrap(attestationValid, "endpoint authentification failed")) + gCtx.JSON(http.StatusForbidden, uploadErr("Forbidden: on-behalf attestation failed")) + return + } + file := gCtx.Param("file") + if strings.TrimSpace(file) == "" { + gCtx.JSON(http.StatusBadRequest, uploadErr("filehash is required")) + return + } + if err := s.storageClient.Delete(token.PubKey(), token.MasterPubKey(), file); err != nil { + log.Printf("ERROR: %v", errors.Wrap(err, "failed to delete file")) + if errors.Is(err, storage.ErrNotFound) || errors.Is(err, storage.ErrForbidden) { + gCtx.JSON(http.StatusForbidden, uploadErr("user do not own file")) + return + } + gCtx.JSON(http.StatusInternalServerError, uploadErr("oops, error occured!")) + return + } + gCtx.JSON(http.StatusOK, map[string]any{"status": "success", "message": "deleted"}) + } +} + +func (s *storageHandler) ListFiles() gin.HandlerFunc { + return func(gCtx *gin.Context) { + now := time.Now() + authHeader := getAuthHeader(gCtx) + token, authErr := s.auth.VerifyToken(gCtx, authHeader, now) + if authErr != nil { + log.Printf("ERROR: endpoint authentification failed: %v", errors.Wrap(authErr, "endpoint authentification failed")) + gCtx.JSON(http.StatusUnauthorized, uploadErr("Unauthorized")) + return + } + var params struct { + Page uint32 `form:"page"` + Count uint32 `form:"count"` + } + if err := gCtx.ShouldBindWith(¶ms, binding.Query); err != nil { + log.Printf("ERROR: failed to bind data : %v", errors.Wrap(err, "failed to bind data")) + gCtx.JSON(http.StatusBadRequest, uploadErr("invalid data")) + return + } + if params.Count == 0 { + params.Count = 10 + } + total, filesList, err := s.storageClient.ListFiles(token.MasterPubKey(), params.Page, params.Count) + if err != nil { + log.Printf("ERROR: %v", errors.Wrapf(err, "failed to list files for user %v", token.MasterPubKey())) + gCtx.JSON(http.StatusInternalServerError, uploadErr("oops, error occured!")) + return + } + res := &listedFiles{ + Total: total, + Page: params.Page, + Files: []struct { + Tags nostr.Tags `json:"tags"` + Content string `json:"content"` + CreatedAt uint64 `json:"created_at"` + }{}, + } + for _, f := range filesList { + res.Files = append(res.Files, struct { + Tags nostr.Tags `json:"tags"` + Content string `json:"content"` + CreatedAt uint64 `json:"created_at"` + }{Tags: f.ToTags(), Content: f.Content, CreatedAt: f.CreatedAt}) + } + gCtx.JSON(http.StatusOK, res) + } +} + +func uploadErr(message string) any { + return map[string]any{"status": "error", "message": message} +} + +func NewUploadHandler(ctx context.Context) Uploader { + s := &storageHandler{storageClient: storage.Client(), auth: NewAuth()} + return s +} diff --git a/server/http/storage_nip_96_test.go b/server/http/storage_nip_96_test.go new file mode 100644 index 0000000..5207fca --- /dev/null +++ b/server/http/storage_nip_96_test.go @@ -0,0 +1,511 @@ +// SPDX-License-Identifier: ice License 1.0 + +package http + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "embed" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "math/big" + "mime/multipart" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/cockroachdb/errors" + gomime "github.com/cubewise-code/go-mime" + "github.com/jamiealquiza/tachymeter" + "github.com/nbd-wtf/go-nostr" + "github.com/nbd-wtf/go-nostr/nip94" + "github.com/nbd-wtf/go-nostr/nip96" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/http2" + + "github.com/ice-blockchain/subzero/database/query" + "github.com/ice-blockchain/subzero/model" + "github.com/ice-blockchain/subzero/storage" + storagefixture "github.com/ice-blockchain/subzero/storage/fixture" +) + +//go:embed .testdata +var testdata embed.FS + +func TestNIP96(t *testing.T) { + t.Parallel() + now := time.Now().Unix() + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + defer cancel() + defer func() { + require.NoError(t, storage.Client().Close()) + require.NoError(t, os.RemoveAll("./../../.test-uploads")) + require.NoError(t, os.RemoveAll("./../../.test-uploads2")) + }() + master := "c24f7ab5b42254d6558e565ec1c170b266a7cd2be1edf9f42bfb375640f7f559" + masterPubKey, _ := nostr.GetPublicKey(master) + user1 := "3c00c01e6556c4b603b4c49d12059e02c42161d055b658e5635fa6206f594306" + user2 := "cea41ff6c6e9eb0cde6740a1fbe8c134bda650ce819e43b68bf61add2c68f8d9" + var tagsToBroadcast nostr.Tags + var contentToBroadcast string + t.Run("create on-behalf attestations", func(t *testing.T) { + user1PubKey, _ := nostr.GetPublicKey(user1) + user2PubKey, _ := nostr.GetPublicKey(user2) + var ev model.Event + ev.Kind = model.CustomIONKindAttestation + ev.CreatedAt = 1 + ev.Tags = model.Tags{ + {model.TagAttestationName, user1PubKey, "", model.CustomIONAttestationKindActive + ":" + strconv.Itoa(int(now-10))}, + {model.TagAttestationName, user2PubKey, "", model.CustomIONAttestationKindActive + ":" + strconv.Itoa(int(now-5))}, + } + require.NoError(t, ev.Sign(master)) + require.NoError(t, query.AcceptEvent(ctx, &ev)) + }) + t.Run("files are uploaded, response is ok", func(t *testing.T) { + var responses chan *nip96.UploadResponse + responses = make(chan *nip96.UploadResponse, 2) + upload(t, ctx, user1, masterPubKey, ".testdata/image2.png", "profile.png", "ice profile pic", func(resp *nip96.UploadResponse) { + responses <- resp + }) + upload(t, ctx, user1, masterPubKey, ".testdata/image.jpg", "ice.jpg", "ice logo", func(resp *nip96.UploadResponse) { + responses <- resp + }) + upload(t, ctx, user2, masterPubKey, ".testdata/image2.png", "profile.png", "ice profile pic", func(resp *nip96.UploadResponse) {}) + upload(t, ctx, user2, masterPubKey, ".testdata/image.jpg", "ice.jpg", "ice logo", func(resp *nip96.UploadResponse) {}) + upload(t, ctx, user2, masterPubKey, ".testdata/text.txt", "text.txt", "text file", func(resp *nip96.UploadResponse) {}) + upload(t, ctx, master, "", ".testdata/text-master.txt", "master.txt", "master's file", func(resp *nip96.UploadResponse) {}) + close(responses) + for resp := range responses { + verifyFile(t, resp.Nip94Event.Content, resp.Nip94Event.Tags) + tagsToBroadcast = resp.Nip94Event.Tags + contentToBroadcast = resp.Nip94Event.Content + } + }) + const newStorageRoot = "./../../.test-uploads2" + var nip94EventToSign *model.Event + t.Run("nip-94 event is broadcasted, it causes download to other node", func(t *testing.T) { + tagsToBroadcast = tagsToBroadcast.AppendUnique(model.Tag{"b", masterPubKey}) + nip94EventToSign = &model.Event{nostr.Event{ + CreatedAt: nostr.Timestamp(time.Now().Unix()), + Kind: nostr.KindFileMetadata, + Tags: tagsToBroadcast, + Content: contentToBroadcast, + }} + require.NoError(t, nip94EventToSign.Sign(user1)) + // Simulate another storage node where we broadcast event/bag, and it needs to download it. + initStorage(ctx, newStorageRoot) + require.NoError(t, query.AcceptEvent(ctx, nip94EventToSign)) + require.NoError(t, storage.AcceptEvent(ctx, nip94EventToSign)) + downloadedProfileHash, err := storagefixture.WaitForFile(ctx, newStorageRoot, filepath.Join(newStorageRoot, masterPubKey, "profile.png"), "b2b8cf9202b45dad7e137516bcf44b915ce30b39c3b294629a9b6b8fa1585292", int64(182744)) + require.NoError(t, err) + require.Equal(t, "b2b8cf9202b45dad7e137516bcf44b915ce30b39c3b294629a9b6b8fa1585292", downloadedProfileHash) + downloadedLogoHash, err := storagefixture.WaitForFile(ctx, newStorageRoot, filepath.Join(newStorageRoot, masterPubKey, "ice.jpg"), "777d453395088530ce8de776fe54c3e5ace548381007b743e067844858962218", int64(415939)) + require.NoError(t, err) + require.Equal(t, "777d453395088530ce8de776fe54c3e5ace548381007b743e067844858962218", downloadedLogoHash) + }) + + t.Run("download endpoint redirects to same download url over ton storage", func(t *testing.T) { + expected := nip94.ParseFileMetadata(nostr.Event{Tags: expectedResponse("ice logo").Nip94Event.Tags}) + status, location := download(t, ctx, user1, "777d453395088530ce8de776fe54c3e5ace548381007b743e067844858962218", masterPubKey) + require.Equal(t, http.StatusFound, status) + require.Regexp(t, fmt.Sprintf("^http://[0-9a-fA-F]{64}.bag/%v", expected.Summary), location) + + expected = nip94.ParseFileMetadata(nostr.Event{Tags: expectedResponse("ice profile pic").Nip94Event.Tags}) + status, location = download(t, ctx, user1, "b2b8cf9202b45dad7e137516bcf44b915ce30b39c3b294629a9b6b8fa1585292", masterPubKey) + require.Equal(t, http.StatusFound, status) + require.Regexp(t, fmt.Sprintf("^http://[0-9a-fA-F]{64}.bag/%v", expected.Summary), location) + status, _ = download(t, ctx, user1, "non_valid_hash") + require.Equal(t, http.StatusNotFound, status) + }) + t.Run("list files responds with up to all files for the user when total is less than page", func(t *testing.T) { + files := list(t, ctx, user1, 0, 0, masterPubKey) + assert.Equal(t, uint32(4), files.Total) + assert.Len(t, files.Files, 4) + for _, f := range files.Files { + verifyFile(t, f.Content, f.Tags) + } + }) + t.Run("list files with pagination", func(t *testing.T) { + files := list(t, ctx, user1, 0, 1, masterPubKey) + assert.Equal(t, uint32(4), files.Total) + assert.Len(t, files.Files, 1) + uniqFiles := map[string]struct{}{} + for _, f := range files.Files { + verifyFile(t, f.Content, f.Tags) + uniqFiles[f.Content] = struct{}{} + } + files = list(t, ctx, user1, 1, 1, masterPubKey) + assert.Equal(t, uint32(4), files.Total) + assert.Len(t, files.Files, 1) + for _, f := range files.Files { + verifyFile(t, f.Content, f.Tags) + _, presentedBefore := uniqFiles[f.Content] + require.False(t, presentedBefore) + } + }) + t.Run("delete file owned by user 1 on behave of usr 1 (normally)", func(t *testing.T) { + fileHash := "" + if xTag := nip94EventToSign.Tags.GetFirst([]string{"x"}); xTag != nil && len(*xTag) > 1 { + fileHash = xTag.Value() + } else { + t.Fatalf("malformed x tag in nip94 event %v", nip94EventToSign.ID) + } + status := deleteFile(t, ctx, user1, fileHash, masterPubKey) + require.Equal(t, http.StatusOK, status) + fileName := nip94.ParseFileMetadata(nostr.Event{Tags: expectedResponse(nip94EventToSign.Content).Nip94Event.Tags}).Summary + require.NoFileExists(t, filepath.Join(storageRoot, masterPubKey, fileName)) + deletionEventToSign := &model.Event{nostr.Event{ + CreatedAt: nostr.Timestamp(time.Now().Unix()), + Kind: nostr.KindDeletion, + Tags: nostr.Tags{ + nostr.Tag{"e", nip94EventToSign.ID}, + nostr.Tag{"k", strconv.FormatInt(int64(nostr.KindFileMetadata), 10)}, + nostr.Tag{"b", masterPubKey}, + }, + }} + require.NoError(t, deletionEventToSign.Sign(user1)) + require.NoError(t, storage.AcceptEvent(ctx, deletionEventToSign)) + require.NoFileExists(t, filepath.Join(newStorageRoot, masterPubKey, fileName)) + }) + t.Run("delete file owned by user 2 on behave of usr 1 (attestation)", func(t *testing.T) { + status := deleteFile(t, ctx, user1, "982d9e3eb996f559e633f4d194def3761d909f5a3b647d1a851fead67c32c9d1", masterPubKey) + require.Equal(t, http.StatusOK, status) + fileName := "text.txt" + require.NoFileExists(t, filepath.Join(storageRoot, masterPubKey, fileName)) + deletionEventToSign := &model.Event{nostr.Event{ + CreatedAt: nostr.Timestamp(time.Now().Unix()), + Kind: nostr.KindDeletion, + Tags: nostr.Tags{ + nostr.Tag{"e", nip94EventToSign.ID}, + nostr.Tag{"k", strconv.FormatInt(int64(nostr.KindFileMetadata), 10)}, + nostr.Tag{"b", masterPubKey}, + }, + }} + require.NoError(t, deletionEventToSign.Sign(user1)) + require.NoError(t, storage.AcceptEvent(ctx, deletionEventToSign)) + require.NoFileExists(t, filepath.Join(newStorageRoot, masterPubKey, fileName)) + }) + t.Run("delete file owned by master by usr1 (Forbidden)", func(t *testing.T) { + status := deleteFile(t, ctx, user1, "fc613b4dfd6736a7bd268c8a0e74ed0d1c04a959f59dd74ef2874983fd443fc9", masterPubKey) + require.Equal(t, http.StatusForbidden, status) + fileName := "master.txt" + require.FileExists(t, filepath.Join(storageRoot, masterPubKey, fileName)) + }) + +} + +func verifyFile(t *testing.T, content string, tags nostr.Tags) { + t.Helper() + md := nip94.ParseFileMetadata(nostr.Event{Tags: tags}) + expected := nip94.ParseFileMetadata(nostr.Event{Tags: expectedResponse(content).Nip94Event.Tags}) + url := md.URL + bagID := md.TorrentInfoHash + expectedFileName := expected.Summary + expected.Summary = "" + md.URL = "" + md.TorrentInfoHash = "" + require.Equal(t, expected, md) + require.Contains(t, url, fmt.Sprintf("http://%v.bag/%v", bagID, expectedFileName)) + require.Regexp(t, fmt.Sprintf("^http://[0-9a-fA-F]{64}.bag/%v", expectedFileName), url) + require.Regexp(t, "^[0-9a-fA-F]{64}$", bagID) +} + +func upload(t *testing.T, ctx context.Context, sk, master, path, filename, caption string, result func(resp *nip96.UploadResponse)) { + t.Helper() + img, _ := testdata.Open(path) + defer img.Close() + var requestBody bytes.Buffer + fileHash := sha256.New() + writer := multipart.NewWriter(&requestBody) + fileWriter, err := writer.CreateFormFile("file", filename) + require.NoError(t, err) + _, err = io.Copy(fileWriter, io.TeeReader(img, fileHash)) + require.NoError(t, err) + require.NoError(t, writer.WriteField("caption", caption)) + require.NoError(t, writer.WriteField("content_type", gomime.TypeByExtension(filepath.Ext(path)))) + require.NoError(t, writer.WriteField("no_transform", "true")) + err = writer.Close() + require.NoError(t, err) + httpResp := authorizedReq(t, ctx, sk, "POST", "https://localhost:9997/files", hex.EncodeToString(fileHash.Sum(nil)), writer.FormDataContentType(), &requestBody, master) + require.NotNil(t, httpResp) + switch httpResp.StatusCode { + case http.StatusOK, http.StatusCreated, http.StatusAccepted: + var resp nip96.UploadResponse + err = json.NewDecoder(httpResp.Body).Decode(&resp) + require.NoError(t, err) + result(&resp) + default: + t.Fatalf("unexpected http status code %v for upload %v by %v", httpResp.StatusCode, filename, sk) + } +} + +func download(t *testing.T, ctx context.Context, sk, fileHash string, masterPubkey ...string) (status int, locationUrl string) { + t.Helper() + http.DefaultClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + defer func() { + http.DefaultClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { return nil } + }() + resp := authorizedReq(t, ctx, sk, "GET", fmt.Sprintf("https://localhost:9997/files/%v", fileHash), "", "", nil, masterPubkey...) + if resp.StatusCode == http.StatusFound { + require.Equal(t, http.StatusFound, resp.StatusCode) + locationUrl = resp.Header.Get("location") + require.NotEmpty(t, locationUrl) + return resp.StatusCode, locationUrl + } + return resp.StatusCode, "" +} + +func list(t *testing.T, ctx context.Context, sk string, page, limit uint32, masterPubkey ...string) *listedFiles { + t.Helper() + resp := authorizedReq(t, ctx, sk, "GET", fmt.Sprintf("https://localhost:9997/files?page=%v&count=%v", page, limit), "", "", nil, masterPubkey...) + require.Equal(t, http.StatusOK, resp.StatusCode) + var files listedFiles + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(body, &files)) + return &files +} + +func deleteFile(t *testing.T, ctx context.Context, sk string, fileHash string, masterKey ...string) int { + t.Helper() + resp := authorizedReq(t, ctx, sk, "DELETE", fmt.Sprintf("https://localhost:9997/files/%v", fileHash), "", "", nil, masterKey...) + if resp.StatusCode == http.StatusOK { + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + var respBody struct { + Message string `json:"message"` + Status string `json:"status"` + } + require.NoError(t, json.Unmarshal(body, &respBody)) + require.Equal(t, "success", respBody.Status) + require.Equal(t, "deleted", respBody.Message) + return resp.StatusCode + } + return resp.StatusCode +} + +func authorizedReq(t *testing.T, ctx context.Context, sk, method, url, fileHash, contentType string, body io.Reader, masterKey ...string) *http.Response { + t.Helper() + uploadReq, err := http.NewRequest(method, url, body) + uploadReq.Header.Set("Content-Type", contentType) + require.NoError(t, err) + auth, err := generateAuthHeader(t, sk, method, fileHash, uploadReq.URL, masterKey...) + require.NoError(t, err) + uploadReq.Header.Set("Authorization", auth) + resp, err := http.DefaultClient.Do(uploadReq.WithContext(ctx)) + require.NoError(t, err) + require.NotNil(t, resp) + return resp +} + +func expectedResponse(caption string) *nip96.UploadResponse { + expectedResponses := map[string]*nip96.UploadResponse{ + "ice profile pic": &nip96.UploadResponse{ + Status: "success", + Message: "Upload successful.", + ProcessingURL: "", + Nip94Event: struct { + Tags nostr.Tags `json:"tags"` + Content string `json:"content"` + }{ + Tags: nostr.Tags{ + nostr.Tag{"summary", "profile.png"}, + nostr.Tag{"ox", "b2b8cf9202b45dad7e137516bcf44b915ce30b39c3b294629a9b6b8fa1585292"}, + nostr.Tag{"x", "b2b8cf9202b45dad7e137516bcf44b915ce30b39c3b294629a9b6b8fa1585292"}, + nostr.Tag{"m", "image/png"}, + nostr.Tag{"size", "182744"}, + }, + Content: "ice profile pic", + }, + }, + "ice logo": &nip96.UploadResponse{ + Status: "success", + Message: "Upload successful.", + ProcessingURL: "", + Nip94Event: struct { + Tags nostr.Tags `json:"tags"` + Content string `json:"content"` + }{ + Tags: nostr.Tags{ + nostr.Tag{"summary", "ice.jpg"}, + nostr.Tag{"ox", "777d453395088530ce8de776fe54c3e5ace548381007b743e067844858962218"}, + nostr.Tag{"x", "777d453395088530ce8de776fe54c3e5ace548381007b743e067844858962218"}, + nostr.Tag{"m", "image/png"}, + nostr.Tag{"size", "415939"}, + }, + Content: "ice profile pic", + }, + }, + "text file": &nip96.UploadResponse{ + Status: "success", + Message: "Upload successful.", + ProcessingURL: "", + Nip94Event: struct { + Tags nostr.Tags `json:"tags"` + Content string `json:"content"` + }{ + Tags: nostr.Tags{ + nostr.Tag{"summary", "text.txt"}, + nostr.Tag{"ox", "982d9e3eb996f559e633f4d194def3761d909f5a3b647d1a851fead67c32c9d1"}, + nostr.Tag{"x", "982d9e3eb996f559e633f4d194def3761d909f5a3b647d1a851fead67c32c9d1"}, + nostr.Tag{"m", "text/plain"}, + nostr.Tag{"size", "4"}, + }, + Content: "text file", + }, + }, + "master's file": &nip96.UploadResponse{ + Status: "success", + Message: "Upload successful.", + ProcessingURL: "", + Nip94Event: struct { + Tags nostr.Tags `json:"tags"` + Content string `json:"content"` + }{ + Tags: nostr.Tags{ + nostr.Tag{"summary", "master.txt"}, + nostr.Tag{"ox", "fc613b4dfd6736a7bd268c8a0e74ed0d1c04a959f59dd74ef2874983fd443fc9"}, + nostr.Tag{"x", "fc613b4dfd6736a7bd268c8a0e74ed0d1c04a959f59dd74ef2874983fd443fc9"}, + nostr.Tag{"m", "text/plain"}, + nostr.Tag{"size", "6"}, + }, + Content: "master's file", + }, + }, + } + return expectedResponses[caption] +} + +func initStorage(ctx context.Context, path string) { + transportOverride := http.DefaultClient.Transport + http.DefaultClient.Transport = http.DefaultTransport + _, nodeKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + log.Panic(errors.Wrapf(err, "failed to generate node key")) + } + storagePort, err := rand.Int(rand.Reader, big.NewInt(63500)) + if err != nil { + log.Panic(errors.Wrapf(err, "failed to generate port number")) + } + wd, _ := os.Getwd() + rootStorage := filepath.Join(wd, path) + port := int(storagePort.Int64()) + 1024 + storage.MustInit(ctx, nodeKey, storage.DefaultConfigUrl, rootStorage, net.ParseIP("127.0.0.1"), port, true) + http.DefaultClient.Transport = transportOverride +} + +func generateAuthHeader(t *testing.T, sk, method, fileHash string, urlValue *url.URL, masterPubkey ...string) (string, error) { + t.Helper() + pk, err := nostr.GetPublicKey(sk) + if err != nil { + return "", fmt.Errorf("nostr.GetPublicKey: %w", err) + } + + event := nostr.Event{ + Kind: nostrHttpAuthKind, + PubKey: pk, + CreatedAt: nostr.Now(), + Tags: nostr.Tags{ + nostr.Tag{"u", urlValue.String()}, + nostr.Tag{"method", method}, + nostr.Tag{"payload", fileHash}, + }, + } + if len(masterPubkey) > 0 && masterPubkey[0] != "" { + event.Tags = append(event.Tags, nostr.Tag{"b", masterPubkey[0]}) + } + require.NoError(t, event.Sign(sk)) + + b, err := json.Marshal(event) + if err != nil { + return "", fmt.Errorf("json.Marshal: %w", err) + } + + payload := base64.StdEncoding.EncodeToString(b) + + return fmt.Sprintf("Nostr %s", payload), nil +} + +const benchParallelism = 100 + +func BenchmarkUploadFiles(b *testing.B) { + if os.Getenv("CI") != "" { + b.Skip() + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + http.DefaultClient.Transport = &http2.Transport{TLSClientConfig: &tls.Config{}} + meter := tachymeter.New(&tachymeter.Config{Size: b.N}) + b.ResetTimer() + b.ReportAllocs() + fmt.Println(b.N) + b.SetParallelism(benchParallelism) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + sk := nostr.GeneratePrivateKey() + img, _ := testdata.Open(".testdata/image2.png") + defer img.Close() + start := time.Now() + resp, err := nip96.Upload(ctx, nip96.UploadRequest{ + Host: "https://localhost:9997/files", + File: img, + Filename: "profile.png", + Caption: "ice profile pic", + ContentType: "image/png", + SK: sk, + SignPayload: true, + }) + require.NoError(b, err) + meter.AddTime(time.Since(start)) + nip94Event := &model.Event{nostr.Event{ + CreatedAt: nostr.Timestamp(time.Now().Unix()), + Kind: nostr.KindFileMetadata, + Tags: resp.Nip94Event.Tags, + }} + require.NoError(b, nip94Event.Sign(sk)) + + relay := nostr.NewRelay(ctx, "wss://localhost:9998/") + err = relay.ConnectWithTLS(ctx, &tls.Config{}) + if err = nip94Event.Sign(sk); err != nil { + log.Panic(err) + } + if err = relay.Publish(ctx, nip94Event.Event); err != nil { + log.Panic(err) + } + b.Log(nip94Event) + } + }) + helperBenchReportMetrics(b, meter) +} + +func helperBenchReportMetrics( + t interface { + Helper() + ReportMetric(float64, string) + }, + meter *tachymeter.Tachymeter, +) { + t.Helper() + + metric := meter.Calc() + t.ReportMetric(float64(metric.Time.Avg.Milliseconds()), "avg-ms/op") + t.ReportMetric(float64(metric.Time.StdDev.Milliseconds()), "stddev-ms/op") + t.ReportMetric(float64(metric.Time.P50.Milliseconds()), "p50-ms/op") + t.ReportMetric(float64(metric.Time.P95.Milliseconds()), "p95-ms/op") +} diff --git a/server/server.go b/server/server.go index 0138acf..8a2a4c4 100644 --- a/server/server.go +++ b/server/server.go @@ -18,6 +18,12 @@ func ListenAndServe(ctx context.Context, cancel context.CancelFunc, config *wsse wsserver.New(config, &router{}).ListenAndServe(ctx, cancel) } -func (r *router) RegisterRoutes(wsroutes *wsserver.Router) { - wsroutes.Any("/", wsserver.WithWS(wsserver.NewHandler(), httpserver.NewNIP11Handler())) +func (r *router) RegisterRoutes(ctx context.Context, wsroutes wsserver.Router) { + uploader := httpserver.NewUploadHandler(ctx) + wsroutes.Any("/", wsserver.WithWS(wsserver.NewHandler(), httpserver.NewNIP11Handler())). + POST("/files", uploader.Upload()). + GET("/files", uploader.ListFiles()). + GET("/files/:file", uploader.Download()). + DELETE("/files/:file", uploader.Delete()). + GET("/.well-known/nostr/nip96.json", uploader.NIP96Info()) } diff --git a/server/ws/fixture/contract.go b/server/ws/fixture/contract.go index ca209c2..257085f 100644 --- a/server/ws/fixture/contract.go +++ b/server/ws/fixture/contract.go @@ -12,6 +12,8 @@ import ( "sync/atomic" stdlibtime "time" + "github.com/gin-gonic/gin" + h2ec "github.com/ice-blockchain/go/src/net/http" "github.com/ice-blockchain/subzero/server/ws/internal" "github.com/ice-blockchain/subzero/server/ws/internal/adapters" @@ -27,12 +29,14 @@ var ( type ( MockService struct { - server internal.Server - handlersMx sync.Mutex - Handlers map[adapters.WSWriter]struct{} - processingFunc func(ctx context.Context, writer adapters.WSWriter, in []byte, cfg *config.Config) - nip11Handler http.Handler - ReaderExited atomic.Uint64 + server internal.Server + + handlersMx sync.Mutex + Handlers map[adapters.WSWriter]struct{} + processingFunc func(ctx context.Context, writer adapters.WSWriter, in []byte, cfg *config.Config) + nip11Handler http.Handler + extraHttpHandlers map[string]gin.HandlerFunc + ReaderExited atomic.Uint64 } Client interface { Received diff --git a/server/ws/fixture/fixture.go b/server/ws/fixture/fixture.go index 8df8a3e..2ce5acb 100644 --- a/server/ws/fixture/fixture.go +++ b/server/ws/fixture/fixture.go @@ -5,14 +5,17 @@ package fixture import ( "context" "net/http" + "strings" + + "github.com/gin-gonic/gin" "github.com/ice-blockchain/subzero/server/ws/internal" "github.com/ice-blockchain/subzero/server/ws/internal/adapters" "github.com/ice-blockchain/subzero/server/ws/internal/config" ) -func NewTestServer(ctx context.Context, cancel context.CancelFunc, cfg *config.Config, processingFunc func(ctx context.Context, w adapters.WSWriter, in []byte, cfg *config.Config), nonWsHandler http.Handler) *MockService { - service := newMockService(processingFunc, nonWsHandler) +func NewTestServer(ctx context.Context, cancel context.CancelFunc, cfg *config.Config, processingFunc func(ctx context.Context, w adapters.WSWriter, in []byte, cfg *config.Config), nip11 http.Handler, extraHttpHandlers map[string]gin.HandlerFunc) *MockService { + service := newMockService(processingFunc, nip11, extraHttpHandlers) server := internal.NewWSServer(service, cfg) service.server = server go service.server.ListenAndServe(ctx, cancel) @@ -20,8 +23,8 @@ func NewTestServer(ctx context.Context, cancel context.CancelFunc, cfg *config.C return service } -func newMockService(processingFunc func(ctx context.Context, w adapters.WSWriter, in []byte, cfg *config.Config), nip11Handler http.Handler) *MockService { - return &MockService{processingFunc: processingFunc, Handlers: make(map[adapters.WSWriter]struct{}), nip11Handler: nip11Handler} +func newMockService(processingFunc func(ctx context.Context, w adapters.WSWriter, in []byte, cfg *config.Config), nip11Handler http.Handler, extraHttpHandlers map[string]gin.HandlerFunc) *MockService { + return &MockService{processingFunc: processingFunc, Handlers: make(map[adapters.WSWriter]struct{}), nip11Handler: nip11Handler, extraHttpHandlers: extraHttpHandlers} } func (m *MockService) Reset() { @@ -51,8 +54,14 @@ func (m *MockService) Read(ctx context.Context, w internal.WS, cfg *config.Confi } } -func (m *MockService) RegisterRoutes(r *internal.Router) { +func (m *MockService) RegisterRoutes(ctx context.Context, r internal.Router) { + for route, handler := range m.extraHttpHandlers { + parts := strings.Split(route, " ") + method, path := parts[0], parts[1] + r = r.Handle(method, path, handler) + } r.Any("/", internal.WithWS(m, m.nip11Handler)) + } func (m *MockService) Close(ctx context.Context) error { diff --git a/server/ws/internal/config/contract.go b/server/ws/internal/config/contract.go index c276703..738c359 100644 --- a/server/ws/internal/config/contract.go +++ b/server/ws/internal/config/contract.go @@ -12,5 +12,6 @@ type ( Port uint16 `yaml:"port"` WriteTimeout stdlibtime.Duration `yaml:"writeTimeout"` ReadTimeout stdlibtime.Duration `yaml:"readTimeout"` + Debug bool `yaml:"debug"` } ) diff --git a/server/ws/internal/contract.go b/server/ws/internal/contract.go index 6be3996..2142e7a 100644 --- a/server/ws/internal/contract.go +++ b/server/ws/internal/contract.go @@ -15,13 +15,13 @@ import ( ) type ( - Router = gin.Engine + Router = gin.IRoutes Server interface { // ListenAndServe starts everything and blocks indefinitely. ListenAndServe(ctx context.Context, cancel context.CancelFunc) } RegisterRoutes interface { - RegisterRoutes(router *Router) + RegisterRoutes(ctx context.Context, router Router) } WSHandler = adapters.WSHandler @@ -31,7 +31,7 @@ type ( Srv struct { H3Server http3.Server H2Server http2.Server - router *Router + router *gin.Engine cfg *config.Config quit chan<- os.Signal routesSetup RegisterRoutes diff --git a/server/ws/internal/http3/server.go b/server/ws/internal/http3/server.go index 357691b..1a2f579 100644 --- a/server/ws/internal/http3/server.go +++ b/server/ws/internal/http3/server.go @@ -42,7 +42,7 @@ func (s *srv) ListenAndServeTLS(ctx context.Context, certFile, keyFile string) e }, }, } - if /*s.cfg.Development*/ true { + if s.cfg.Debug { noCors := func(_ *http.Request) bool { return true } diff --git a/server/ws/internal/ws.go b/server/ws/internal/ws.go index c413cbb..6b3a273 100644 --- a/server/ws/internal/ws.go +++ b/server/ws/internal/ws.go @@ -33,15 +33,15 @@ func NewWSServer(router RegisterRoutes, cfg *config.Config) Server { s.router.UseRawPath = true s.H3Server = http3.New(s.cfg, s.router) s.H2Server = http2.New(s.cfg, s.router) - s.setupRouter() return s } -func (s *Srv) setupRouter() { - s.routesSetup.RegisterRoutes(s.router) +func (s *Srv) setupRouter(ctx context.Context) { + s.routesSetup.RegisterRoutes(ctx, s.router) } func (s *Srv) ListenAndServe(ctx context.Context, cancel context.CancelFunc) { + s.setupRouter(ctx) ctx = withServer(ctx, s) go s.startServer(ctx, s.H3Server) go s.startServer(ctx, s.H2Server) diff --git a/server/ws/ws_test.go b/server/ws/ws_test.go index abfa44c..f8b39ec 100644 --- a/server/ws/ws_test.go +++ b/server/ws/ws_test.go @@ -12,6 +12,7 @@ import ( stdlibtime "time" "github.com/cockroachdb/errors" + "github.com/gin-gonic/gin" "github.com/gobwas/ws" "github.com/google/uuid" "github.com/stretchr/testify/assert" @@ -50,14 +51,14 @@ func TestMain(m *testing.M) { CertPath: certFilePath, KeyPath: keyFilePath, Port: 9999, - }, echoFunc, nil) + }, echoFunc, nil, map[string]gin.HandlerFunc{}) hdl = new(handler) pubsubServer = fixture.NewTestServer(serverCtx, serverCancel, &Config{ CertPath: certFilePath, KeyPath: keyFilePath, Port: 9998, NIP13MinLeadingZeroBits: NIP13MinLeadingZeroBits, - }, hdl.Handle, nil) + }, hdl.Handle, nil, map[string]gin.HandlerFunc{}) m.Run() serverCancel() } diff --git a/storage/download.go b/storage/download.go new file mode 100644 index 0000000..ed388f7 --- /dev/null +++ b/storage/download.go @@ -0,0 +1,247 @@ +// SPDX-License-Identifier: ice License 1.0 + +package storage + +import ( + "context" + "encoding/base64" + "encoding/hex" + "encoding/json" + "log" + "net/url" + "time" + + "github.com/cockroachdb/errors" + "github.com/xssnick/tonutils-storage/storage" + + "github.com/ice-blockchain/subzero/model" +) + +func (c *client) DownloadUrl(userPubkey string, fileHash string) (string, error) { + bag, err := c.bagByUser(userPubkey) + if err != nil { + return "", errors.Wrapf(err, "failed to get bagID for the user %v", userPubkey) + } + if bag == nil { + return "", ErrNotFound + } + bs, err := c.buildBootstrapNodeInfo(bag) + if err != nil { + return "", errors.Wrapf(err, "failed to build bootstap for bag %v", hex.EncodeToString(bag.BagID)) + } + file, err := c.detectFile(bag, fileHash) + if err != nil { + return "", errors.Wrapf(err, "failed to detect file %v in bag %v", fileHash, hex.EncodeToString(bag.BagID)) + } + return c.buildUrl(hex.EncodeToString(bag.BagID), file, []*Bootstrap{bs}) +} + +func acceptNewBag(ctx context.Context, event *model.Event) error { + infohash := "" + var fileUrl *url.URL + var err error + if iTag := event.Tags.GetFirst([]string{"i"}); iTag != nil && len(*iTag) > 1 { + infohash = iTag.Value() + } else { + return errors.Newf("malformed i tag %v", iTag) + } + if urlTag := event.Tags.GetFirst([]string{"url"}); urlTag != nil && len(*urlTag) > 1 { + fileUrl, err = url.Parse(urlTag.Value()) + if err != nil { + return errors.Wrapf(err, "malformed url in url tag %v", *urlTag) + } + } else { + return errors.Newf("malformed url tag %v", urlTag) + } + bootstrap := fileUrl.Query().Get("bootstrap") + if err = globalClient.newBagIDPromoted(ctx, event.GetMasterPublicKey(), infohash, &bootstrap); err != nil { + return errors.Wrapf(err, "failed to promote new bag ID %v for user %v", infohash, event.PubKey) + } + return nil +} + +func (c *client) newBagIDPromoted(ctx context.Context, user, bagID string, bootstap *string) error { + existingBagForUser, err := c.bagByUser(user) + if err != nil { + return errors.Wrapf(err, "failed to find existing bag for user %s", user) + } + if existingBagForUser != nil { + log.Printf("[STORAGE] INFO: GOT NIP-94 with new files for user %v, replacing %v with %v", user, hex.EncodeToString(existingBagForUser.BagID), bagID) + existingBagForUser.Stop() + if err = c.progressStorage.RemoveTorrent(existingBagForUser, false); err != nil { + return errors.Wrapf(err, "failed to replace bag for user %s", user) + } + } + if err = c.download(ctx, bagID, user, bootstap); err != nil { + return errors.Wrapf(err, "failed to download new bag ID %v for user %v", bagID, user) + } + return nil +} + +func (c *client) download(ctx context.Context, bagID, user string, bootstrap *string) (err error) { + bag, err := hex.DecodeString(bagID) + if err != nil { + return errors.Wrapf(err, "invalid bagID %v", bagID) + } + if len(bag) != 32 { + return errors.Wrapf(err, "invalid bagID %v, should be len 32", bagID) + } + log.Printf("[STORAGE] INFO: ADDING %v for user %v TO DOWNLOADS, Q %v", bagID, user, len(c.downloadQueue)) + tor := c.progressStorage.GetTorrent(bag) + if tor == nil { + tor = storage.NewTorrent(c.rootStoragePath, c.progressStorage, c.conn) + tor.BagID = bag + c.downloadQueue <- queueItem{ + tor: tor, + bootstrap: bootstrap, + user: &user, + } + if err = c.saveTorrent(tor, &user, bootstrap); err != nil { + return errors.Wrapf(err, "failed to store new torrent %v", bagID) + } + } else { + if err = tor.Start(true, true, false); err != nil { + return errors.Wrapf(err, "failed to start existing torrent %v", bagID) + } + } + return nil +} + +func (c *client) torrentStateCallback(tor *storage.Torrent, user *string) func(event storage.Event) { + return func(event storage.Event) { + usr := "" + if user != nil { + usr = *user + } + switch event.Name { + case storage.EventDone: + tor.Stop() + log.Printf("[STORAGE] INFO: bag %v for user %v downloaded (%v files, %v bytes), disabling download", hex.EncodeToString(tor.BagID), usr, tor.Header.FilesCount, tor.Info.FileSize) + if pErr := tor.Start(true, false, false); pErr != nil { + log.Printf("ERROR: failed to stop torrent download after downloading data for bag %v user %v: %v", hex.EncodeToString(tor.BagID), usr, pErr) + } + c.activeDownloadsMx.Lock() + delete(c.activeDownloads, hex.EncodeToString(tor.BagID)) + c.activeDownloadsMx.Unlock() + if pErr := c.saveTorrent(tor, user, nil); pErr != nil { + log.Printf("ERROR: failed save torrent %v with stopped download after downloading: %v", hex.EncodeToString(tor.BagID), pErr) + } + + case storage.EventBagResolved: + if _, isUplActive := tor.IsActive(); !isUplActive { + log.Printf("[STORAGE] INFO: bag %v for user %v header resolved (%v files, %v bytes), enabling upload to serve clients with chunks we own", hex.EncodeToString(tor.BagID), usr, tor.Header.FilesCount, tor.Info.FileSize) + if pErr := tor.StartWithCallback(true, true, false, c.torrentStateCallback(tor, user)); pErr != nil { + log.Printf("ERROR: failed to start torrent %v upload after downloading header: %v", hex.EncodeToString(tor.BagID), pErr) + } + if user != nil { + m, err := c.fileMeta(tor) + if err != nil { + log.Printf("[STORAGE] ERROR: failed to get file meta for bag althrough it is resolved %v: %v", hex.EncodeToString(tor.BagID), err) + } + if m != nil { + *user = m.Master + } + } + if pErr := c.saveTorrent(tor, user, nil); pErr != nil { + log.Printf("ERROR: failed save torrent %v with stopped download after downloading: %v", hex.EncodeToString(tor.BagID), pErr) + } + } + case storage.EventFileDownloaded: + log.Printf("[STORAGE] DEBUG: bag %v for user %v downloaded FILE %v", hex.EncodeToString(tor.BagID), usr, event.Value) + case storage.EventErr: + log.Printf("[STORAGE] ERROR: bag %v for user %v, occured error: %v", hex.EncodeToString(tor.BagID), usr, event.Value) + } + } +} + +func (c *client) connectToBootstrap(ctx context.Context, torrent *storage.Torrent, bootstrap string) error { + b64, err := base64.StdEncoding.DecodeString(bootstrap) + if err != nil { + return errors.Wrapf(err, "failed to decode bootstrap %v", bootstrap) + } + var bootstraps []Bootstrap + if err = json.Unmarshal(b64, &bootstraps); err != nil { + return errors.Wrapf(err, "failed to decode bootstrap %v", string(b64)) + } + for _, bs := range bootstraps { + pk := bs.Overlay.ID.(map[string]any)["Key"].(string) + var pubKey []byte + pubKey, err = base64.StdEncoding.DecodeString(pk) + if err != nil { + return errors.Wrapf(err, "failed to decode bootstrap %v, invalid pubkey %v", string(b64), string(pk)) + } + if err = c.server.ConnectToNode(ctx, torrent, bs.Overlay, bs.DHT.AddrList, pubKey); err != nil { + return errors.Wrapf(err, "failed to connect to bootstrap node %#v", bs.DHT.AddrList.Addresses[0]) + } + } + return nil +} + +func (c *client) saveTorrent(tr *storage.Torrent, userPubKey *string, bs *string) error { + if err := c.progressStorage.SetTorrent(tr); err != nil { + return errors.Wrap(err, "failed to save torrent into storage") + } + if userPubKey != nil && *userPubKey != "" { + k := make([]byte, 3+64) + copy(k, "ub:") + copy(k[3:], *userPubKey) + if err := c.db.Put(k, tr.BagID, nil); err != nil { + return errors.Wrapf(err, "failed to save userID:bag mapping for bag %v", hex.EncodeToString(tr.BagID)) + } + } + if bs != nil { + k := make([]byte, 3+32) + copy(k, "bs:") + copy(k[3:], tr.BagID) + if err := c.db.Put(k, []byte(*bs), nil); err != nil { + return errors.Wrapf(err, "failed to save bootstrap node for bag %v", hex.EncodeToString(tr.BagID)) + } + } + + return nil +} + +func (c *client) startDownloadsFromQueue() { +outerLoop: + for { + c.activeDownloadsMx.RLock() + l := len(c.activeDownloads) + c.activeDownloadsMx.RUnlock() + for l < ConcurrentBagsDownloading { + select { + case q := <-c.downloadQueue: + tor := q.tor + if downloading, _ := tor.IsActive(); downloading { + continue + } + usr := "" + if q.user != nil { + usr = *q.user + } + log.Printf("[STORAGE] INFO: starting download %v for user %v Q %v", hex.EncodeToString(tor.BagID), usr, len(c.downloadQueue)) + if err := tor.StartWithCallback(false, true, false, c.torrentStateCallback(tor, q.user)); err != nil { + log.Printf("ERROR: %v", errors.Wrapf(err, "failed to start new torrent %v", q.tor.BagID)) + } + if q.bootstrap != nil && *q.bootstrap != "" { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + if err := c.connectToBootstrap(ctx, tor, *q.bootstrap); err != nil { + log.Printf("WARN: failed to connect to bootstrap node for bag %v, waiting for DHT: %v", hex.EncodeToString(q.tor.BagID), err) + } + cancel() + } + if err := c.saveTorrent(tor, q.user, q.bootstrap); err != nil { + log.Printf("ERROR: failed save updated upload / download torrrent state %v: %v", hex.EncodeToString(q.tor.BagID), err) + } + c.activeDownloadsMx.Lock() + c.activeDownloads[hex.EncodeToString(tor.BagID)] = true + c.activeDownloadsMx.Unlock() + l += 1 + default: + time.Sleep(100 * time.Millisecond) + continue outerLoop + } + } + time.Sleep(100 * time.Millisecond) + } + +} diff --git a/storage/fixture/fixture.go b/storage/fixture/fixture.go new file mode 100644 index 0000000..18b28e8 --- /dev/null +++ b/storage/fixture/fixture.go @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: ice License 1.0 + +package fixture + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "io" + "os" + "time" + + "github.com/cockroachdb/errors" + "github.com/fsnotify/fsnotify" +) + +func WaitForFile(ctx context.Context, watchPath, expectedPath, expectedHash string, expectedSize int64) (hash string, err error) { + if ctx.Err() != nil { + return "", ctx.Err() + } + skipWatch := false + fileInfo, err := os.Stat(expectedPath) + if err == nil && fileInfo.Size() == expectedSize { + skipWatch = true + } + if !skipWatch { + if err = watchFile(ctx, watchPath, expectedPath, expectedSize); err != nil { + return "", errors.Wrapf(err, "failed to monitor file %v", expectedPath) + } + } + f, err := os.Open(expectedPath) + if err != nil { + return "", errors.Wrapf(err, "failed to open %v to check hash", expectedPath) + } + defer f.Close() + hashCalc := sha256.New() + var n int64 + if n, err = io.Copy(hashCalc, f); err != nil { + return "", errors.Wrapf(err, "failed to calc hash of %v", expectedPath) + } + if n != expectedSize { + return WaitForFile(ctx, watchPath, expectedPath, expectedHash, expectedSize) + } + hash = hex.EncodeToString(hashCalc.Sum(nil)) + if hash != expectedHash { + return WaitForFile(ctx, watchPath, expectedPath, expectedHash, expectedSize) + } + return hash, nil +} + +func watchFile(ctx context.Context, monitorPath, expectedPath string, expectedSize int64) error { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return errors.Wrapf(err, "failed to create fsnotify watcher %s", expectedPath) + } + defer watcher.Close() + err = watcher.Add(monitorPath) + if err != nil { + return errors.Wrapf(err, "failed to monitor expectedPath %s", expectedPath) + } +loop: + for { + select { + case event := <-watcher.Events: + if event.Op == fsnotify.Write && event.Name == expectedPath { + fileInfo, err := os.Stat(expectedPath) + if err != nil { + return errors.Wrapf(err, "failed to stat file %s", expectedPath) + } + if fileInfo.Size() == expectedSize { + break loop + } + } + case <-time.After(1 * time.Second): + fileInfo, err := os.Stat(expectedPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + continue loop + } + return errors.Wrapf(err, "failed to stat file %s", expectedPath) + } + if fileInfo.Size() == expectedSize { + break loop + } + case err = <-watcher.Errors: + return errors.Wrapf(err, "got error from fsnotify") + case <-ctx.Done(): + return errors.New("timeout") + } + } + return nil +} diff --git a/storage/fs.go b/storage/fs.go new file mode 100644 index 0000000..150b61d --- /dev/null +++ b/storage/fs.go @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: ice License 1.0 + +package storage + +import ( + "io" + "os" + + "github.com/xssnick/tonutils-storage/storage" +) + +func init() { + storage.Fs = newFS() +} + +type fs struct{} +type fd struct { + f *os.File +} + +func (f *fd) Get() io.ReaderAt { + return f.f +} +func (f *fd) Close() error { + return f.f.Close() +} + +func newFS() storage.FSController { + return &fs{} +} + +func (f *fs) Acquire(path string) (storage.FDesc, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + return &fd{file}, nil +} + +func (fs *fs) Free(f storage.FDesc) { + f.Close() +} diff --git a/storage/global.go b/storage/global.go new file mode 100644 index 0000000..c53aec0 --- /dev/null +++ b/storage/global.go @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: ice License 1.0 + +package storage + +import ( + "context" + "crypto/ed25519" + "encoding/hex" + "fmt" + "log" + "net" + "net/url" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "time" + + "github.com/cockroachdb/errors" + "github.com/nbd-wtf/go-nostr" + "github.com/syndtr/goleveldb/leveldb" + ldbstorage "github.com/syndtr/goleveldb/leveldb/storage" + "github.com/xssnick/tonutils-go/adnl" + "github.com/xssnick/tonutils-go/adnl/dht" + "github.com/xssnick/tonutils-go/liteclient" + "github.com/xssnick/tonutils-storage/db" + "github.com/xssnick/tonutils-storage/storage" + + "github.com/ice-blockchain/subzero/database/query" + "github.com/ice-blockchain/subzero/model" + "github.com/ice-blockchain/subzero/storage/statistics" +) + +var globalClient *client + +const DefaultConfigUrl = "https://ton.org/global.config.json" + +var ConcurrentBagsDownloading = runtime.NumCPU() * 10 + +const threadsPerBagForDownloading = 7 + +func Client() StorageClient { + return globalClient +} + +func AcceptEvent(ctx context.Context, event *model.Event) error { + switch event.Kind { + case nostr.KindFileMetadata: + return acceptNewBag(ctx, event) + case nostr.KindDeletion: + matchKTag := false + if kTag := event.Tags.GetFirst([]string{"k"}); kTag != nil && len(*kTag) > 1 { + if kTag.Value() == strconv.FormatInt(int64(nostr.KindFileMetadata), 10) { + matchKTag = true + } + } + if matchKTag { + return acceptDeletion(ctx, event) + } + return nil + default: + return nil + } +} + +func acceptDeletion(ctx context.Context, event *model.Event) error { + refs, err := model.ParseEventReference(event.Tags) + if err != nil { + return errors.Wrapf(err, "failed to detect events for delete") + } + filters := model.Filters{} + for _, r := range refs { + filters = append(filters, r.Filter()) + } + events := query.GetStoredEvents(ctx, &model.Subscription{Filters: filters}) + var originalEvent *model.Event + for fileEvent, err := range events { + if err != nil { + return errors.Wrapf(err, "failed to query referenced deletion file event") + } + if fileEvent.Kind != nostr.KindFileMetadata { + return errors.Errorf("event mismatch: event %v is %v not file metadata (%v)", fileEvent.ID, fileEvent.Kind, nostr.KindFileMetadata) + } + if fileEvent.PubKey != event.PubKey { + return errors.Errorf("user mismatch: event %v is signed by %v not %v", fileEvent.ID, fileEvent.PubKey, event.PubKey) + } + originalEvent = fileEvent + break + } + if originalEvent == nil { + return nil + } + fileHash := "" + if xTag := originalEvent.Tags.GetFirst([]string{"x"}); xTag != nil && len(*xTag) > 1 { + fileHash = xTag.Value() + } else { + return errors.Errorf("malformed x tag in event %v", originalEvent.ID) + } + bag, err := globalClient.bagByUser(event.GetMasterPublicKey()) + if err != nil { + return errors.Wrapf(err, "failed to get bagID for the user %v", event.GetMasterPublicKey()) + } + if bag == nil { + return errors.Errorf("bagID for user %v not found", event.GetMasterPublicKey()) + } + file, err := globalClient.detectFile(bag, fileHash) + if err != nil { + return errors.Wrapf(err, "failed to detect file %v in bag %v", fileHash, hex.EncodeToString(bag.BagID)) + } + userRoot, _ := globalClient.BuildUserPath(event.GetMasterPublicKey(), "") + if err := os.Remove(filepath.Join(userRoot, file)); err != nil && !errors.Is(err, os.ErrNotExist) { + return errors.Wrapf(err, "failed to delete file %v", file) + } + if _, _, _, err := globalClient.StartUpload(ctx, event.PubKey, event.GetMasterPublicKey(), file, fileHash, nil); err != nil { + return errors.Wrapf(err, "failed to rebuild bag with deleted file") + } + return nil +} + +func MustInit(ctx context.Context, nodeKey ed25519.PrivateKey, tonConfigUrl, rootStorage string, externalAddress net.IP, port int, debug bool) { + globalClient = mustInit(ctx, nodeKey, tonConfigUrl, rootStorage, externalAddress, port, debug) +} + +func mustInit(ctx context.Context, nodeKey ed25519.PrivateKey, tonConfigUrl, rootStorage string, externalAddress net.IP, port int, debug bool) *client { + storage.Logger = func(a ...any) { + if debug { + log.Println(a...) + } + if len(a) > 0 { + if s, isStr := a[0].(string); isStr { + if strings.Contains(strings.ToLower(s), "err") { + log.Println(a) + } + } + } + } + storage.DownloadThreads = threadsPerBagForDownloading + adnl.Logger = func(v ...any) {} + var lsCfg *liteclient.GlobalConfig + u, err := url.Parse(tonConfigUrl) + if err != nil { + log.Panic(errors.Wrapf(err, "invalid ton config url: %v", tonConfigUrl)) + } + if u.Scheme == "file" { + lsCfg, err = liteclient.GetConfigFromFile(u.Path) + if err != nil { + log.Panic(errors.Wrapf(err, "failed to load ton network config from file: %v", u.Path)) + } + } else { + downloadConfigCtx, cancelDownloadConfig := context.WithTimeout(ctx, 30*time.Second) + defer cancelDownloadConfig() + lsCfg, err = liteclient.GetConfigFromUrl(downloadConfigCtx, tonConfigUrl) + if err != nil { + log.Panic(errors.Wrapf(err, "failed to load ton network config from url: %v", u.String())) + } + } + + gate := adnl.NewGateway(nodeKey) + gate.SetExternalIP(externalAddress) + if err = gate.StartServer(fmt.Sprintf(":%v", port)); err != nil { + log.Panic(errors.Wrapf(err, "failed to start adnl gateway")) + } + dhtGate := adnl.NewGateway(nodeKey) + if err = dhtGate.StartClient(); err != nil { + log.Panic(errors.Wrapf(err, "failed to start dht")) + } + + dhtClient, err := dht.NewClientFromConfig(dhtGate, lsCfg) + if err != nil { + log.Panic(errors.Wrapf(err, "failed to create dht client")) + } + srv := storage.NewServer(dhtClient, gate, nodeKey, true) + conn := storage.NewConnector(srv) + fStorage, err := ldbstorage.OpenFile(filepath.Join(rootStorage, "db"), false) + if err != nil { + log.Panic(errors.Wrapf(err, "failed to open leveldb storage %v", filepath.Join(rootStorage, "db"))) + } + progressDb, err := leveldb.Open(fStorage, nil) + if err != nil { + log.Panic(errors.Wrapf(err, "failed to open leveldb")) + } + cl := &client{ + conn: conn, + db: progressDb, + server: srv, + gateway: gate, + dht: dhtClient, + rootStoragePath: rootStorage, + newFiles: make(map[string]map[string]*FileMetaInput), + newFilesMx: &sync.RWMutex{}, + stats: statistics.NewStatistics(rootStorage, debug), + downloadQueue: make(chan queueItem, 1000000), + activeDownloads: make(map[string]bool), + activeDownloadsMx: &sync.RWMutex{}, + debug: debug, + } + if debug { + go cl.report(ctx) + } + loadMonitoringCh := make(chan *db.Event, 1000000) + go func() { + for ev := range loadMonitoringCh { + if ev.Event == db.EventTorrentLoaded { + if ev.Torrent != nil { + if _, uploading := ev.Torrent.IsActive(); !uploading { + if downloading := ev.Torrent.IsDownloadAll(); !downloading { + bs, bsErr := cl.bootstrapForBag(ev.Torrent.BagID) + if bsErr != nil { + log.Printf("WARN: failed to find stored bootstrap for bag %v: %v", hex.EncodeToString(ev.Torrent.BagID), bsErr) + } + var usr string + if ev.Torrent.Header != nil { + var m *headerData + m, err = cl.fileMeta(ev.Torrent) + if err != nil { + log.Printf("INFO:loading bag %v into queue but it is not resolved yet: %v", hex.EncodeToString(ev.Torrent.BagID), err) + } + if m != nil { + usr = m.Master + } + } + log.Printf("[STORAGE] INFO: bag %v not yet started before restart put it into queue", hex.EncodeToString(ev.Torrent.BagID)) + cl.downloadQueue <- queueItem{ + tor: ev.Torrent, + bootstrap: &bs, + user: &usr, + } + } + } + } + } + } + }() + progressStorage, err := db.NewStorage(progressDb, conn, true, loadMonitoringCh) + if err != nil { + log.Panic(errors.Wrapf(err, "failed to open storage")) + } + cl.progressStorage = progressStorage + cl.server.SetStorage(progressStorage) + cl.progressStorage.SetNotifier(nil) + close(loadMonitoringCh) + go cl.startDownloadsFromQueue() + return cl +} diff --git a/storage/statistics/metadata/generic.go b/storage/statistics/metadata/generic.go new file mode 100644 index 0000000..fc3a917 --- /dev/null +++ b/storage/statistics/metadata/generic.go @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: ice License 1.0 + +package metadata + +import "path/filepath" + +type genericMetaExtractor struct{} + +func newGenericExtractor() Extractor { + return &genericMetaExtractor{} +} + +func (*genericMetaExtractor) Extract(filePath, _ string, size uint64) (*Metadata, error) { + ext := filepath.Ext(filePath) + return &Metadata{ + Ext: ext, + Size: size, + }, nil +} + +func (*genericMetaExtractor) Close() error { + return nil +} diff --git a/storage/statistics/metadata/image.go b/storage/statistics/metadata/image.go new file mode 100644 index 0000000..1f363c3 --- /dev/null +++ b/storage/statistics/metadata/image.go @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: ice License 1.0 + +package metadata + +import ( + "path/filepath" + + "github.com/cockroachdb/errors" + "github.com/davidbyttow/govips/v2/vips" +) + +type imageMetaExtractor struct{} + +type ImageMetadata struct { + Width int + Height int +} + +func newImageExtractor() Extractor { + vips.Startup(nil) + return &imageMetaExtractor{} +} + +func (i *imageMetaExtractor) Extract(filePath, _ string, size uint64) (*Metadata, error) { + ext := filepath.Ext(filePath) + im, err := vips.LoadImageFromFile(filePath, nil) + if err != nil { + return nil, errors.Wrapf(err, "failed to load image %v", filePath) + } + defer im.Close() + return &Metadata{ + Ext: ext, + Size: size, + TypeMeta: &ImageMetadata{ + Width: im.Width(), + Height: im.Height(), + }, + }, nil +} + +func (*imageMetaExtractor) Close() error { + vips.Shutdown() + return nil +} diff --git a/storage/statistics/metadata/metadata.go b/storage/statistics/metadata/metadata.go new file mode 100644 index 0000000..2484906 --- /dev/null +++ b/storage/statistics/metadata/metadata.go @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: ice License 1.0 + +package metadata + +import ( + "io" + "strings" + + "github.com/cockroachdb/errors" + "github.com/hashicorp/go-multierror" +) + +type Extractor interface { + io.Closer + Extract(filePath, contentType string, size uint64) (*Metadata, error) +} +type Metadata struct { + TypeMeta any + Ext string + Size uint64 +} +type extractor struct { + extractorsByFileType map[string]Extractor + generic *genericMetaExtractor +} + +func NewExtractor() Extractor { + return &extractor{ + extractorsByFileType: map[string]Extractor{ + "video": newVideoExtractor(), + "image": newImageExtractor(), + }, + } +} +func (e *extractor) Extract(filePath, contentType string, size uint64) (*Metadata, error) { + fileType := strings.Split(contentType, "/")[0] + if ext, hasExtractor := e.extractorsByFileType[fileType]; hasExtractor { + return ext.Extract(filePath, contentType, size) + } + return e.generic.Extract(filePath, contentType, size) +} +func (e *extractor) Close() error { + var mErr *multierror.Error + for k, ex := range e.extractorsByFileType { + if clErr := ex.Close(); clErr != nil { + mErr = multierror.Append(mErr, errors.Wrapf(clErr, "failed to close %v meta extractor", k)) + } + + } + if err := e.generic.Close(); err != nil { + mErr = multierror.Append(mErr, errors.Wrapf(err, "failed to close generic meta extractor")) + } + return mErr.ErrorOrNil() +} diff --git a/storage/statistics/metadata/video.go b/storage/statistics/metadata/video.go new file mode 100644 index 0000000..e46d981 --- /dev/null +++ b/storage/statistics/metadata/video.go @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: ice License 1.0 + +package metadata + +import ( + "encoding/json" + "log" + "path/filepath" + "strings" + + "github.com/cockroachdb/errors" + ffmpeg "github.com/u2takey/ffmpeg-go" +) + +type videoMetaExtractor struct{} +type ( + VideoMetadata struct { + Format *VideoFormat `json:"format"` + Streams []*VideoStream `json:"streams"` + } + VideoFormat struct { + Tags struct { + MajorBrand string `json:"major_brand"` + MinorVersion string `json:"minor_version"` + CompatibleBrands string `json:"compatible_brands"` + Encoder string `json:"encoder"` + LocationEng string `json:"location-eng"` + Location string `json:"location"` + } `json:"tags"` + Filename string `json:"filename"` + FormatName string `json:"format_name"` + FormatLongName string `json:"format_long_name"` + StartTime string `json:"start_time"` + Duration string `json:"duration"` + Size string `json:"size"` + BitRate string `json:"bit_rate"` + NbStreams int `json:"nb_streams"` + NbPrograms int `json:"nb_programs"` + ProbeScore int `json:"probe_score"` + } + VideoStream struct { + Tags struct { + Language string `json:"language"` + HandlerName string `json:"handler_name"` + } `json:"tags"` + CodecTimeBase string `json:"codec_time_base"` + Duration string `json:"duration"` + SampleRate string `json:"sample_rate"` + CodecType string `json:"codec_type"` + BitRate string `json:"bit_rate"` + CodecTagString string `json:"codec_tag_string"` + CodecTag string `json:"codec_tag"` + SampleFmt string `json:"sample_fmt"` + Profile string `json:"profile"` + CodecLongName string `json:"codec_long_name"` + CodecName string `json:"codec_name"` + ChannelLayout string `json:"channel_layout"` + RFrameRate string `json:"r_frame_rate"` + AvgFrameRate string `json:"avg_frame_rate"` + TimeBase string `json:"time_base"` + NbFrames string `json:"nb_frames"` + StartTime string `json:"start_time"` + MaxBitRate string `json:"max_bit_rate"` + Disposition struct { + Default int `json:"default"` + Dub int `json:"dub"` + Original int `json:"original"` + Comment int `json:"comment"` + Lyrics int `json:"lyrics"` + Karaoke int `json:"karaoke"` + Forced int `json:"forced"` + HearingImpaired int `json:"hearing_impaired"` + VisualImpaired int `json:"visual_impaired"` + CleanEffects int `json:"clean_effects"` + AttachedPic int `json:"attached_pic"` + TimedThumbnails int `json:"timed_thumbnails"` + } `json:"disposition"` + BitsPerSample int `json:"bits_per_sample"` + DurationTs int `json:"duration_ts"` + StartPts int `json:"start_pts"` + Width int `json:"width"` + Height int `json:"height"` + Index int `json:"index"` + Channels int `json:"channels"` + } +) + +func (v *videoMetaExtractor) Close() error { + return nil +} + +func (v *videoMetaExtractor) Extract(filePath, _ string, size uint64) (*Metadata, error) { + res, err := ffmpeg.Probe(filePath) + if err != nil { + return nil, errors.Wrapf(err, "failed to fetch video metadata from %s", filePath) + } + var md VideoMetadata + if err = json.Unmarshal([]byte(res), &md); err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal video metadata from %s", filePath) + } + ext := filepath.Ext(filePath) + return &Metadata{ + Ext: ext, + Size: size, + TypeMeta: &md, + }, nil +} + +func newVideoExtractor() Extractor { + res, err := ffmpeg.Probe("") + if err != nil || !strings.Contains(res, "You have to specify one input file") { + if err == nil { + err = errors.New(res) + } + if strings.Contains(err.Error(), "ffprobe version ") { + err = nil + } + if err != nil { + log.Panic(errors.Wrapf(err, "failed to call ffprobe, is ffmpeg installed?")) + } + } + return &videoMetaExtractor{} +} diff --git a/storage/statistics/statistics.go b/storage/statistics/statistics.go new file mode 100644 index 0000000..010964d --- /dev/null +++ b/storage/statistics/statistics.go @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: ice License 1.0 + +package statistics + +import ( + "fmt" + "io" + "log" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/cockroachdb/errors" + "github.com/rcrowley/go-metrics" + + "github.com/ice-blockchain/subzero/storage/statistics/metadata" +) + +type ( + Statistics interface { + io.Closer + ProcessFile(filePath, contentType string, size uint64) + } + statistics struct { + metaExtractor metadata.Extractor + metrics metrics.Registry + rootStorageDir string + } + noopStats struct{} +) + +func (n *noopStats) Close() error { + return nil +} + +func (n *noopStats) ProcessFile(filePath, contentType string, size uint64) { +} + +const ( + imageWidth = "imageWidth" + imageHeight = "imageHeight" + fileSize = "fileSize" + duration = "duration" + videoWidth = "videoWidth" + videoHeight = "videoHeight" + videoBitrate = "videoBitrate" + audioBitrate = "audioBitrate" +) + +func NewStatistics(rootStorageDir string, debug bool) Statistics { + if !debug { + return &noopStats{} + } + s := &statistics{ + metaExtractor: metadata.NewExtractor(), + metrics: metrics.NewRegistry(), + rootStorageDir: rootStorageDir, + } + if err := s.metrics.Register(imageWidth, metrics.NewHistogram(metrics.NewExpDecaySample(10000, 0.15))); err != nil { + log.Panic(errors.Wrapf(err, "failed to register metric %v", imageWidth)) + } + if err := s.metrics.Register(imageHeight, metrics.NewHistogram(metrics.NewExpDecaySample(10000, 0.15))); err != nil { + log.Panic(errors.Wrapf(err, "failed to register metric %v", imageHeight)) + } + if err := s.metrics.Register(videoWidth, metrics.NewHistogram(metrics.NewExpDecaySample(10000, 0.15))); err != nil { + log.Panic(errors.Wrapf(err, "failed to register metric %v", videoWidth)) + } + if err := s.metrics.Register(videoHeight, metrics.NewHistogram(metrics.NewExpDecaySample(10000, 0.15))); err != nil { + log.Panic(errors.Wrapf(err, "failed to register metric %v", videoHeight)) + } + if err := s.metrics.Register(duration, metrics.NewHistogram(metrics.NewExpDecaySample(10000, 0.15))); err != nil { + log.Panic(errors.Wrapf(err, "failed to register metric %v", duration)) + } + if err := s.metrics.Register(videoBitrate, metrics.NewHistogram(metrics.NewExpDecaySample(10000, 0.15))); err != nil { + log.Panic(errors.Wrapf(err, "failed to register metric %v", videoBitrate)) + } + if err := s.metrics.Register(audioBitrate, metrics.NewHistogram(metrics.NewExpDecaySample(10000, 0.15))); err != nil { + log.Panic(errors.Wrapf(err, "failed to register metric %v", audioBitrate)) + } + if err := s.metrics.Register(fileSize, metrics.NewHistogram(metrics.NewExpDecaySample(10000, 0.15))); err != nil { + log.Panic(errors.Wrapf(err, "failed to register metric %v", fileSize)) + } + go func() { + for _ = range time.Tick(60 * time.Second) { + s.writeJSON() + } + }() + return s +} + +func (s *statistics) writeJSON() { + statsFile, err := os.OpenFile(filepath.Join(s.rootStorageDir, "stats.json"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + log.Printf("ERROR: %v", errors.Wrapf(err, "failed to open file for stats collection")) + } + defer func() { + statsFile.Sync() + statsFile.Close() + }() + metrics.WriteJSONOnce(s.metrics, statsFile) + +} + +func (s *statistics) Close() error { + s.writeJSON() + if err := s.metaExtractor.Close(); err != nil { + return errors.Wrapf(err, "failed to close metadata extractors") + } + return nil + +} +func (s *statistics) ProcessFile(filePath, contentType string, size uint64) { + go func() { + md, err := s.metaExtractor.Extract(filePath, contentType, size) + if err != nil { + log.Printf("Error extracting metadata for file stats: %v\n", err) + } + s.registerMetadataStats(md) + }() +} + +func (s *statistics) registerMetadataStats(md *metadata.Metadata) { + s.registerFileStats(md) + switch typedMD := md.TypeMeta.(type) { + case *metadata.ImageMetadata: + s.registerImageStats(md, typedMD) + case *metadata.VideoMetadata: + s.registerVideoStats(md, typedMD) + } + +} +func (s *statistics) registerImageStats(md *metadata.Metadata, imageMetadata *metadata.ImageMetadata) { + s.metrics.Get(imageWidth).(metrics.Histogram).Update(int64(imageMetadata.Width)) + s.metrics.Get(imageHeight).(metrics.Histogram).Update(int64(imageMetadata.Height)) +} +func (s *statistics) registerVideoStats(md *metadata.Metadata, videoMetadata *metadata.VideoMetadata) { + for _, stream := range videoMetadata.Streams { + switch stream.CodecType { + case "video": + s.metrics.Get(videoWidth).(metrics.Histogram).Update(int64(stream.Width)) + s.metrics.Get(videoHeight).(metrics.Histogram).Update(int64(stream.Height)) + dur, err := strconv.ParseFloat(stream.Duration, 64) + if err == nil { + s.metrics.Get(duration).(metrics.Histogram).Update(int64(dur)) + } + bitrate, err := strconv.ParseInt(stream.BitRate, 10, 64) + if err == nil { + s.metrics.Get(videoBitrate).(metrics.Histogram).Update(bitrate) + } + s.metrics.GetOrRegister(fmt.Sprintf("videoCodec/%v/%v", stream.CodecName, stream.CodecTagString), metrics.NewCounter()).(metrics.Counter).Inc(1) + case "audio": + bitrate, err := strconv.ParseInt(stream.BitRate, 10, 64) + if err == nil { + s.metrics.Get(audioBitrate).(metrics.Histogram).Update(bitrate) + } + s.metrics.GetOrRegister(fmt.Sprintf("audioCodec/%v/%v", stream.CodecName, stream.CodecTagString), metrics.NewCounter()).(metrics.Counter).Inc(1) + } + } +} +func (s *statistics) registerFileStats(md *metadata.Metadata) { + s.metrics.Get(fileSize).(metrics.Histogram).Update(int64(md.Size)) + s.metrics.GetOrRegister("ext/"+md.Ext, metrics.NewCounter()).(metrics.Counter).Inc(1) +} diff --git a/storage/storage.go b/storage/storage.go new file mode 100644 index 0000000..b1356ad --- /dev/null +++ b/storage/storage.go @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: ice License 1.0 + +package storage + +import ( + "context" + "encoding/hex" + "encoding/json" + "io" + "log" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/cockroachdb/errors" + gomime "github.com/cubewise-code/go-mime" + "github.com/hashicorp/go-multierror" + "github.com/nbd-wtf/go-nostr/nip94" + "github.com/syndtr/goleveldb/leveldb" + "github.com/xssnick/tonutils-go/adnl" + "github.com/xssnick/tonutils-go/adnl/dht" + "github.com/xssnick/tonutils-go/adnl/overlay" + "github.com/xssnick/tonutils-storage/db" + "github.com/xssnick/tonutils-storage/storage" + + "github.com/ice-blockchain/subzero/storage/statistics" +) + +type ( + StorageClient interface { + io.Closer + StartUpload(ctx context.Context, userPubKey, masterKey, relativePathToFileForUrl, fileHash string, newFile *FileMetaInput) (bagID, url string, existed bool, err error) + BuildUserPath(masterKey, contentType string) (string, string) + DownloadUrl(masterKey, fileSha256 string) (string, error) + ListFiles(masterKey string, page, count uint32) (totalFiles uint32, files []*FileMetadata, err error) + Delete(userPubkey, masterKey string, fileSha256 string) error + } + Bootstrap struct { + Overlay *overlay.Node + DHT *dht.Node + } + headerData struct { + FileMetadata map[string]*FileMetaInput `json:"f"` + FileHash map[string]string `json:"fh"` + Master string `json:"m"` + } + FileMetaInput struct { + Caption string `json:"c"` + Alt string `json:"a"` + Owner string `json:"o"` + Hash []byte `json:"h"` + CreatedAt uint64 `json:"cAt"` + } + FileMetadata struct { + *nip94.FileMetadata + CreatedAt uint64 `json:"created_at"` + } + client struct { + stats statistics.Statistics + progressStorage *db.Storage + server *storage.Server + conn *storage.Connector + gateway *adnl.Gateway + dht *dht.Client + newFiles map[string]map[string]*FileMetaInput + newFilesMx *sync.RWMutex + db *leveldb.DB + downloadQueue chan queueItem + activeDownloads map[string]bool + activeDownloadsMx *sync.RWMutex + rootStoragePath string + debug bool + } + queueItem struct { + tor *storage.Torrent + bootstrap *string + user *string + } +) + +var ( + ErrNotFound = storage.ErrFileNotExist + ErrForbidden = errors.New("forbidden") +) + +func (c *client) fileMeta(bag *storage.Torrent) (*headerData, error) { + var desc headerData + if bag.Header == nil { + return nil, errors.Errorf("No header fetched yet for %v", hex.EncodeToString(bag.BagID)) + } + hData := bag.Header.Data + if len(hData) == 0 { + hData = []byte("{}") + } + if err := json.Unmarshal(hData, &desc); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal bag header data") + } + return &desc, nil +} + +func (c *client) detectFile(bag *storage.Torrent, fileHash string) (string, error) { + metadata, err := c.fileMeta(bag) + if err != nil { + return "", errors.Wrapf(err, "failed to parse bag header data %v", hex.EncodeToString(bag.BagID)) + } + return c.detectFileFromMeta(bag, metadata, fileHash) +} + +func (c *client) detectFileFromMeta(bag *storage.Torrent, metadata *headerData, fileHash string) (string, error) { + name := metadata.FileHash[fileHash] + f, err := bag.GetFileOffsets(name) + if err != nil { + return "", errors.Wrapf(err, "failed to locate file %v in bag %v", name, hex.EncodeToString(bag.BagID)) + } + return f.Name, nil +} + +func (c *client) bagByUser(userPubKey string) (*storage.Torrent, error) { + k := make([]byte, 3+64) + copy(k, "ub:") + copy(k[3:], userPubKey) + bagID, err := c.db.Get(k, nil) + if err != nil && !errors.Is(err, leveldb.ErrNotFound) { + return nil, errors.Wrap(err, "failed to read userID:bag mapping") + } + tr := c.progressStorage.GetTorrent(bagID) + + return tr, nil +} +func (c *client) bootstrapForBag(bagID []byte) (string, error) { + k := make([]byte, 3+32) + copy(k, "bs:") + copy(k[3:], bagID) + bs, err := c.db.Get(k, nil) + if err != nil && !errors.Is(err, leveldb.ErrNotFound) { + return "", errors.Wrapf(err, "failed to read stored bootstrap node for %v =, will wait for DHT discovery", hex.EncodeToString(bagID)) + } + return string(bs), nil +} + +func (c *client) BuildUserPath(userPubKey string, contentType string) (userStorage string, uploadPath string) { + spl := strings.Split(contentType, "/") + return filepath.Join(c.rootStoragePath, userPubKey), spl[0] +} + +func (c *client) ListFiles(userPubKey string, page, limit uint32) (total uint32, res []*FileMetadata, err error) { + bag, err := c.bagByUser(userPubKey) + if err != nil { + return 0, nil, errors.Wrapf(err, "failed to get bagID for the user %v", userPubKey) + } + metadata, err := c.fileMeta(bag) + if err != nil { + return 0, nil, errors.Wrapf(err, "failed to parse bag header data %v", hex.EncodeToString(bag.BagID)) + } + startOffset := page * limit + if startOffset >= bag.Header.FilesCount { + return bag.Header.FilesCount, []*FileMetadata{}, nil + } + endOffset := page*limit + limit + if endOffset >= bag.Header.FilesCount { + endOffset = bag.Header.FilesCount + } + res = make([]*FileMetadata, 0, limit) + bs, err := c.buildBootstrapNodeInfo(bag) + if err != nil { + return 0, nil, errors.Wrapf(err, "failed to build bootstap for bag %v", hex.EncodeToString(bag.BagID)) + } + files, err := bag.ListFiles() + if err != nil { + return 0, nil, errors.Wrapf(err, "failed to parse bag info for files %v", hex.EncodeToString(bag.BagID)) + } + for i, f := range files[startOffset:endOffset] { + idx := page*limit + uint32(i) + fileInfo, _ := bag.GetFileOffsets(f) + md, hasMD := metadata.FileMetadata[fileInfo.Name] + if !hasMD { + continue + } + url, _ := c.buildUrl(hex.EncodeToString(bag.BagID), f, []*Bootstrap{bs}) + res = append(res, &FileMetadata{ + FileMetadata: &nip94.FileMetadata{ + Size: strconv.FormatUint(uint64(fileInfo.Size), 10), + Summary: md.Alt, + URL: url, + M: gomime.TypeByExtension(filepath.Ext(files[idx])), + X: hex.EncodeToString(md.Hash), + OX: hex.EncodeToString(md.Hash), + TorrentInfoHash: hex.EncodeToString(bag.BagID), + Content: md.Caption, + }, + CreatedAt: uint64(time.Unix(0, int64(md.CreatedAt)).Unix()), + }) + } + return bag.Header.FilesCount, res, nil +} + +func (c *client) Delete(userPubKey, masterKey, fileHash string) error { + bag, err := c.bagByUser(masterKey) + if err != nil { + return errors.Wrapf(err, "failed to get bagID for the user %v", userPubKey) + } + if bag == nil { + return ErrNotFound + } + var metadata *headerData + metadata, err = c.fileMeta(bag) + if err != nil { + return errors.Wrapf(err, "failed to parse bag header data %v", hex.EncodeToString(bag.BagID)) + } + file, err := c.detectFileFromMeta(bag, metadata, fileHash) + if err != nil { + return errors.Wrapf(err, "failed to detect file %v in bag %v", fileHash, hex.EncodeToString(bag.BagID)) + } + if userPubKey != masterKey { + if metadata.FileMetadata[file].Owner == masterKey { + return ErrForbidden + } + } + userPath, _ := c.BuildUserPath(masterKey, "") + err = os.Remove(filepath.Join(userPath, file)) + if err != nil { + return errors.Wrapf(err, "failed to remove file %v (%v)", fileHash, filepath.Join(userPath, file)) + } + return nil +} + +func (c *client) Close() error { + var err *multierror.Error + c.server.Stop() + c.dht.Close() + if gClose := c.gateway.Close(); gClose != nil { + err = multierror.Append(err, errors.Wrapf(gClose, "failed to stop gateway")) + } + if sClose := c.stats.Close(); sClose != nil { + err = multierror.Append(err, errors.Wrapf(sClose, "failed to close stats file")) + } + if dErr := c.db.Close(); dErr != nil { + err = multierror.Append(err, errors.Wrapf(dErr, "failed to close db")) + } + close(c.downloadQueue) + return err.ErrorOrNil() +} + +func (c *client) report(ctx context.Context) { + period := 1 * time.Hour + if c.debug { + period = 1 * time.Minute + } + for ctx.Err() == nil { + select { + case <-ctx.Done(): + return + case <-time.After(period): + activelyDownloading := 0 + activeUploading := 0 + notResolvedHeader := 0 + notResolvedInfo := 0 + all := c.progressStorage.GetAll() + for _, t := range all { + if t.IsDownloadAll() { + activelyDownloading++ + } + if _, upl := t.IsActive(); upl { + activeUploading++ + } + if t.Info == nil { + notResolvedInfo++ + } + if t.Header == nil { + notResolvedHeader++ + } + } + log.Printf("[STORAGE STATS] DEBUG: Q TO DOWNLOAD %v, DOWNLOADING %v, UPLOADING %v, RESOLVING INFO %v, RESOLVING HEADER %v TOTAL %v", len(c.downloadQueue), activelyDownloading, activeUploading, notResolvedInfo, notResolvedHeader, len(all)) + } + } +} diff --git a/storage/upload.go b/storage/upload.go new file mode 100644 index 0000000..bf29782 --- /dev/null +++ b/storage/upload.go @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: ice License 1.0 + +package storage + +import ( + "context" + "crypto/ed25519" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "log" + "path/filepath" + "sync" + "time" + + "github.com/cockroachdb/errors" + gomime "github.com/cubewise-code/go-mime" + "github.com/xssnick/tonutils-go/adnl" + "github.com/xssnick/tonutils-go/adnl/dht" + "github.com/xssnick/tonutils-go/adnl/overlay" + "github.com/xssnick/tonutils-go/tl" + "github.com/xssnick/tonutils-storage/storage" +) + +func (c *client) StartUpload(ctx context.Context, userPubKey, masterPubKey, relativePathToFileForUrl, hash string, newFile *FileMetaInput) (bagID, url string, existed bool, err error) { + existingBagForUser, err := c.bagByUser(masterPubKey) + if err != nil { + return "", "", false, errors.Wrapf(err, "failed to find existing bag for user %s", masterPubKey) + } + var existingHD headerData + if existingBagForUser != nil { + if len(existingBagForUser.Header.Data) > 0 { + if err = json.Unmarshal(existingBagForUser.Header.Data, &existingHD); err != nil { + return "", "", false, errors.Wrapf(err, "corrupted header metadata for bag %v", hex.EncodeToString(existingBagForUser.BagID)) + } + } + } + _, existed = existingHD.FileHash[hash] + if existed { + if existingBagForUser != nil { + url, err = c.DownloadUrl(masterPubKey, hash) + if err != nil { + if errors.Is(err, storage.ErrFileNotExist) { + existed = false + } + return "", "", false, + errors.Wrapf(err, "failed to build download url for already existing file %v/%v(%v)", masterPubKey, relativePathToFileForUrl, hash) + } + if existed { + return hex.EncodeToString(existingBagForUser.BagID), url, existed, nil + } + + } + } + var bs []*Bootstrap + var bag *storage.Torrent + bag, bs, err = c.upload(ctx, userPubKey, masterPubKey, relativePathToFileForUrl, hash, newFile, &existingHD) + if err != nil { + return "", "", false, errors.Wrapf(err, "failed to start upload of %v", relativePathToFileForUrl) + } + bagID = hex.EncodeToString(bag.BagID) + log.Printf("[STORAGE] INFO: new upload %v for user %v hash %v resulted in bag %v, total %v", relativePathToFileForUrl, masterPubKey, hash, bagID, bag.Header.FilesCount) + if newFile != nil && c.debug { + uplFile, err := bag.GetFileOffsets(relativePathToFileForUrl) + if err != nil { + return "", "", false, errors.Wrapf(err, "failed to get just created file from new bag") + } + fullFilePath := filepath.Join(c.rootStoragePath, masterPubKey, relativePathToFileForUrl) + go c.stats.ProcessFile(fullFilePath, gomime.TypeByExtension(filepath.Ext(fullFilePath)), uplFile.Size) + } + url, err = c.buildUrl(bagID, relativePathToFileForUrl, bs) + if err != nil { + return "", "", false, errors.Wrapf(err, "failed to build url for %v (bag %v)", relativePathToFileForUrl, bagID) + } + return bagID, url, existed, err +} + +func (c *client) upload(ctx context.Context, user, master, relativePath, hash string, fileMeta *FileMetaInput, headerMetadata *headerData) (torrent *storage.Torrent, bootstrap []*Bootstrap, err error) { + if fileMeta != nil { + c.newFilesMx.Lock() + if userNewFiles, hasNewFiles := c.newFiles[master]; !hasNewFiles || userNewFiles == nil { + c.newFiles[master] = make(map[string]*FileMetaInput) + } + c.newFiles[master][relativePath] = fileMeta + c.newFilesMx.Unlock() + } + rootUserPath, _ := c.BuildUserPath(master, "") + refs, err := c.progressStorage.GetAllFilesRefsInDir(rootUserPath) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to detect shareable files") + } + headerMD := &headerData{ + Master: master, + FileMetadata: headerMetadata.FileMetadata, + FileHash: headerMetadata.FileHash, + } + if headerMD.FileHash == nil { + headerMD.FileHash = make(map[string]string) + } + if headerMD.FileMetadata == nil { + headerMD.FileMetadata = make(map[string]*FileMetaInput) + } + if fileMeta != nil { + fileMeta.Owner = user + headerMD.FileMetadata[relativePath] = fileMeta + headerMD.FileHash[hex.EncodeToString(fileMeta.Hash)] = relativePath + } else { + delete(headerMD.FileMetadata, relativePath) + delete(headerMD.FileHash, hash) + } + c.newFilesMx.RLock() + for key, value := range c.newFiles[master] { + headerMD.FileMetadata[key] = value + headerMD.FileHash[hex.EncodeToString(value.Hash)] = key + } + c.newFilesMx.RUnlock() + var headerMDSerialized []byte + headerMDSerialized, err = json.Marshal(headerMD) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to put file hashes") + } + header := &storage.TorrentHeader{ + DirNameSize: uint32(len(master)), + DirName: []byte(master), + Data: headerMDSerialized, + TotalDataSize: uint64(len(headerMDSerialized)), + } + var wg sync.WaitGroup + wg.Add(1) + tr, err := storage.CreateTorrentWithInitialHeader(ctx, c.rootStoragePath, master, header, c.progressStorage, c.conn, refs, func(done uint64, max uint64) { + if done == max { + wg.Done() + } + }, false) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to initialize bag") + } + err = tr.Start(true, false, false) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to start bag upload") + } + wg.Wait() + err = c.saveUploadTorrent(tr, master) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to save updated bag") + } + bootstrapNode, err := c.buildBootstrapNodeInfo(tr) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to build bootstrap node info") + } + return tr, []*Bootstrap{bootstrapNode}, nil +} + +func (c *client) buildBootstrapNodeInfo(tr *storage.Torrent) (*Bootstrap, error) { + key := c.server.GetADNLPrivateKey() + overlayNode, err := overlay.NewNode(tr.BagID, key) + if err != nil { + return nil, errors.Wrap(err, "failed to build overlay node") + } + addr := c.gateway.GetAddressList() + + dNode := dht.Node{ + ID: adnl.PublicKeyED25519{Key: key.Public().(ed25519.PublicKey)}, + AddrList: &addr, + Version: int32(time.Now().Unix()), + Signature: nil, + } + + toVerify, err := tl.Serialize(dNode, true) + if err != nil { + return nil, errors.Wrapf(err, "failed to sign dht bootstrap, serialize failure") + } + dNode.Signature = ed25519.Sign(key, toVerify) + + return &Bootstrap{ + Overlay: overlayNode, + DHT: &dNode, + }, nil +} + +func (c *client) buildUrl(bagID, relativePath string, bs []*Bootstrap) (string, error) { + b, err := json.Marshal(bs) + if err != nil { + return "", errors.Wrapf(err, "failed to marshal %#v", bs) + } + bootstrap := base64.StdEncoding.EncodeToString(b) + url := fmt.Sprintf("http://%v.bag/%v?bootstrap=%v", bagID, relativePath, bootstrap) + + return url, nil +} + +func (c *client) saveUploadTorrent(tr *storage.Torrent, userPubKey string) error { + if err := c.saveTorrent(tr, &userPubKey, nil); err != nil { + return errors.Wrap(err, "failed to save upload torrent into storage") + } + c.newFilesMx.Lock() + for k := range c.newFiles[userPubKey] { + if _, err := tr.GetFileOffsets(k); err == nil { + delete(c.newFiles[userPubKey], k) + } + } + c.newFilesMx.Unlock() + return nil +}