diff --git a/database/query/query_fuzz_test.go b/database/query/query_fuzz_test.go index 2af94c8..01d6fde 100644 --- a/database/query/query_fuzz_test.go +++ b/database/query/query_fuzz_test.go @@ -22,6 +22,8 @@ type structElement struct { Name []string // Full index of the field. Addr []int + // Number of elements to generate for this slice, if applicable. + NumElem int } func (f *structElement) Clone() *structElement { @@ -30,8 +32,9 @@ func (f *structElement) Clone() *structElement { } return &structElement{ - Name: append([]string{}, f.Name...), - Addr: append([]int{}, f.Addr...), + Name: append([]string{}, f.Name...), + Addr: append([]int{}, f.Addr...), + NumElem: f.NumElem, } } @@ -60,7 +63,13 @@ func helperParseFilterStruct(t *testing.T, typ reflect.Type, parent *structEleme el := parent.Clone() el.Name = append(el.Name, field.Name) el.Addr = append(el.Addr, field.Index...) + el.NumElem = 1 fields = append(fields, el) + if field.Type.Kind() == reflect.Slice { + next := el.Clone() + next.NumElem = int(rand.Int31n(6)) + 1 + fields = append(fields, next) + } case reflect.String: for _, v := range []string{"Images", "Quotes", "References", "Videos", "Expiration"} { @@ -93,23 +102,25 @@ func helperNewFilterFromElements(t *testing.T, fields []*structElement) model.Fi value := reflect.ValueOf(&f).Elem().FieldByIndex(field.GetAddress()) switch field.GetName() { case "Authors", "IDs": - n := rand.Int31n(4) - vals := make([]string, n) - for i := range n { + vals := make([]string, field.NumElem) + for i := range field.NumElem { vals[i] = generateHexString() } value.Set(reflect.ValueOf(vals)) case "Kinds": - k := []int{generateKind()} - value.Set(reflect.ValueOf(k)) + vals := make([]int, field.NumElem) + for i := range field.NumElem { + vals[i] = generateKind() + } + value.Set(reflect.ValueOf(vals)) case "Tags": - values := []string{} - for range rand.Intn(3) { - values = append(values, generateHexString()) + vals := make([]string, field.NumElem) + for i := range field.NumElem { + vals[i] = generateHexString() } - m := model.TagMap{}.SetLiterals("e", values...) + m := model.TagMap{}.SetLiterals("e", vals...) value.Set(reflect.ValueOf(m)) @@ -203,15 +214,22 @@ func TestQueryFuzzNoUseTempBTREEOrScan(t *testing.T) { rows, err := stmt.QueryContext(context.Background(), params) require.NoError(t, err) + var hasPK bool for rows.Next() { var s1, s2, s3, s4 string err := rows.Scan(&s1, &s2, &s3, &s4) require.NoError(t, err) op[s4]++ + if strings.Contains(s4, "SEARCH e USING PRIMARY KEY") { + hasPK = true + } if s4 == "USE TEMP B-TREE FOR ORDER BY" || (strings.HasPrefix(s4, "SCAN ") && !strings.Contains(s4, "INDEX")) { if strings.Contains(filter.Search, "Expiration:true") { // It uses SCAN over CTE, which is expected. continue + } else if (hasPK || len(filter.Authors) > 0) && s4 == "USE TEMP B-TREE FOR ORDER BY" { + // Allow B-TREE for ORDER BY if there are multiple authors or PK is used. + continue } t.Logf("filter: %#v", filter) t.Logf("set #%d: %s (%+v)", i+1, sql, params) diff --git a/database/query/query_where_builder.go b/database/query/query_where_builder.go index a3325b0..3dcb833 100644 --- a/database/query/query_where_builder.go +++ b/database/query/query_where_builder.go @@ -5,10 +5,8 @@ package query import ( "cmp" "log" - "slices" "strconv" "strings" - "sync" "github.com/cockroachdb/errors" "github.com/nbd-wtf/go-nostr" @@ -70,35 +68,8 @@ type ( Tag string Marker string } - filterBuilder struct { - Name string - EventIds []string - EventIdsString string - sync.Once - } ) -func (f *filterBuilder) HasEvents() bool { - return len(f.EventIds) > 0 -} - -func (f *filterBuilder) BuildEvents(w *whereBuilder) string { - f.Do(func() { - f.EventIdsString = buildFromSlice( - &whereBuilder{ - Params: w.Params, - }, - sqlOpCodeAND, - f.Name, - f.EventIds, - "event_id", - "", - ).String() - }) - - return f.EventIdsString -} - func parseEventAsFilterForDelete(e *model.Event) (*databaseFilterDelete, error) { filter := databaseFilterDelete{ Author: e.PubKey, @@ -174,12 +145,9 @@ func buildFromSlice[T comparable](builder *whereBuilder, op int, filterID string } maybeOpCode(builder, op) - if len(s) > 1 && (name == "id" || name == "pubkey") { - builder.WriteRune('+') - } builder.WriteString(name) s = model.DeduplicateSlice(s, func(elem T) T { return elem }) - if len(s) == 1 && name != "kind" { + if len(s) == 1 { // X = :X_name. builder.WriteString(" = :") builder.WriteString(builder.addParam(filterID, paramName, s[0])) @@ -227,7 +195,7 @@ func (w *whereBuilder) maybeOR() { w.WriteString(" OR ") } -func (w *whereBuilder) applyFilterTagMarkers(filter *filterBuilder, markers []databaseFilterMarker) { +func (w *whereBuilder) applyFilterTagMarkers(name string, markers []databaseFilterMarker) { if len(markers) == 0 { return } @@ -235,14 +203,14 @@ func (w *whereBuilder) applyFilterTagMarkers(filter *filterBuilder, markers []da for id, marker := range markers { w.maybeAND() w.WriteString("EXISTS (select true from event_tags where event_id = e.id AND event_tag_key = :") - w.WriteString(w.addParam(filter.Name, "mtag"+strconv.Itoa(id), marker.Tag)) + w.WriteString(w.addParam(name, "mtag"+strconv.Itoa(id), marker.Tag)) w.WriteString(" AND event_tag_value3 = :") - w.WriteString(w.addParam(filter.Name, "mtagvalue"+strconv.Itoa(id), marker.Marker)) + w.WriteString(w.addParam(name, "mtagvalue"+strconv.Itoa(id), marker.Marker)) w.WriteRune(')') } } -func (w *whereBuilder) applyFilterTags(filter *filterBuilder, tags model.TagMap) { +func (w *whereBuilder) applyFilterTags(name string, tags model.TagMap) { const valuesMax = 21 if len(tags) == 0 { @@ -254,7 +222,7 @@ func (w *whereBuilder) applyFilterTags(filter *filterBuilder, tags model.TagMap) tagID++ w.maybeAND() - tagParam := w.addParam(filter.Name, "tag"+strconv.Itoa(tagID), tagName) + tagParam := w.addParam(name, "tag"+strconv.Itoa(tagID), tagName) // Only the tag name is specified, no values. if !tags.HasValues(tagName) { @@ -288,7 +256,7 @@ func (w *whereBuilder) applyFilterTags(filter *filterBuilder, tags model.TagMap) w.WriteString("event_tag_value") w.WriteString(strconv.Itoa(j + 1)) w.WriteString(" = :") - w.WriteString(w.addParam(filter.Name, "tagvalue"+strconv.Itoa(tagID<<8|(j+1)*(i+1)), *values[j])) + w.WriteString(w.addParam(name, "tagvalue"+strconv.Itoa(tagID<<8|(j+1)*(i+1)), *values[j])) } w.WriteRune(')') } @@ -311,12 +279,12 @@ func isFilterEmpty(filter *databaseFilterSearch) bool { filter.Images == nil } -func (w *whereBuilder) applyTimeRange(filter *filterBuilder, since, until *model.Timestamp) error { +func (w *whereBuilder) applyTimeRange(name string, since, until *model.Timestamp) error { if since != nil && until != nil { if *since == *until { w.maybeAND() w.WriteString("created_at = :") - w.WriteString(w.addParam(filter.Name, "timestamp", *since)) + w.WriteString(w.addParam(name, "timestamp", *since)) return nil } else if *since > *until { @@ -328,14 +296,14 @@ func (w *whereBuilder) applyTimeRange(filter *filterBuilder, since, until *model if since != nil && *since > 0 { w.maybeAND() w.WriteString("created_at >= :") - w.WriteString(w.addParam(filter.Name, "since", *since)) + w.WriteString(w.addParam(name, "since", *since)) } // The `until` property is similar except that `created_at` must be less than or equal to `until`. if until != nil && *until > 0 { w.maybeAND() w.WriteString("created_at <= :") - w.WriteString(w.addParam(filter.Name, "until", *until)) + w.WriteString(w.addParam(name, "until", *until)) } return nil @@ -368,22 +336,24 @@ func filterHasExtensions(filter *databaseFilterSearch) (positive, negative int) return } -func (w *whereBuilder) applyFilterForExtensions(filter *databaseFilterSearch, builder *filterBuilder, include bool) { +func (w *whereBuilder) applyFilterForExtensions(filter *databaseFilterSearch, include bool) { separator := w.maybeOR - w.WriteString("select event_id from event_tags where ") - if include && builder.HasEvents() { - w.WriteString(builder.BuildEvents(w)) - w.maybeAND() + if !include { + w.WriteString("NOT ") } + w.WriteString("exists (select true from event_tags where event_id in (e.id, e.reference_id) AND (") - w.WriteRune('(') if filter.Quotes != nil && *filter.Quotes == include { separator() w.WriteString("(event_tag_key = 'q')") } if filter.References != nil && *filter.References == include { separator() - w.WriteString("(event_tag_key = 'e')") + result := "true" + if !include { + result = "false" + } + w.WriteString("(case when e.reference_id is not null then " + result + " else event_tag_key = 'e' end)") } if filter.Images != nil && *filter.Images == include { separator() @@ -409,42 +379,27 @@ func (w *whereBuilder) applyFilterForExtensions(filter *databaseFilterSearch, bu w.WriteString(" as integer) > unixepoch())") } } - w.WriteRune(')') + w.WriteString("))") } -func (w *whereBuilder) applyRepostFilter(filter *databaseFilterSearch, builder *filterBuilder, positiveExtensions, negativeExtensions *int) (applied bool) { - if (*positiveExtensions + *negativeExtensions) == 0 { - // No extensions in the filter. - return +func filterMainIndexField(filter *databaseFilterSearch) string { + if len(filter.Authors) > 0 { + return "master_pubkey" } - if !slices.ContainsFunc(filter.Kinds, func(k int) bool { - return k == nostr.KindRepost || k == nostr.KindGenericRepost - }) { - // No reposts in the filter. - return + if len(filter.Kinds) > 0 { + return "kind" } - // Not allowed. - filter.References = nil - *positiveExtensions &= ^extensionReferences - *negativeExtensions &= ^extensionReferences - - if *positiveExtensions > 0 { - w.maybeAND() - w.WriteString("(+id IN (select e.id from events subev where subev.id = e.reference_id and subev.kind = 1 and exists (") - w.applyFilterForExtensions(filter, builder, true) - w.WriteString(")))") - } + return "" +} - if *negativeExtensions > 0 { - w.maybeAND() - w.WriteString("(+id NOT IN (select e.id from events subev where subev.id = e.reference_id and subev.kind = 1 and exists (") - w.applyFilterForExtensions(filter, builder, false) - w.WriteString(")))") +func filterMaybeForceIndex(filter *databaseFilterSearch, field string) string { + main := filterMainIndexField(filter) + if main == field { + field = "+" + field } - - return (*positiveExtensions + *negativeExtensions) > 0 + return field } func (w *whereBuilder) applyFilter(idx int, filter *databaseFilterSearch) error { @@ -452,42 +407,32 @@ func (w *whereBuilder) applyFilter(idx int, filter *databaseFilterSearch) error return nil } - builder := &filterBuilder{ - Name: "filter" + strconv.Itoa(idx) + "_", - EventIds: filter.IDs, - } + name := "filter" + strconv.Itoa(idx) + "_" positiveExtensions, negativeExtensions := filterHasExtensions(filter) w.WriteRune('(') // Begin the filter section. - if w.applyRepostFilter(filter, builder, &positiveExtensions, &negativeExtensions) { - buildFromSlice(w, sqlOpCodeAND, builder.Name, filter.IDs, "id", "") - } else { - if positiveExtensions > 0 { - w.WriteString("+id IN (") - w.applyFilterForExtensions(filter, builder, true) - w.WriteRune(')') - } else { - buildFromSlice(w, sqlOpCodeAND, builder.Name, filter.IDs, "id", "") - } - if negativeExtensions > 0 { - w.maybeAND() - w.WriteString("(+id NOT IN (") - w.applyFilterForExtensions(filter, builder, false) - w.WriteString("))") - } + buildFromSlice(w, sqlOpCodeNONE, name, filter.IDs, "id", "") + buildFromSlice(w, sqlOpCodeAND, name, filter.Kinds, filterMaybeForceIndex(filter, "kind"), "kind") + if positiveExtensions > 0 { + w.maybeAND() + w.applyFilterForExtensions(filter, true) + } + if negativeExtensions > 0 { + w.maybeAND() + w.applyFilterForExtensions(filter, false) } - buildFromSlice(w, sqlOpCodeAND, builder.Name, filter.Kinds, "kind", "") if len(filter.Authors) > 0 { w.maybeAND() w.WriteRune('(') - buildFromSlice(w, sqlOpCodeNONE, builder.Name, filter.Authors, "pubkey", "") - buildFromSlice(w, sqlOpCodeOR, builder.Name, filter.Authors, "master_pubkey", "pubkey") - w.WriteRune(')') + buildFromSlice(w, sqlOpCodeNONE, name, filter.Authors, "pubkey", "") + w.WriteString(" and hidden=0 OR ") + buildFromSlice(w, sqlOpCodeNONE, name, filter.Authors, "master_pubkey", "pubkey") + w.WriteString(" and hidden=0)") } - if err := w.applyTimeRange(builder, filter.Since, filter.Until); err != nil { + if err := w.applyTimeRange(name, filter.Since, filter.Until); err != nil { return err } - w.applyFilterTags(builder, filter.Tags) - w.applyFilterTagMarkers(builder, filter.TagMarkers) + w.applyFilterTags(name, filter.Tags) + w.applyFilterTagMarkers(name, filter.TagMarkers) w.WriteRune(')') // End the filter section. diff --git a/database/query/query_where_builder_db_test.go b/database/query/query_where_builder_db_test.go index 668cc5a..8c18923 100644 --- a/database/query/query_where_builder_db_test.go +++ b/database/query/query_where_builder_db_test.go @@ -740,7 +740,7 @@ func TestSelectWithExtensions(t *testing.T) { event.Kind = nostr.KindTextNote event.ID = "expired" event.PubKey = "1" - event.Tags = model.Tags{{"expiration", strconv.FormatInt(time.Now().Unix()-0xff, 10)}, {"q", "fooo"}} + event.Tags = model.Tags{{"expiration", "1"}, {"q", "fooo"}} event.CreatedAt = 1 err := db.AcceptEvents(context.TODO(), &event) @@ -749,7 +749,7 @@ func TestSelectWithExtensions(t *testing.T) { event.Kind = nostr.KindTextNote event.ID = "alive" event.PubKey = "2" - event.Tags = model.Tags{{"expiration", strconv.FormatInt(time.Now().Unix()+0xff, 10)}, {"e", "bar"}} + event.Tags = model.Tags{{"expiration", "2177366400"}, {"e", "bar"}} event.CreatedAt = 1 err = db.AcceptEvents(context.TODO(), &event) @@ -801,15 +801,15 @@ func TestSelectRepostWithReference(t *testing.T) { event.ID = "1" event.PubKey = "1" event.Tags = model.Tags{{"e", "fooo"}, {"bar", "foo"}} + event.Content = `{"id":"3","pubkey":"4","created_at":1712594952,"kind":1,"tags":[["imeta","url https://example.com/foo.jpg","ox f63ccef25fcd9b9a181ad465ae40d282eeadd8a4f5c752434423cb0539f73e69 https://nostr.build","x f9c8b660532a6e8236779283950d875fbfbdc6f4dbc7c675bc589a7180299c30","m image/jpeg","dim 1066x1600","bh L78C~=$%0%ERjENbWX$g0jNI}:-S","blurhash L78C~=$%0%ERjENbWX$g0jNI}:-S"]],"content":"foo","sig":"sig"}` event.CreatedAt = 1 err := db.AcceptEvents(context.TODO(), &event) require.NoError(t, err) }) - t.Run("SelectRepost", func(t *testing.T) { + t.Run("Reference extension must be ignored for reposts", func(t *testing.T) { count, err := db.CountEvents(context.TODO(), helperNewFilterSubscription(func(apply *model.Filter) { apply.Search = "references:false" - apply.Kinds = []int{nostr.KindRepost} })) require.NoError(t, err) require.Equal(t, int64(1), count) @@ -839,7 +839,6 @@ func TestSelectFilterKind6AsKind1(t *testing.T) { t.Run("SelectRepost", func(t *testing.T) { filter := helperNewFilterSubscription(func(apply *model.Filter) { apply.Search = "images:yes" - apply.Kinds = []int{nostr.KindRepost} }) t.Run("Count", func(t *testing.T) { count, err := db.CountEvents(context.TODO(), filter) diff --git a/go.mod b/go.mod index f395ac6..404a59d 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,7 @@ require ( github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect github.com/gabriel-vasile/mimetype v1.4.7 // indirect - github.com/getsentry/sentry-go v0.29.1 // indirect + github.com/getsentry/sentry-go v0.30.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect @@ -72,7 +72,7 @@ require ( github.com/goccy/go-json v0.10.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/pprof v0.0.0-20241128161848-dc51965c6481 // indirect + github.com/google/pprof v0.0.0-20241203143554-1e3fdc7de467 // indirect github.com/gookit/color v1.5.4 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect diff --git a/go.sum b/go.sum index 682fd47..ab66d22 100644 --- a/go.sum +++ b/go.sum @@ -71,8 +71,8 @@ github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/ github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA= github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU= -github.com/getsentry/sentry-go v0.29.1 h1:DyZuChN8Hz3ARxGVV8ePaNXh1dQ7d76AiB117xcREwA= -github.com/getsentry/sentry-go v0.29.1/go.mod h1:x3AtIzN01d6SiWkderzaH28Tm0lgkafpJ5Bm3li39O0= +github.com/getsentry/sentry-go v0.30.0 h1:lWUwDnY7sKHaVIoZ9wYqRHJ5iEmoc0pqcRqFkosKzBo= +github.com/getsentry/sentry-go v0.30.0/go.mod h1:WU9B9/1/sHDqeV8T+3VwwbjeR5MSXs/6aqG3mqZrezA= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= @@ -112,8 +112,8 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20241128161848-dc51965c6481 h1:yudKIrXagAOl99WQzrP1gbz5HLB9UjhcOFnPzdd6Qec= -github.com/google/pprof v0.0.0-20241128161848-dc51965c6481/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20241203143554-1e3fdc7de467 h1:keEZFtbLJugfE0qHn+Ge1JCE71spzkchQobDf3mzS/4= +github.com/google/pprof v0.0.0-20241203143554-1e3fdc7de467/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=