perf(chstorage): use keys index for logs attributes

``` │ benchstat.report.txt.old │ benchstat.report.txt │ │ sec/op │ sec/op vs base │ LogQL/Lookup_by_materialzied_attribute 19.65m ± 17% 23.97m ± 31% ~ (p=0.217 n=15) LogQL/Lookup_by_regular_attribute 7501.61m ± 1% 47.10m ± 22% -99.37% (p=0.000 n=15) geomean 384.0m 33.60m -91.25% │ benchstat.report.txt.old │ benchstat.report.txt │ │ ch-sec/op │ ch-sec/op vs base │ LogQL/Lookup_by_materialzied_attribute 12.67m ± 19% 12.43m ± 46% ~ (p=0.436 n=15) LogQL/Lookup_by_regular_attribute 7499.13m ± 2% 44.70m ± 23% -99.40% (p=0.000 n=15) geomean 308.2m 23.57m -92.35% │ benchstat.report.txt.old │ benchstat.report.txt │ │ ch-mem-bytes/op │ ch-mem-bytes/op vs base │ LogQL/Lookup_by_materialzied_attribute 28.46Mi ± 0% 29.63Mi ± 15% +4.12% (p=0.001 n=15) LogQL/Lookup_by_regular_attribute 284.53Mi ± 3% 31.90Mi ± 13% -88.79% (p=0.000 n=15) geomean 89.99Mi 30.75Mi -65.83% │ benchstat.report.txt.old │ benchstat.report.txt │ │ ch-read-bytes/op │ ch-read-bytes/op vs base │ LogQL/Lookup_by_materialzied_attribute 2.970Mi ± 0% 3.227Mi ± 0% +8.64% (p=0.000 n=15) LogQL/Lookup_by_regular_attribute 745771.8Ki ± 0% 605.6Ki ± 0% -99.92% (p=0.000 n=15) geomean 46.51Mi 1.381Mi -97.03% │ benchstat.report.txt.old │ benchstat.report.txt │ │ ch-read-rows/op │ ch-read-rows/op vs base │ LogQL/Lookup_by_materialzied_attribute 76.49k ± 0% 76.49k ± 0% ~ (p=1.000 n=15) ¹ LogQL/Lookup_by_regular_attribute 63630.02k ± 0% 10.89k ± 0% -99.98% (p=0.000 n=15) geomean 2.206M 28.86k -98.69% ¹ all samples are equal ```
go-faster · Nov 30, 2024 · acc5453 · acc5453
1 parent f341ca4
commit acc5453
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 4 deletions.
diff --git a/internal/chstorage/querier_logs_query.go b/internal/chstorage/querier_logs_query.go
@@ -530,8 +530,7 @@ func (q *Querier) logQLLabelMatcher(
 	case logstorage.LabelTraceID:
 		return matchHex(chsql.Ident("trace_id"), m)
 	default:
-		expr, ok := q.getMaterializedLabelColumn(unmappedLabel)
-		if ok {
+		if expr, ok := q.getMaterializedLabelColumn(unmappedLabel); ok {
 			switch m.Op {
 			case logql.OpEq, logql.OpNotEq:
 				return chsql.Eq(expr, chsql.String(m.Value))
@@ -543,11 +542,12 @@ func (q *Querier) logQLLabelMatcher(
 		}
 
 		exprs := make([]chsql.Expr, 0, 3)
+		keysExprs := make([]chsql.Expr, 0, cap(exprs))
 		// Search in all attributes.
 		for _, column := range []string{
 			colAttrs,
-			colResource,
 			colScope,
+			colResource,
 		} {
 			// TODO: how to match integers, booleans, floats, arrays?
 			var (
@@ -563,8 +563,16 @@ func (q *Querier) logQLLabelMatcher(
 				panic(fmt.Sprintf("unexpected label matcher op %v", m.Op))
 			}
 			exprs = append(exprs, sub)
+			keysExprs = append(keysExprs, chsql.JSONExtractKeys(chsql.Ident(column)))
 		}
-		return chsql.JoinOr(exprs...)
+		// Force Clickhouse to use index.
+		return chsql.And(
+			chsql.Has(
+				chsql.ArrayConcat(keysExprs...),
+				chsql.String(labelName),
+			),
+			chsql.JoinOr(exprs...),
+		)
 	}
 }
 

diff --git a/internal/chstorage/schema_logs.go b/internal/chstorage/schema_logs.go
@@ -33,6 +33,7 @@ const (
     INDEX idx_trace_id trace_id TYPE bloom_filter(0.001) GRANULARITY 1,
     INDEX idx_body body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1,
 	INDEX idx_ts timestamp TYPE minmax GRANULARITY 8192,
+	INDEX attribute_keys arrayConcat(JSONExtractKeys(attribute), JSONExtractKeys(scope), JSONExtractKeys(resource)) TYPE set(100),
 )
   ENGINE = MergeTree
   PARTITION BY toYYYYMMDD(timestamp)