Skip to content

Commit

Permalink
perf(chstorage): use keys index for logs attributes
Browse files Browse the repository at this point in the history
```
                                       │ benchstat.report.txt.old │           benchstat.report.txt            │
                                       │     ch-read-rows/op      │ ch-read-rows/op  vs base                  │
LogQL/Lookup_by_materialzied_attribute                76.49k ± 0%       76.49k ± 0%        ~ (p=1.000 n=15) ¹
LogQL/Lookup_by_regular_attribute                  63630.02k ± 0%       10.89k ± 0%  -99.98% (p=0.000 n=15)
geomean                                               2.206M            28.86k       -98.69%
¹ all samples are equal
```
  • Loading branch information
tdakkota committed Nov 30, 2024
1 parent f341ca4 commit 18a8e7e
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 4 deletions.
16 changes: 12 additions & 4 deletions internal/chstorage/querier_logs_query.go
Original file line number Diff line number Diff line change
Expand Up @@ -530,8 +530,7 @@ func (q *Querier) logQLLabelMatcher(
case logstorage.LabelTraceID:
return matchHex(chsql.Ident("trace_id"), m)
default:
expr, ok := q.getMaterializedLabelColumn(unmappedLabel)
if ok {
if expr, ok := q.getMaterializedLabelColumn(unmappedLabel); ok {
switch m.Op {
case logql.OpEq, logql.OpNotEq:
return chsql.Eq(expr, chsql.String(m.Value))
Expand All @@ -543,11 +542,12 @@ func (q *Querier) logQLLabelMatcher(
}

exprs := make([]chsql.Expr, 0, 3)
keysExprs := make([]chsql.Expr, 0, cap(exprs))
// Search in all attributes.
for _, column := range []string{
colAttrs,
colResource,
colScope,
colResource,
} {
// TODO: how to match integers, booleans, floats, arrays?
var (
Expand All @@ -563,8 +563,16 @@ func (q *Querier) logQLLabelMatcher(
panic(fmt.Sprintf("unexpected label matcher op %v", m.Op))
}
exprs = append(exprs, sub)
keysExprs = append(keysExprs, chsql.JSONExtractKeys(chsql.Ident(column)))
}
return chsql.JoinOr(exprs...)
// Force Clickhouse to use index.
return chsql.And(
chsql.Has(
chsql.ArrayConcat(keysExprs...),
chsql.String(labelName),
),
chsql.JoinOr(exprs...),
)
}
}

Expand Down
1 change: 1 addition & 0 deletions internal/chstorage/schema_logs.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ const (
INDEX idx_trace_id trace_id TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_body body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1,
INDEX idx_ts timestamp TYPE minmax GRANULARITY 8192,
INDEX attribute_keys arrayConcat(JSONExtractKeys(attribute), JSONExtractKeys(scope), JSONExtractKeys(resource)) TYPE set(100),
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(timestamp)
Expand Down

0 comments on commit 18a8e7e

Please sign in to comment.