From 6fcd8b4db78a2e576d0ae900b7e49ee0edf20980 Mon Sep 17 00:00:00 2001 From: Aleksandr Razumov Date: Wed, 4 Dec 2024 10:34:58 +0300 Subject: [PATCH] perf: add some unnecessary indexes --- internal/chstorage/_golden/log_columns_ddl.sql | 13 +++++++++---- internal/chstorage/_golden/schema.logs.sql | 13 +++++++++---- .../chstorage/_golden/schema.metrics_exemplars.sql | 9 ++++++++- .../_golden/schema.metrics_exp_histograms.sql | 9 ++++++++- .../chstorage/_golden/schema.metrics_points.sql | 8 +++++++- internal/chstorage/_golden/schema.traces_spans.sql | 8 +++++++- internal/chstorage/attributes.go | 14 ++++++++++++++ internal/chstorage/columns_logs.go | 6 ------ 8 files changed, 62 insertions(+), 18 deletions(-) diff --git a/internal/chstorage/_golden/log_columns_ddl.sql b/internal/chstorage/_golden/log_columns_ddl.sql index e5c9bd5d..4ec95f33 100644 --- a/internal/chstorage/_golden/log_columns_ddl.sql +++ b/internal/chstorage/_golden/log_columns_ddl.sql @@ -16,10 +16,15 @@ CREATE TABLE IF NOT EXISTS `table` `scope_version` LowCardinality(String), `scope` LowCardinality(String), - INDEX `idx_trace_id` trace_id TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX `idx_body` body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1, - INDEX `idx_ts` timestamp TYPE minmax GRANULARITY 8192, - INDEX `attribute_keys` arrayConcat(JSONExtractKeys(attribute), JSONExtractKeys(scope), JSONExtractKeys(resource)) TYPE set(100) + INDEX `idx_trace_id` trace_id TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX `idx_body` body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1, + INDEX `idx_ts` timestamp TYPE minmax GRANULARITY 8192, + INDEX `idx_arr_join_attribute` arrayJoin(JSONExtractKeys(attribute)) TYPE set(100), + INDEX `idx_keys_attribute` JSONExtractKeys(attribute) TYPE set(100), + INDEX `idx_arr_join_resource` arrayJoin(JSONExtractKeys(resource)) TYPE set(100), + INDEX `idx_keys_resource` JSONExtractKeys(resource) TYPE set(100), + INDEX `idx_arr_join_scope` arrayJoin(JSONExtractKeys(scope)) TYPE set(100), + INDEX `idx_keys_scope` JSONExtractKeys(scope) TYPE set(100) ) ENGINE = MergeTree PARTITION BY toYYYYMMDD(timestamp) diff --git a/internal/chstorage/_golden/schema.logs.sql b/internal/chstorage/_golden/schema.logs.sql index da011959..d3db279f 100644 --- a/internal/chstorage/_golden/schema.logs.sql +++ b/internal/chstorage/_golden/schema.logs.sql @@ -16,10 +16,15 @@ CREATE TABLE IF NOT EXISTS `logs` `scope_version` LowCardinality(String), `scope` LowCardinality(String), - INDEX `idx_trace_id` trace_id TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX `idx_body` body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1, - INDEX `idx_ts` timestamp TYPE minmax GRANULARITY 8192, - INDEX `attribute_keys` arrayConcat(JSONExtractKeys(attribute), JSONExtractKeys(scope), JSONExtractKeys(resource)) TYPE set(100) + INDEX `idx_trace_id` trace_id TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX `idx_body` body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1, + INDEX `idx_ts` timestamp TYPE minmax GRANULARITY 8192, + INDEX `idx_arr_join_attribute` arrayJoin(JSONExtractKeys(attribute)) TYPE set(100), + INDEX `idx_keys_attribute` JSONExtractKeys(attribute) TYPE set(100), + INDEX `idx_arr_join_resource` arrayJoin(JSONExtractKeys(resource)) TYPE set(100), + INDEX `idx_keys_resource` JSONExtractKeys(resource) TYPE set(100), + INDEX `idx_arr_join_scope` arrayJoin(JSONExtractKeys(scope)) TYPE set(100), + INDEX `idx_keys_scope` JSONExtractKeys(scope) TYPE set(100) ) ENGINE = MergeTree PARTITION BY toYYYYMMDD(timestamp) diff --git a/internal/chstorage/_golden/schema.metrics_exemplars.sql b/internal/chstorage/_golden/schema.metrics_exemplars.sql index a86ee2c0..1b01b8e6 100644 --- a/internal/chstorage/_golden/schema.metrics_exemplars.sql +++ b/internal/chstorage/_golden/schema.metrics_exemplars.sql @@ -10,7 +10,14 @@ CREATE TABLE IF NOT EXISTS `metrics_exemplars` `span_id` FixedString(8), `attribute` LowCardinality(String), `resource` LowCardinality(String), - `scope` LowCardinality(String) + `scope` LowCardinality(String), + + INDEX `idx_arr_join_attribute` arrayJoin(JSONExtractKeys(attribute)) TYPE set(100), + INDEX `idx_keys_attribute` JSONExtractKeys(attribute) TYPE set(100), + INDEX `idx_arr_join_resource` arrayJoin(JSONExtractKeys(resource)) TYPE set(100), + INDEX `idx_keys_resource` JSONExtractKeys(resource) TYPE set(100), + INDEX `idx_arr_join_scope` arrayJoin(JSONExtractKeys(scope)) TYPE set(100), + INDEX `idx_keys_scope` JSONExtractKeys(scope) TYPE set(100) ) ENGINE = MergeTree PARTITION BY toYYYYMMDD(timestamp) diff --git a/internal/chstorage/_golden/schema.metrics_exp_histograms.sql b/internal/chstorage/_golden/schema.metrics_exp_histograms.sql index 5e35be32..56b29e54 100644 --- a/internal/chstorage/_golden/schema.metrics_exp_histograms.sql +++ b/internal/chstorage/_golden/schema.metrics_exp_histograms.sql @@ -16,7 +16,14 @@ CREATE TABLE IF NOT EXISTS `metrics_exp_histograms` `flags` UInt8 CODEC(T64, ZSTD(1)), `attribute` LowCardinality(String), `resource` LowCardinality(String), - `scope` LowCardinality(String) + `scope` LowCardinality(String), + + INDEX `idx_arr_join_attribute` arrayJoin(JSONExtractKeys(attribute)) TYPE set(100), + INDEX `idx_keys_attribute` JSONExtractKeys(attribute) TYPE set(100), + INDEX `idx_arr_join_resource` arrayJoin(JSONExtractKeys(resource)) TYPE set(100), + INDEX `idx_keys_resource` JSONExtractKeys(resource) TYPE set(100), + INDEX `idx_arr_join_scope` arrayJoin(JSONExtractKeys(scope)) TYPE set(100), + INDEX `idx_keys_scope` JSONExtractKeys(scope) TYPE set(100) ) ENGINE = MergeTree PARTITION BY toYYYYMMDD(timestamp) diff --git a/internal/chstorage/_golden/schema.metrics_points.sql b/internal/chstorage/_golden/schema.metrics_points.sql index a35c8e54..2732dcb4 100644 --- a/internal/chstorage/_golden/schema.metrics_points.sql +++ b/internal/chstorage/_golden/schema.metrics_points.sql @@ -20,7 +20,13 @@ CREATE TABLE IF NOT EXISTS `metrics_points` `resource` LowCardinality(String), `scope` LowCardinality(String), - INDEX `idx_ts` timestamp TYPE minmax GRANULARITY 8192 + INDEX `idx_ts` timestamp TYPE minmax GRANULARITY 8192, + INDEX `idx_arr_join_attribute` arrayJoin(JSONExtractKeys(attribute)) TYPE set(100), + INDEX `idx_keys_attribute` JSONExtractKeys(attribute) TYPE set(100), + INDEX `idx_arr_join_resource` arrayJoin(JSONExtractKeys(resource)) TYPE set(100), + INDEX `idx_keys_resource` JSONExtractKeys(resource) TYPE set(100), + INDEX `idx_arr_join_scope` arrayJoin(JSONExtractKeys(scope)) TYPE set(100), + INDEX `idx_keys_scope` JSONExtractKeys(scope) TYPE set(100) ) ENGINE = MergeTree PARTITION BY toYYYYMMDD(timestamp) diff --git a/internal/chstorage/_golden/schema.traces_spans.sql b/internal/chstorage/_golden/schema.traces_spans.sql index 1732dd9e..83af0c88 100644 --- a/internal/chstorage/_golden/schema.traces_spans.sql +++ b/internal/chstorage/_golden/schema.traces_spans.sql @@ -28,7 +28,13 @@ CREATE TABLE IF NOT EXISTS `traces_spans` `links_tracestates` Array(String), `links_attributes` Array(String), - INDEX `idx_trace_id` trace_id TYPE bloom_filter(0.001) GRANULARITY 1 + INDEX `idx_trace_id` trace_id TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX `idx_arr_join_attribute` arrayJoin(JSONExtractKeys(attribute)) TYPE set(100), + INDEX `idx_keys_attribute` JSONExtractKeys(attribute) TYPE set(100), + INDEX `idx_arr_join_resource` arrayJoin(JSONExtractKeys(resource)) TYPE set(100), + INDEX `idx_keys_resource` JSONExtractKeys(resource) TYPE set(100), + INDEX `idx_arr_join_scope` arrayJoin(JSONExtractKeys(scope)) TYPE set(100), + INDEX `idx_keys_scope` JSONExtractKeys(scope) TYPE set(100) ) ENGINE = MergeTree ORDER BY (`service_namespace`, `service_name`, `resource`, `start`) diff --git a/internal/chstorage/attributes.go b/internal/chstorage/attributes.go index c37677fe..2c621477 100644 --- a/internal/chstorage/attributes.go +++ b/internal/chstorage/attributes.go @@ -282,6 +282,20 @@ func (a *Attributes) DDL(table *ddl.Table) { Type: a.Value.Type(), }, ) + table.Indexes = append(table.Indexes, + ddl.Index{ + Name: "idx_arr_join_" + a.Name, + Target: "arrayJoin(JSONExtractKeys(" + a.Name + "))", + Type: "set", + Params: []string{"100"}, + }, + ddl.Index{ + Name: "idx_keys_" + a.Name, + Target: "JSONExtractKeys(" + a.Name + ")", + Type: "set", + Params: []string{"100"}, + }, + ) } func attrsToLabels(m otelstorage.Attrs, to map[string]string) { diff --git a/internal/chstorage/columns_logs.go b/internal/chstorage/columns_logs.go index da1c36e5..5349c841 100644 --- a/internal/chstorage/columns_logs.go +++ b/internal/chstorage/columns_logs.go @@ -48,12 +48,6 @@ func (c *logColumns) DDL() ddl.Table { Type: "minmax", Granularity: 8192, }, - { - Name: "attribute_keys", - Target: "arrayConcat(JSONExtractKeys(attribute), JSONExtractKeys(scope), JSONExtractKeys(resource))", - Type: "set", - Params: []string{"100"}, - }, }, Columns: []ddl.Column{ {