perf: add some unnecessary indexes

go-faster · Dec 4, 2024 · 6fcd8b4 · 6fcd8b4
1 parent 2a8ea8d
commit 6fcd8b4
Show file tree

Hide file tree

Showing 8 changed files with 62 additions and 18 deletions.
diff --git a/internal/chstorage/_golden/log_columns_ddl.sql b/internal/chstorage/_golden/log_columns_ddl.sql
@@ -16,10 +16,15 @@ CREATE TABLE IF NOT EXISTS `table`
 	`scope_version`       LowCardinality(String),
 	`scope`               LowCardinality(String),
 
-	INDEX `idx_trace_id`   trace_id TYPE bloom_filter(0.001) GRANULARITY 1,
-	INDEX `idx_body`       body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1,
-	INDEX `idx_ts`         timestamp TYPE minmax GRANULARITY 8192,
-	INDEX `attribute_keys` arrayConcat(JSONExtractKeys(attribute), JSONExtractKeys(scope), JSONExtractKeys(resource)) TYPE set(100)
+	INDEX `idx_trace_id`           trace_id TYPE bloom_filter(0.001) GRANULARITY 1,
+	INDEX `idx_body`               body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1,
+	INDEX `idx_ts`                 timestamp TYPE minmax GRANULARITY 8192,
+	INDEX `idx_arr_join_attribute` arrayJoin(JSONExtractKeys(attribute)) TYPE set(100),
+	INDEX `idx_keys_attribute`     JSONExtractKeys(attribute) TYPE set(100),
+	INDEX `idx_arr_join_resource`  arrayJoin(JSONExtractKeys(resource)) TYPE set(100),
+	INDEX `idx_keys_resource`      JSONExtractKeys(resource) TYPE set(100),
+	INDEX `idx_arr_join_scope`     arrayJoin(JSONExtractKeys(scope)) TYPE set(100),
+	INDEX `idx_keys_scope`         JSONExtractKeys(scope) TYPE set(100)
 )
 ENGINE = MergeTree
 PARTITION BY toYYYYMMDD(timestamp)

diff --git a/internal/chstorage/_golden/schema.logs.sql b/internal/chstorage/_golden/schema.logs.sql
@@ -16,10 +16,15 @@ CREATE TABLE IF NOT EXISTS `logs`
 	`scope_version`       LowCardinality(String),
 	`scope`               LowCardinality(String),
 
-	INDEX `idx_trace_id`   trace_id TYPE bloom_filter(0.001) GRANULARITY 1,
-	INDEX `idx_body`       body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1,
-	INDEX `idx_ts`         timestamp TYPE minmax GRANULARITY 8192,
-	INDEX `attribute_keys` arrayConcat(JSONExtractKeys(attribute), JSONExtractKeys(scope), JSONExtractKeys(resource)) TYPE set(100)
+	INDEX `idx_trace_id`           trace_id TYPE bloom_filter(0.001) GRANULARITY 1,
+	INDEX `idx_body`               body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1,
+	INDEX `idx_ts`                 timestamp TYPE minmax GRANULARITY 8192,
+	INDEX `idx_arr_join_attribute` arrayJoin(JSONExtractKeys(attribute)) TYPE set(100),
+	INDEX `idx_keys_attribute`     JSONExtractKeys(attribute) TYPE set(100),
+	INDEX `idx_arr_join_resource`  arrayJoin(JSONExtractKeys(resource)) TYPE set(100),
+	INDEX `idx_keys_resource`      JSONExtractKeys(resource) TYPE set(100),
+	INDEX `idx_arr_join_scope`     arrayJoin(JSONExtractKeys(scope)) TYPE set(100),
+	INDEX `idx_keys_scope`         JSONExtractKeys(scope) TYPE set(100)
 )
 ENGINE = MergeTree
 PARTITION BY toYYYYMMDD(timestamp)

diff --git a/internal/chstorage/_golden/schema.metrics_exemplars.sql b/internal/chstorage/_golden/schema.metrics_exemplars.sql
@@ -10,7 +10,14 @@ CREATE TABLE IF NOT EXISTS `metrics_exemplars`
 	`span_id`             FixedString(8),
 	`attribute`           LowCardinality(String),
 	`resource`            LowCardinality(String),
-	`scope`               LowCardinality(String)
+	`scope`               LowCardinality(String),
+
+	INDEX `idx_arr_join_attribute` arrayJoin(JSONExtractKeys(attribute)) TYPE set(100),
+	INDEX `idx_keys_attribute`     JSONExtractKeys(attribute) TYPE set(100),
+	INDEX `idx_arr_join_resource`  arrayJoin(JSONExtractKeys(resource)) TYPE set(100),
+	INDEX `idx_keys_resource`      JSONExtractKeys(resource) TYPE set(100),
+	INDEX `idx_arr_join_scope`     arrayJoin(JSONExtractKeys(scope)) TYPE set(100),
+	INDEX `idx_keys_scope`         JSONExtractKeys(scope) TYPE set(100)
 )
 ENGINE = MergeTree
 PARTITION BY toYYYYMMDD(timestamp)

diff --git a/internal/chstorage/_golden/schema.metrics_exp_histograms.sql b/internal/chstorage/_golden/schema.metrics_exp_histograms.sql
@@ -16,7 +16,14 @@ CREATE TABLE IF NOT EXISTS `metrics_exp_histograms`
 	`flags`                                UInt8                  CODEC(T64, ZSTD(1)),
 	`attribute`                            LowCardinality(String),
 	`resource`                             LowCardinality(String),
-	`scope`                                LowCardinality(String)
+	`scope`                                LowCardinality(String),
+
+	INDEX `idx_arr_join_attribute` arrayJoin(JSONExtractKeys(attribute)) TYPE set(100),
+	INDEX `idx_keys_attribute`     JSONExtractKeys(attribute) TYPE set(100),
+	INDEX `idx_arr_join_resource`  arrayJoin(JSONExtractKeys(resource)) TYPE set(100),
+	INDEX `idx_keys_resource`      JSONExtractKeys(resource) TYPE set(100),
+	INDEX `idx_arr_join_scope`     arrayJoin(JSONExtractKeys(scope)) TYPE set(100),
+	INDEX `idx_keys_scope`         JSONExtractKeys(scope) TYPE set(100)
 )
 ENGINE = MergeTree
 PARTITION BY toYYYYMMDD(timestamp)

diff --git a/internal/chstorage/_golden/schema.metrics_points.sql b/internal/chstorage/_golden/schema.metrics_points.sql
@@ -20,7 +20,13 @@ CREATE TABLE IF NOT EXISTS `metrics_points`
 	`resource`        LowCardinality(String),
 	`scope`           LowCardinality(String),
 
-	INDEX `idx_ts` timestamp TYPE minmax GRANULARITY 8192
+	INDEX `idx_ts`                 timestamp TYPE minmax GRANULARITY 8192,
+	INDEX `idx_arr_join_attribute` arrayJoin(JSONExtractKeys(attribute)) TYPE set(100),
+	INDEX `idx_keys_attribute`     JSONExtractKeys(attribute) TYPE set(100),
+	INDEX `idx_arr_join_resource`  arrayJoin(JSONExtractKeys(resource)) TYPE set(100),
+	INDEX `idx_keys_resource`      JSONExtractKeys(resource) TYPE set(100),
+	INDEX `idx_arr_join_scope`     arrayJoin(JSONExtractKeys(scope)) TYPE set(100),
+	INDEX `idx_keys_scope`         JSONExtractKeys(scope) TYPE set(100)
 )
 ENGINE = MergeTree
 PARTITION BY toYYYYMMDD(timestamp)

diff --git a/internal/chstorage/_golden/schema.traces_spans.sql b/internal/chstorage/_golden/schema.traces_spans.sql
@@ -28,7 +28,13 @@ CREATE TABLE IF NOT EXISTS `traces_spans`
 	`links_tracestates`   Array(String),
 	`links_attributes`    Array(String),
 
-	INDEX `idx_trace_id` trace_id TYPE bloom_filter(0.001) GRANULARITY 1
+	INDEX `idx_trace_id`           trace_id TYPE bloom_filter(0.001) GRANULARITY 1,
+	INDEX `idx_arr_join_attribute` arrayJoin(JSONExtractKeys(attribute)) TYPE set(100),
+	INDEX `idx_keys_attribute`     JSONExtractKeys(attribute) TYPE set(100),
+	INDEX `idx_arr_join_resource`  arrayJoin(JSONExtractKeys(resource)) TYPE set(100),
+	INDEX `idx_keys_resource`      JSONExtractKeys(resource) TYPE set(100),
+	INDEX `idx_arr_join_scope`     arrayJoin(JSONExtractKeys(scope)) TYPE set(100),
+	INDEX `idx_keys_scope`         JSONExtractKeys(scope) TYPE set(100)
 )
 ENGINE = MergeTree
 ORDER BY (`service_namespace`, `service_name`, `resource`, `start`)

diff --git a/internal/chstorage/attributes.go b/internal/chstorage/attributes.go
@@ -282,6 +282,20 @@ func (a *Attributes) DDL(table *ddl.Table) {
 			Type: a.Value.Type(),
 		},
 	)
+	table.Indexes = append(table.Indexes,
+		ddl.Index{
+			Name:   "idx_arr_join_" + a.Name,
+			Target: "arrayJoin(JSONExtractKeys(" + a.Name + "))",
+			Type:   "set",
+			Params: []string{"100"},
+		},
+		ddl.Index{
+			Name:   "idx_keys_" + a.Name,
+			Target: "JSONExtractKeys(" + a.Name + ")",
+			Type:   "set",
+			Params: []string{"100"},
+		},
+	)
 }
 
 func attrsToLabels(m otelstorage.Attrs, to map[string]string) {

diff --git a/internal/chstorage/columns_logs.go b/internal/chstorage/columns_logs.go
@@ -48,12 +48,6 @@ func (c *logColumns) DDL() ddl.Table {
 				Type:        "minmax",
 				Granularity: 8192,
 			},
-			{
-				Name:   "attribute_keys",
-				Target: "arrayConcat(JSONExtractKeys(attribute), JSONExtractKeys(scope), JSONExtractKeys(resource))",
-				Type:   "set",
-				Params: []string{"100"},
-			},
 		},
 		Columns: []ddl.Column{
 			{