feat(chstorage): use ddl generation

go-faster · Dec 3, 2024 · 5fc2dd7 · 5fc2dd7
1 parent b7ffb8a
commit 5fc2dd7
Show file tree

Hide file tree

Showing 25 changed files with 794 additions and 360 deletions.
diff --git a/internal/chstorage/_golden/log_columns_ddl.sql b/internal/chstorage/_golden/log_columns_ddl.sql
@@ -1,9 +1,9 @@
-CREATE TABLE IF NOT EXISTS table
+CREATE TABLE IF NOT EXISTS `table`
 (
 	`service_instance_id` LowCardinality(String) COMMENT 'service.instance.id',
 	`service_name`        LowCardinality(String) COMMENT 'service.name',
 	`service_namespace`   LowCardinality(String) COMMENT 'service.namespace',
-	`timestamp`           DateTime64(9),
+	`timestamp`           DateTime64(9)          CODEC(Delta, ZSTD(1)),
 	`severity_number`     UInt8,
 	`severity_text`       LowCardinality(String),
 	`trace_id`            FixedString(16),
@@ -19,7 +19,7 @@ CREATE TABLE IF NOT EXISTS table
 	INDEX `idx_trace_id`   trace_id TYPE bloom_filter(0.001) GRANULARITY 1,
 	INDEX `idx_body`       body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1,
 	INDEX `idx_ts`         timestamp TYPE minmax GRANULARITY 8192,
-	INDEX `attribute_keys` arrayConcat(JSONExtractKeys(attribute), JSONExtractKeys(scope), JSONExtractKeys(resource)) TYPE set(100),
+	INDEX `attribute_keys` arrayConcat(JSONExtractKeys(attribute), JSONExtractKeys(scope), JSONExtractKeys(resource)) TYPE set(100)
 )
 ENGINE = MergeTree
 PARTITION BY toYYYYMMDD(timestamp)

diff --git a/internal/chstorage/_golden/schema. logs.sql b/internal/chstorage/_golden/schema. logs.sql
@@ -0,0 +1,27 @@
+CREATE TABLE IF NOT EXISTS `logs`
+(
+	`service_instance_id` LowCardinality(String) COMMENT 'service.instance.id',
+	`service_name`        LowCardinality(String) COMMENT 'service.name',
+	`service_namespace`   LowCardinality(String) COMMENT 'service.namespace',
+	`timestamp`           DateTime64(9)          CODEC(Delta, ZSTD(1)),
+	`severity_number`     UInt8,
+	`severity_text`       LowCardinality(String),
+	`trace_id`            FixedString(16),
+	`span_id`             FixedString(8),
+	`trace_flags`         UInt8,
+	`body`                String,
+	`attribute`           String,
+	`resource`            LowCardinality(String),
+	`scope_name`          LowCardinality(String),
+	`scope_version`       LowCardinality(String),
+	`scope`               LowCardinality(String),
+
+	INDEX `idx_trace_id`   trace_id TYPE bloom_filter(0.001) GRANULARITY 1,
+	INDEX `idx_body`       body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1,
+	INDEX `idx_ts`         timestamp TYPE minmax GRANULARITY 8192,
+	INDEX `attribute_keys` arrayConcat(JSONExtractKeys(attribute), JSONExtractKeys(scope), JSONExtractKeys(resource)) TYPE set(100)
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(timestamp)
+ORDER BY (`severity_number`, `service_namespace`, `service_name`, `resource`, `timestamp`)
+PRIMARY KEY (`severity_number`, `service_namespace`, `service_name`, `resource`)
diff --git a/internal/chstorage/_golden/schema. logs_attrs.sql b/internal/chstorage/_golden/schema. logs_attrs.sql
@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS `logs_attrs`
+(
+	`name` String COMMENT 'foo_bar',
+	`key`  String COMMENT 'foo.bar'
+)
+ENGINE = ReplacingMergeTree
+ORDER BY (`name`)
diff --git a/internal/chstorage/_golden/schema. metrics_exemplars.sql b/internal/chstorage/_golden/schema. metrics_exemplars.sql
@@ -0,0 +1,17 @@
+CREATE TABLE IF NOT EXISTS `metrics_exemplars`
+(
+	`name`                LowCardinality(String),
+	`name_normalized`     LowCardinality(String),
+	`timestamp`           DateTime64(9)          CODEC(Delta, ZSTD(1)),
+	`filtered_attributes` String,
+	`exemplar_timestamp`  DateTime64(9)          CODEC(Delta, ZSTD(1)),
+	`value`               Float64,
+	`trace_id`            FixedString(16),
+	`span_id`             FixedString(8),
+	`attribute`           LowCardinality(String),
+	`resource`            LowCardinality(String),
+	`scope`               LowCardinality(String)
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(timestamp)
+ORDER BY (`name_normalized`, `resource`, `attribute`, `timestamp`)
diff --git a/internal/chstorage/_golden/schema. metrics_exp_histograms.sql b/internal/chstorage/_golden/schema. metrics_exp_histograms.sql
@@ -0,0 +1,23 @@
+CREATE TABLE IF NOT EXISTS `metrics_exp_histograms`
+(
+	`name`                                 LowCardinality(String) CODEC(ZSTD(1)),
+	`name_normalized`                      LowCardinality(String),
+	`timestamp`                            DateTime64(9)          CODEC(Delta, ZSTD(1)),
+	`exp_histogram_count`                  UInt64,
+	`exp_histogram_sum`                    Nullable(Float64),
+	`exp_histogram_min`                    Nullable(Float64),
+	`exp_histogram_max`                    Nullable(Float64),
+	`exp_histogram_scale`                  Int32,
+	`exp_histogram_zerocount`              UInt64,
+	`exp_histogram_positive_offset`        Int32,
+	`exp_histogram_positive_bucket_counts` Array(UInt64),
+	`exp_histogram_negative_offset`        Int32,
+	`exp_histogram_negative_bucket_counts` Array(UInt64),
+	`flags`                                UInt8                  CODEC(T64, ZSTD(1)),
+	`attribute`                            LowCardinality(String),
+	`resource`                             LowCardinality(String),
+	`scope`                                LowCardinality(String)
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(timestamp)
+ORDER BY (`timestamp`)
diff --git a/internal/chstorage/_golden/schema. metrics_labels.sql b/internal/chstorage/_golden/schema. metrics_labels.sql
@@ -0,0 +1,10 @@
+CREATE TABLE IF NOT EXISTS `metrics_labels`
+(
+	`name`             LowCardinality(String) COMMENT 'original name, i.e. foo.bar',
+	`name_normalized`  LowCardinality(String) COMMENT 'normalized name, foo_bar',
+	`value`            String,
+	`value_normalized` String,
+	`scope`            Enum8('NONE' = 0, 'RESOURCE' = 1, 'INSTRUMENTATION' = 2, 'ATTRIBUTE' = 4)
+)
+ENGINE = ReplacingMergeTree
+ORDER BY (`name_normalized`, `value`, `scope`)
diff --git a/internal/chstorage/_golden/schema. metrics_points.sql b/internal/chstorage/_golden/schema. metrics_points.sql
@@ -0,0 +1,28 @@
+CREATE TABLE IF NOT EXISTS `metrics_points`
+(
+	`name`            LowCardinality(String) CODEC(ZSTD(1)),
+	`name_normalized` LowCardinality(String),
+	`timestamp`       DateTime64(9)          CODEC(Delta, ZSTD(1)),
+	`mapping`         Enum8(
+		'NO_MAPPING' = 0,
+		'HISTOGRAM_COUNT' = 1,
+		'HISTOGRAM_SUM' = 2,
+		'HISTOGRAM_MIN' = 3,
+		'HISTOGRAM_MAX' = 4,
+		'HISTOGRAM_BUCKET' = 5,
+		'SUMMARY_COUNT' = 6,
+		'SUMMARY_SUM' = 7,
+		'SUMMARY_QUANTILE' = 8
+		) CODEC(T64, ZSTD(1)),
+	`value`           Float64                CODEC(Gorilla, ZSTD(1)),
+	`flags`           UInt8                  CODEC(T64, ZSTD(1)),
+	`attribute`       LowCardinality(String),
+	`resource`        LowCardinality(String),
+	`scope`           LowCardinality(String),
+
+	INDEX `idx_ts` timestamp TYPE minmax GRANULARITY 8192
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(timestamp)
+ORDER BY (`name_normalized`, `mapping`, `resource`, `attribute`, `timestamp`)
+PRIMARY KEY (`name_normalized`, `mapping`, `resource`, `attribute`)
diff --git a/internal/chstorage/_golden/schema. migration.sql b/internal/chstorage/_golden/schema. migration.sql
@@ -0,0 +1,8 @@
+CREATE TABLE IF NOT EXISTS `migration`
+(
+	`table` String,
+	`ddl`   String,
+	`ts`    DateTime DEFAULT now()
+)
+ENGINE = ReplacingMergeTree(ts)
+ORDER BY (`table`)
diff --git a/internal/chstorage/_golden/schema. traces_spans.sql b/internal/chstorage/_golden/schema. traces_spans.sql
@@ -0,0 +1,35 @@
+CREATE TABLE IF NOT EXISTS `traces_spans`
+(
+	`service_instance_id` LowCardinality(String) COMMENT 'service.instance.id',
+	`service_name`        LowCardinality(String) COMMENT 'service.name',
+	`service_namespace`   LowCardinality(String) COMMENT 'service.namespace',
+	`trace_id`            FixedString(16),
+	`span_id`             FixedString(8),
+	`trace_state`         String,
+	`parent_span_id`      FixedString(8),
+	`name`                LowCardinality(String),
+	`kind`                Enum8('KIND_UNSPECIFIED' = 0,'KIND_INTERNAL' = 1,'KIND_SERVER' = 2,'KIND_CLIENT' = 3,'KIND_PRODUCER' = 4,'KIND_CONSUMER' = 5),
+	`start`               DateTime64(9)          CODEC(Delta, ZSTD(1)),
+	`end`                 DateTime64(9)          CODEC(Delta, ZSTD(1)),
+	`duration_ns`         UInt64                 MATERIALIZED toUnixTimestamp64Nano(end)-toUnixTimestamp64Nano(start) CODEC(T64, ZSTD(1)),
+	`status_code`         UInt8,
+	`status_message`      LowCardinality(String),
+	`batch_id`            UUID,
+	`attribute`           LowCardinality(String),
+	`resource`            LowCardinality(String),
+	`scope`               LowCardinality(String),
+	`scope_name`          LowCardinality(String),
+	`scope_version`       LowCardinality(String),
+	`events_timestamps`   Array(DateTime64(9)),
+	`events_names`        Array(String),
+	`events_attributes`   Array(String),
+	`links_trace_ids`     Array(FixedString(16)),
+	`links_span_ids`      Array(FixedString(8)),
+	`links_tracestates`   Array(String),
+	`links_attributes`    Array(String),
+
+	INDEX `idx_trace_id` trace_id TYPE bloom_filter(0.001) GRANULARITY 1
+)
+ENGINE = MergeTree
+ORDER BY (`service_namespace`, `service_name`, `resource`, `start`)
+PRIMARY KEY (`service_namespace`, `service_name`, `resource`)
diff --git a/internal/chstorage/_golden/schema. traces_tags.sql b/internal/chstorage/_golden/schema. traces_tags.sql
@@ -0,0 +1,9 @@
+CREATE TABLE IF NOT EXISTS `traces_tags`
+(
+	`name`       LowCardinality(String),
+	`value`      String,
+	`value_type` Enum8('EMPTY' = 0,'STR' = 1,'INT' = 2,'DOUBLE' = 3,'BOOL' = 4,'MAP' = 5,'SLICE' = 6,'BYTES' = 7),
+	`scope`      Enum8('NONE' = 0, 'RESOURCE' = 1, 'SPAN' = 2, 'INSTRUMENTATION' = 3)
+)
+ENGINE = ReplacingMergeTree
+ORDER BY (`value_type`, `name`, `value`)
diff --git a/internal/chstorage/_golden/trace_spans.sql b/internal/chstorage/_golden/trace_spans.sql
diff --git a/internal/chstorage/columns_logs.go b/internal/chstorage/columns_logs.go
@@ -72,8 +72,9 @@ func (c *logColumns) DDL() ddl.Table {
 				Comment: "service.namespace",
 			},
 			{
-				Name: "timestamp",
-				Type: c.timestamp.Type(),
+				Name:  "timestamp",
+				Type:  c.timestamp.Type(),
+				Codec: "Delta, ZSTD(1)",
 			},
 			{
 				Name: "severity_number",
@@ -344,3 +345,22 @@ func (c *logAttrMapColumns) AddRow(name []byte, key string) {
 	c.name.AppendBytes(name)
 	c.key.Append(key)
 }
+
+func (c *logAttrMapColumns) DDL() ddl.Table {
+	return ddl.Table{
+		OrderBy: []string{"name"},
+		Engine:  "ReplacingMergeTree",
+		Columns: []ddl.Column{
+			{
+				Name:    "name",
+				Type:    c.name.Type(),
+				Comment: "foo_bar",
+			},
+			{
+				Name:    "key",
+				Type:    c.key.Type(),
+				Comment: "foo.bar",
+			},
+		},
+	}
+}