From 2286a9464c02602cbdc7241c337497956f36d593 Mon Sep 17 00:00:00 2001 From: duanda Date: Mon, 26 Aug 2024 19:05:13 +0800 Subject: [PATCH] feat: support data visibility --- .../db/mysql/migration/rawsql/init.sql | 10 ++++ .../mysql/migration/rawsql/issu/6.6.1.13.sql | 13 +++++ .../controller/db/mysql/migration/version.go | 2 +- server/controller/db/mysql/model/ch_model.go | 8 +++ .../tagrecorder/ch_statistic_tag.go | 50 +++++++++++++++++++ server/controller/tagrecorder/const.go | 17 +++++++ server/controller/tagrecorder/constraint.go | 4 +- server/controller/tagrecorder/key.go | 6 +++ 8 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 server/controller/db/mysql/migration/rawsql/issu/6.6.1.13.sql create mode 100644 server/controller/tagrecorder/ch_statistic_tag.go diff --git a/server/controller/db/mysql/migration/rawsql/init.sql b/server/controller/db/mysql/migration/rawsql/init.sql index c7973162bfe..01054e7e963 100644 --- a/server/controller/db/mysql/migration/rawsql/init.sql +++ b/server/controller/db/mysql/migration/rawsql/init.sql @@ -2826,3 +2826,13 @@ CREATE TABLE IF NOT EXISTS ch_user ( updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP )ENGINE=innodb DEFAULT CHARSET=utf8; TRUNCATE TABLE ch_user; + +CREATE TABLE IF NOT EXISTS ch_statistic_tag ( + `db` VARCHAR(128) NOT NULL, + `table` VARCHAR(256) NOT NULL, + `type` VARCHAR(128) NOT NULL, + `name` VARCHAR(256) NOT NULL, + `updated_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + PRIMARY KEY (`db`, `table`, `type`, `name`) +)ENGINE=innodb DEFAULT CHARSET=utf8; +TRUNCATE TABLE ch_statistic_tag; diff --git a/server/controller/db/mysql/migration/rawsql/issu/6.6.1.13.sql b/server/controller/db/mysql/migration/rawsql/issu/6.6.1.13.sql new file mode 100644 index 00000000000..68c71f62c71 --- /dev/null +++ b/server/controller/db/mysql/migration/rawsql/issu/6.6.1.13.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS ch_statistic_tag ( + `db` VARCHAR(128) NOT NULL, + `table` VARCHAR(256) NOT NULL, + `type` VARCHAR(128) NOT NULL, + `name` VARCHAR(256) NOT NULL, + `updated_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + PRIMARY KEY (`db`, `table`, `type`, `name`) +)ENGINE=innodb DEFAULT CHARSET=utf8; +TRUNCATE TABLE ch_statistic_tag; + +-- update db_version to latest, remeber update DB_VERSION_EXPECT in migrate/version.go +UPDATE db_version SET version='6.6.1.13'; +-- modify end \ No newline at end of file diff --git a/server/controller/db/mysql/migration/version.go b/server/controller/db/mysql/migration/version.go index f6d5c998c55..2b4e9e9952e 100644 --- a/server/controller/db/mysql/migration/version.go +++ b/server/controller/db/mysql/migration/version.go @@ -18,5 +18,5 @@ package migration const ( DB_VERSION_TABLE = "db_version" - DB_VERSION_EXPECTED = "6.6.1.12" + DB_VERSION_EXPECTED = "6.6.1.13" ) diff --git a/server/controller/db/mysql/model/ch_model.go b/server/controller/db/mysql/model/ch_model.go index 7a9ae55f66f..41030d6ae02 100644 --- a/server/controller/db/mysql/model/ch_model.go +++ b/server/controller/db/mysql/model/ch_model.go @@ -680,3 +680,11 @@ type ChUser struct { Name string `gorm:"column:name;type:varchar(256)" json:"NAME"` UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime:now,type:timestamp" json:"UPDATED_AT"` } + +type ChStatisticTag struct { + Name string `gorm:"primaryKey;column:name;type:varchar(256)" json:"NAME"` + Db string `gorm:"primaryKey;column:db;type:varchar(128)" json:"DB"` + Table string `gorm:"primaryKey;column:table;type:varchar(256)" json:"TABLE"` + Type string `gorm:"primaryKey;column:type;type:varchar(128)" json:"TYPE"` + UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime:now,type:timestamp" json:"UPDATED_AT"` +} diff --git a/server/controller/tagrecorder/ch_statistic_tag.go b/server/controller/tagrecorder/ch_statistic_tag.go new file mode 100644 index 00000000000..20d2c1ad141 --- /dev/null +++ b/server/controller/tagrecorder/ch_statistic_tag.go @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2024 Yunshan Networks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tagrecorder + +import ( + "github.com/deepflowio/deepflow/server/controller/db/mysql" + mysqlmodel "github.com/deepflowio/deepflow/server/controller/db/mysql/model" +) + +type ChStatisticTag struct { + UpdaterComponent[mysqlmodel.ChStatisticTag, StatisticTagKey] +} + +func NewChStatisticTag() *ChStatisticTag { + updater := &ChStatisticTag{ + newUpdaterComponent[mysqlmodel.ChStatisticTag, StatisticTagKey]( + RESOURCE_TYPE_STATISTIC_TAG, + ), + } + updater.updaterDG = updater + return updater +} + +func (r *ChStatisticTag) generateNewData(db *mysql.DB) (map[StatisticTagKey]mysqlmodel.ChStatisticTag, bool) { + + return nil, false +} + +func (r *ChStatisticTag) generateKey(dbItem mysqlmodel.ChStatisticTag) StatisticTagKey { + return StatisticTagKey{Db: dbItem.Db, Table: dbItem.Table, Type: dbItem.Type, Name: dbItem.Name} +} + +func (r *ChStatisticTag) generateUpdateInfo(oldItem, newItem mysqlmodel.ChStatisticTag) (map[string]interface{}, bool) { + + return nil, false +} diff --git a/server/controller/tagrecorder/const.go b/server/controller/tagrecorder/const.go index 4f9640192b5..9c8caf7ed1e 100644 --- a/server/controller/tagrecorder/const.go +++ b/server/controller/tagrecorder/const.go @@ -119,6 +119,8 @@ const ( RESOURCE_TYPE_CH_LABEL_NAME = "ch_prometheus_label_name" RESOURCE_TYPE_CH_METRIC_NAME = "ch_prometheus_metric_name" RESOURCE_TYPE_CH_PROMETHEUS_TARGET_LABEL_LAYOUT = "ch_prometheus_target_label_layout" + + RESOURCE_TYPE_STATISTIC_TAG = "ch_statistic_tag" ) const ( @@ -192,6 +194,8 @@ const ( CH_APP_LABEL_LIVE_VIEW = "app_label_live_view" CH_TARGET_LABEL_LIVE_VIEW = "target_label_live_view" + + CH_STATISTIC_TAG = "statistic_tag" ) const ( @@ -880,6 +884,17 @@ const ( SQL_SOURCE_MYSQL + SQL_LIFETIME + SQL_LAYOUT_FLAT + CREATE_STATISTIC_TAG_SQL = SQL_CREATE_DICT + + "(\n" + + " `db` String,\n" + + " `table` String,\n" + + " `type` String,\n" + + " `name` String\n" + + ")\n" + + "PRIMARY KEY db, table, type, name\n" + + SQL_SOURCE_MYSQL + + SQL_LIFETIME + + SQL_LAYOUT_FLAT ) const ( @@ -989,6 +1004,8 @@ var CREATE_SQL_MAP = map[string]string{ CH_APP_LABEL_LIVE_VIEW: CREATE_APP_LABEL_LIVE_VIEW_SQL, CH_TARGET_LABEL_LIVE_VIEW: CREATE_TARGET_LABEL_LIVE_VIEW_SQL, + + CH_STATISTIC_TAG: CREATE_STATISTIC_TAG_SQL, } var VTAP_TYPE_TO_DEVICE_TYPE = map[int]int{ diff --git a/server/controller/tagrecorder/constraint.go b/server/controller/tagrecorder/constraint.go index 5704537c564..e3cc0b35c00 100644 --- a/server/controller/tagrecorder/constraint.go +++ b/server/controller/tagrecorder/constraint.go @@ -31,12 +31,12 @@ type MySQLChModel interface { mysqlmodel.ChDevice | mysqlmodel.ChIPRelation | mysqlmodel.ChPodGroup | mysqlmodel.ChNetwork | mysqlmodel.ChPod | mysqlmodel.ChPodCluster | mysqlmodel.ChPodNode | mysqlmodel.ChPodNamespace | mysqlmodel.ChTapType | mysqlmodel.ChVTap | mysqlmodel.ChPodK8sLabels | mysqlmodel.ChNodeType | mysqlmodel.ChGProcess | mysqlmodel.ChPodK8sAnnotation | mysqlmodel.ChPodK8sAnnotations | mysqlmodel.ChPodServiceK8sAnnotation | mysqlmodel.ChPodServiceK8sAnnotations | - mysqlmodel.ChPodK8sEnv | mysqlmodel.ChPodK8sEnvs | mysqlmodel.ChPodService | mysqlmodel.ChChost | mysqlmodel.ChPolicy | mysqlmodel.ChNpbTunnel + mysqlmodel.ChPodK8sEnv | mysqlmodel.ChPodK8sEnvs | mysqlmodel.ChPodService | mysqlmodel.ChChost | mysqlmodel.ChPolicy | mysqlmodel.ChNpbTunnel | mysqlmodel.ChStatisticTag } // ch资源的组合key type ChModelKey interface { PrometheusTargetLabelKey | PrometheusAPPLabelKey | OSAPPTagKey | OSAPPTagsKey | CloudTagsKey | CloudTagKey | IntEnumTagKey | StringEnumTagKey | VtapPortKey | IPResourceKey | K8sLabelKey | PortIDKey | PortIPKey | PortDeviceKey | IDKey | DeviceKey | IPRelationKey | TapTypeKey | K8sLabelsKey | NodeTypeKey | K8sAnnotationKey | K8sAnnotationsKey | - K8sEnvKey | K8sEnvsKey | PolicyKey + K8sEnvKey | K8sEnvsKey | PolicyKey | StatisticTagKey } diff --git a/server/controller/tagrecorder/key.go b/server/controller/tagrecorder/key.go index 2c7fa30b9bc..f475602ad3b 100644 --- a/server/controller/tagrecorder/key.go +++ b/server/controller/tagrecorder/key.go @@ -138,3 +138,9 @@ type PolicyKey struct { ACLGID int TunnelType int } +type StatisticTagKey struct { + Db string + Table string + Type string + Name string +}