diff --git a/server/controller/db/mysql/migrator/schema/rawsql/init.sql b/server/controller/db/mysql/migrator/schema/rawsql/init.sql index 3ef75feabe2..1a108042718 100644 --- a/server/controller/db/mysql/migrator/schema/rawsql/init.sql +++ b/server/controller/db/mysql/migrator/schema/rawsql/init.sql @@ -2205,12 +2205,13 @@ CREATE TABLE IF NOT EXISTS mail_server ( )ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; TRUNCATE TABLE mail_server; - CREATE TABLE IF NOT EXISTS ch_string_enum ( tag_name VARCHAR(256) NOT NULL , value VARCHAR(256) NOT NULL, - name VARCHAR(256) , - description VARCHAR(256) , + name_zh VARCHAR(256) , + name_en VARCHAR(256) , + description_zh VARCHAR(256) , + description_en VARCHAR(256) , updated_at DATETIME NOT NULL ON UPDATE CURRENT_TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (tag_name,value) )ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; @@ -2219,8 +2220,10 @@ TRUNCATE TABLE ch_string_enum; CREATE TABLE IF NOT EXISTS ch_int_enum ( tag_name VARCHAR(256) NOT NULL, value INTEGER DEFAULT 0, - name VARCHAR(256) , - description VARCHAR(256) , + name_zh VARCHAR(256) , + name_en VARCHAR(256) , + description_zh VARCHAR(256) , + description_en VARCHAR(256) , updated_at DATETIME NOT NULL ON UPDATE CURRENT_TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (tag_name,value) )ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; diff --git a/server/controller/db/mysql/migrator/schema/rawsql/issu/6.6.1.19.sql b/server/controller/db/mysql/migrator/schema/rawsql/issu/6.6.1.19.sql new file mode 100644 index 00000000000..e6aa432578e --- /dev/null +++ b/server/controller/db/mysql/migrator/schema/rawsql/issu/6.6.1.19.sql @@ -0,0 +1,29 @@ +DROP TABLE IF EXISTS ch_string_enum; +DROP TABLE IF EXISTS ch_int_enum; + +CREATE TABLE IF NOT EXISTS ch_string_enum ( + tag_name VARCHAR(256) NOT NULL , + value VARCHAR(256) NOT NULL, + name_zh VARCHAR(256) , + name_en VARCHAR(256) , + description_zh VARCHAR(256) , + description_en VARCHAR(256) , + updated_at DATETIME NOT NULL ON UPDATE CURRENT_TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (tag_name,value) +)ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; + +CREATE TABLE IF NOT EXISTS ch_int_enum ( + tag_name VARCHAR(256) NOT NULL, + value INTEGER DEFAULT 0, + name_zh VARCHAR(256) , + name_en VARCHAR(256) , + description_zh VARCHAR(256) , + description_en VARCHAR(256) , + updated_at DATETIME NOT NULL ON UPDATE CURRENT_TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (tag_name,value) +)ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; + +-- update db_version to latest, remeber update DB_VERSION_EXPECT in migrate/init.go +UPDATE db_version SET version='6.6.1.19'; +-- modify end + diff --git a/server/controller/db/mysql/migrator/schema/version.go b/server/controller/db/mysql/migrator/schema/version.go index 30d31ba92c7..e4363eb03ee 100644 --- a/server/controller/db/mysql/migrator/schema/version.go +++ b/server/controller/db/mysql/migrator/schema/version.go @@ -18,7 +18,7 @@ package schema const ( DB_VERSION_TABLE = "db_version" - DB_VERSION_EXPECTED = "6.6.1.18" + DB_VERSION_EXPECTED = "6.6.1.19" ) const ( diff --git a/server/controller/db/mysql/model/ch_model.go b/server/controller/db/mysql/model/ch_model.go index 185cad37045..37e7fb8bf4e 100644 --- a/server/controller/db/mysql/model/ch_model.go +++ b/server/controller/db/mysql/model/ch_model.go @@ -329,19 +329,23 @@ type ChPodServiceK8sLabels struct { } type ChStringEnum struct { - TagName string `gorm:"primaryKey;column:tag_name;type:varchar(256);default:null" json:"TAG_NAME"` - Value string `gorm:"primaryKey;column:value;type:varchar(256);default:null" json:"VALUE"` - Name string `gorm:"column:name;type:varchar(256);default:null" json:"NAME"` - Description string `gorm:"column:description;type:varchar(256);default:null" json:"DESCRIPTION"` - UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime:now,type:timestamp" json:"UPDATED_AT"` + TagName string `gorm:"primaryKey;column:tag_name;type:varchar(256);default:null" json:"TAG_NAME"` + Value string `gorm:"primaryKey;column:value;type:varchar(256);default:null" json:"VALUE"` + NameZH string `gorm:"column:name_zh;type:varchar(256);default:null" json:"NAME_ZH"` + NameEN string `gorm:"column:name_en;type:varchar(256);default:null" json:"NAME_EN"` + DescriptionZH string `gorm:"column:description_zh;type:varchar(256);default:null" json:"DESCRIPTION_ZH"` + DescriptionEN string `gorm:"column:description_en;type:varchar(256);default:null" json:"DESCRIPTION_EN"` + UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime:now,type:timestamp" json:"UPDATED_AT"` } type ChIntEnum struct { - TagName string `gorm:"primaryKey;column:tag_name;type:varchar(256);default:null" json:"TAG_NAME"` - Value int `gorm:"primaryKey;column:value;type:int;default:0" json:"VALUE"` - Name string `gorm:"column:name;type:varchar(256);default:null" json:"NAME"` - Description string `gorm:"column:description;type:varchar(256);default:null" json:"DESCRIPTION"` - UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime:now,type:timestamp" json:"UPDATED_AT"` + TagName string `gorm:"primaryKey;column:tag_name;type:varchar(256);default:null" json:"TAG_NAME"` + Value int `gorm:"primaryKey;column:value;type:int;default:0" json:"VALUE"` + NameZH string `gorm:"column:name_zh;type:varchar(256);default:null" json:"NAME_ZH"` + NameEN string `gorm:"column:name_en;type:varchar(256);default:null" json:"NAME_EN"` + DescriptionZH string `gorm:"column:description_zh;type:varchar(256);default:null" json:"DESCRIPTION_ZH"` + DescriptionEN string `gorm:"column:description_en;type:varchar(256);default:null" json:"DESCRIPTION_EN"` + UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime:now,type:timestamp" json:"UPDATED_AT"` } type ChNodeType struct { diff --git a/server/controller/http/service/resource/domain.go b/server/controller/http/service/resource/domain.go index 35e8f0f143a..1d9fa59ba0e 100644 --- a/server/controller/http/service/resource/domain.go +++ b/server/controller/http/service/resource/domain.go @@ -872,7 +872,13 @@ func UpdateSubDomain(lcuuid string, db *mysql.DB, userInfo *httpcommon.UserInfo, } // pub to tagrecorder - teamIDInt := int(teamID.(float64)) + teamIDInt := 0 + teamIDFloat64, ok := teamID.(float64) + if ok { + teamIDInt = int(teamIDFloat64) + } else { + teamIDInt = domain.TeamID + } if teamIDInt != subDomain.TeamID { metadata := message.NewMetadata(db.ORGID, message.MetadataSubDomainID(subDomain.ID), message.MetadataTeamID(teamIDInt)) for _, s := range tagrecorder.GetSubscriberManager().GetSubscribers("sub_domain") { diff --git a/server/controller/tagrecorder/ch_int_enum.go b/server/controller/tagrecorder/ch_int_enum.go index d94502b0e89..e57acda6996 100644 --- a/server/controller/tagrecorder/ch_int_enum.go +++ b/server/controller/tagrecorder/ch_int_enum.go @@ -17,12 +17,8 @@ package tagrecorder import ( - "strconv" - "strings" - "github.com/deepflowio/deepflow/server/controller/db/mysql" mysqlmodel "github.com/deepflowio/deepflow/server/controller/db/mysql/model" - "github.com/deepflowio/deepflow/server/querier/config" "github.com/deepflowio/deepflow/server/querier/engine/clickhouse/tag" ) @@ -51,28 +47,29 @@ func (e *ChIntEnum) generateNewData(dbClient *mysql.DB) (map[IntEnumTagKey]mysql } for name, tagValues := range respMap { - tagName := strings.TrimSuffix(name, "."+config.Cfg.Language) for _, valueAndName := range tagValues { tagValue := valueAndName.([]interface{})[0] - tagDisplayName := valueAndName.([]interface{})[1] - tagDescription := valueAndName.([]interface{})[2] - tagValueInt, err := strconv.Atoi(tagValue.(string)) - if err == nil { + tagDisplayNameZH := valueAndName.([]interface{})[1] + tagDisplayNameEN := valueAndName.([]interface{})[2] + tagDescriptionZH := valueAndName.([]interface{})[3] + tagDescriptionEN := valueAndName.([]interface{})[4] + tagValueInt, ok := tagValue.(int) + if ok { key := IntEnumTagKey{ - TagName: tagName, + TagName: name, TagValue: tagValueInt, } keyToItem[key] = mysqlmodel.ChIntEnum{ - TagName: tagName, - Value: tagValueInt, - Name: tagDisplayName.(string), - Description: tagDescription.(string), + TagName: name, + Value: tagValueInt, + NameZH: tagDisplayNameZH.(string), + NameEN: tagDisplayNameEN.(string), + DescriptionZH: tagDescriptionZH.(string), + DescriptionEN: tagDescriptionEN.(string), } } - } } - return keyToItem, true } @@ -88,11 +85,17 @@ func (e *ChIntEnum) generateUpdateInfo(oldItem, newItem mysqlmodel.ChIntEnum) (m if oldItem.Value != newItem.Value { updateInfo["value"] = newItem.Value } - if oldItem.Name != newItem.Name { - updateInfo["name"] = newItem.Name + if oldItem.NameZH != newItem.NameZH { + updateInfo["name_zh"] = newItem.NameZH + } + if oldItem.NameEN != newItem.NameEN { + updateInfo["name_en"] = newItem.NameEN + } + if oldItem.DescriptionZH != newItem.DescriptionZH { + updateInfo["description_zh"] = newItem.DescriptionZH } - if oldItem.Description != newItem.Description { - updateInfo["description"] = newItem.Description + if oldItem.DescriptionEN != newItem.DescriptionEN { + updateInfo["description_en"] = newItem.DescriptionEN } if len(updateInfo) > 0 { return updateInfo, true diff --git a/server/controller/tagrecorder/ch_string_enum.go b/server/controller/tagrecorder/ch_string_enum.go index 487450138d2..9396ee3ba07 100644 --- a/server/controller/tagrecorder/ch_string_enum.go +++ b/server/controller/tagrecorder/ch_string_enum.go @@ -17,11 +17,8 @@ package tagrecorder import ( - "strings" - "github.com/deepflowio/deepflow/server/controller/db/mysql" mysqlmodel "github.com/deepflowio/deepflow/server/controller/db/mysql/model" - "github.com/deepflowio/deepflow/server/querier/config" "github.com/deepflowio/deepflow/server/querier/engine/clickhouse/tag" ) @@ -50,24 +47,26 @@ func (e *ChStringEnum) generateNewData(dbClient *mysql.DB) (map[StringEnumTagKey } for name, tagValues := range respMap { - tagName := strings.TrimSuffix(name, "."+config.Cfg.Language) for _, valueAndName := range tagValues { tagValue := valueAndName.([]interface{})[0] - tagDisplayName := valueAndName.([]interface{})[1] - tagDescription := valueAndName.([]interface{})[2] + tagDisplayNameZH := valueAndName.([]interface{})[1] + tagDisplayNameEN := valueAndName.([]interface{})[2] + tagDescriptionZH := valueAndName.([]interface{})[3] + tagDescriptionEN := valueAndName.([]interface{})[4] key := StringEnumTagKey{ - TagName: tagName, + TagName: name, TagValue: tagValue.(string), } keyToItem[key] = mysqlmodel.ChStringEnum{ - TagName: tagName, - Value: tagValue.(string), - Name: tagDisplayName.(string), - Description: tagDescription.(string), + TagName: name, + Value: tagValue.(string), + NameZH: tagDisplayNameZH.(string), + NameEN: tagDisplayNameEN.(string), + DescriptionZH: tagDescriptionZH.(string), + DescriptionEN: tagDescriptionEN.(string), } } } - return keyToItem, true } @@ -83,11 +82,17 @@ func (e *ChStringEnum) generateUpdateInfo(oldItem, newItem mysqlmodel.ChStringEn if oldItem.Value != newItem.Value { updateInfo["value"] = newItem.Value } - if oldItem.Name != newItem.Name { - updateInfo["name"] = newItem.Name + if oldItem.NameZH != newItem.NameZH { + updateInfo["name_zh"] = newItem.NameZH + } + if oldItem.NameEN != newItem.NameEN { + updateInfo["name_en"] = newItem.NameEN + } + if oldItem.DescriptionZH != newItem.DescriptionZH { + updateInfo["description_zh"] = newItem.DescriptionZH } - if oldItem.Description != newItem.Description { - updateInfo["description"] = newItem.Description + if oldItem.DescriptionEN != newItem.DescriptionEN { + updateInfo["description_en"] = newItem.DescriptionEN } if len(updateInfo) > 0 { return updateInfo, true diff --git a/server/controller/tagrecorder/check/ch_int_enum.go b/server/controller/tagrecorder/check/ch_int_enum.go index a5613af17cd..6e824425d3a 100644 --- a/server/controller/tagrecorder/check/ch_int_enum.go +++ b/server/controller/tagrecorder/check/ch_int_enum.go @@ -17,11 +17,7 @@ package tagrecorder import ( - "strconv" - "strings" - mysqlmodel "github.com/deepflowio/deepflow/server/controller/db/mysql/model" - "github.com/deepflowio/deepflow/server/querier/config" "github.com/deepflowio/deepflow/server/querier/engine/clickhouse/tag" ) @@ -50,25 +46,27 @@ func (e *ChIntEnum) generateNewData() (map[IntEnumTagKey]mysqlmodel.ChIntEnum, b } for name, tagValues := range respMap { - tagName := strings.TrimSuffix(name, "."+config.Cfg.Language) for _, valueAndName := range tagValues { tagValue := valueAndName.([]interface{})[0] - tagDisplayName := valueAndName.([]interface{})[1] - tagDescription := valueAndName.([]interface{})[2] - tagValueInt, err := strconv.Atoi(tagValue.(string)) - if err == nil { + tagDisplayNameZH := valueAndName.([]interface{})[1] + tagDisplayNameEN := valueAndName.([]interface{})[2] + tagDescriptionZH := valueAndName.([]interface{})[3] + tagDescriptionEN := valueAndName.([]interface{})[4] + tagValueInt, ok := tagValue.(int) + if ok { key := IntEnumTagKey{ - TagName: tagName, + TagName: name, TagValue: tagValueInt, } keyToItem[key] = mysqlmodel.ChIntEnum{ - TagName: tagName, - Value: tagValueInt, - Name: tagDisplayName.(string), - Description: tagDescription.(string), + TagName: name, + Value: tagValueInt, + NameZH: tagDisplayNameZH.(string), + NameEN: tagDisplayNameEN.(string), + DescriptionZH: tagDescriptionZH.(string), + DescriptionEN: tagDescriptionEN.(string), } } - } } @@ -87,11 +85,17 @@ func (e *ChIntEnum) generateUpdateInfo(oldItem, newItem mysqlmodel.ChIntEnum) (m if oldItem.Value != newItem.Value { updateInfo["value"] = newItem.Value } - if oldItem.Name != newItem.Name { - updateInfo["name"] = newItem.Name + if oldItem.NameZH != newItem.NameZH { + updateInfo["name_zh"] = newItem.NameZH + } + if oldItem.NameEN != newItem.NameEN { + updateInfo["name_en"] = newItem.NameEN + } + if oldItem.DescriptionZH != newItem.DescriptionZH { + updateInfo["description_zh"] = newItem.DescriptionZH } - if oldItem.Description != newItem.Description { - updateInfo["description"] = newItem.Description + if oldItem.DescriptionEN != newItem.DescriptionEN { + updateInfo["description_en"] = newItem.DescriptionEN } if len(updateInfo) > 0 { return updateInfo, true diff --git a/server/controller/tagrecorder/check/ch_string_enum.go b/server/controller/tagrecorder/check/ch_string_enum.go index a558f0e6ad1..6d67c107761 100644 --- a/server/controller/tagrecorder/check/ch_string_enum.go +++ b/server/controller/tagrecorder/check/ch_string_enum.go @@ -17,10 +17,7 @@ package tagrecorder import ( - "strings" - mysqlmodel "github.com/deepflowio/deepflow/server/controller/db/mysql/model" - "github.com/deepflowio/deepflow/server/querier/config" "github.com/deepflowio/deepflow/server/querier/engine/clickhouse/tag" ) @@ -49,24 +46,26 @@ func (e *ChStringEnum) generateNewData() (map[StringEnumTagKey]mysqlmodel.ChStri } for name, tagValues := range respMap { - tagName := strings.TrimSuffix(name, "."+config.Cfg.Language) for _, valueAndName := range tagValues { tagValue := valueAndName.([]interface{})[0] - tagDisplayName := valueAndName.([]interface{})[1] - tagDescription := valueAndName.([]interface{})[2] + tagDisplayNameZH := valueAndName.([]interface{})[1] + tagDisplayNameEN := valueAndName.([]interface{})[2] + tagDescriptionZH := valueAndName.([]interface{})[3] + tagDescriptionEN := valueAndName.([]interface{})[4] key := StringEnumTagKey{ - TagName: tagName, + TagName: name, TagValue: tagValue.(string), } keyToItem[key] = mysqlmodel.ChStringEnum{ - TagName: tagName, - Value: tagValue.(string), - Name: tagDisplayName.(string), - Description: tagDescription.(string), + TagName: name, + Value: tagValue.(string), + NameZH: tagDisplayNameZH.(string), + NameEN: tagDisplayNameEN.(string), + DescriptionZH: tagDescriptionZH.(string), + DescriptionEN: tagDescriptionEN.(string), } } } - return keyToItem, true } @@ -82,11 +81,17 @@ func (e *ChStringEnum) generateUpdateInfo(oldItem, newItem mysqlmodel.ChStringEn if oldItem.Value != newItem.Value { updateInfo["value"] = newItem.Value } - if oldItem.Name != newItem.Name { - updateInfo["name"] = newItem.Name + if oldItem.NameZH != newItem.NameZH { + updateInfo["name_zh"] = newItem.NameZH + } + if oldItem.NameEN != newItem.NameEN { + updateInfo["name_en"] = newItem.NameEN + } + if oldItem.DescriptionZH != newItem.DescriptionZH { + updateInfo["description_zh"] = newItem.DescriptionZH } - if oldItem.Description != newItem.Description { - updateInfo["description"] = newItem.Description + if oldItem.DescriptionEN != newItem.DescriptionEN { + updateInfo["description_en"] = newItem.DescriptionEN } if len(updateInfo) > 0 { return updateInfo, true diff --git a/server/controller/tagrecorder/check/const.go b/server/controller/tagrecorder/check/const.go index 8233de1246b..2307d10c6ea 100644 --- a/server/controller/tagrecorder/check/const.go +++ b/server/controller/tagrecorder/check/const.go @@ -439,8 +439,10 @@ const ( "(\n" + " `tag_name` String,\n" + " `value` String,\n" + - " `name` String,\n" + - " `description` String\n" + + " `name_zh` String,\n" + + " `name_en` String,\n" + + " `description_zh` String,\n" + + " `description_en` String\n" + ")\n" + "PRIMARY KEY tag_name, value\n" + "SOURCE(MYSQL(PORT %s USER '%s' PASSWORD '%s' %s DB %s TABLE %s INVALIDATE_QUERY 'select(select updated_at from %s order by updated_at desc limit 1) as updated_at'))\n" + @@ -450,8 +452,10 @@ const ( "(\n" + " `tag_name` String,\n" + " `value` UInt64,\n" + - " `name` String,\n" + - " `description` String\n" + + " `name_zh` String,\n" + + " `name_en` String,\n" + + " `description_zh` String,\n" + + " `description_en` String\n" + ")\n" + "PRIMARY KEY tag_name, value\n" + "SOURCE(MYSQL(PORT %s USER '%s' PASSWORD '%s' %s DB %s TABLE %s INVALIDATE_QUERY 'select(select updated_at from %s order by updated_at desc limit 1) as updated_at'))\n" + diff --git a/server/controller/tagrecorder/const.go b/server/controller/tagrecorder/const.go index 6f91ed2a28e..47779cc8dd6 100644 --- a/server/controller/tagrecorder/const.go +++ b/server/controller/tagrecorder/const.go @@ -549,8 +549,10 @@ const ( "(\n" + " `tag_name` String,\n" + " `value` String,\n" + - " `name` String,\n" + - " `description` String\n" + + " `name_zh` String,\n" + + " `name_en` String,\n" + + " `description_zh` String,\n" + + " `description_en` String\n" + ")\n" + "PRIMARY KEY tag_name, value\n" + SQL_SOURCE_MYSQL + @@ -560,8 +562,10 @@ const ( "(\n" + " `tag_name` String,\n" + " `value` UInt64,\n" + - " `name` String,\n" + - " `description` String\n" + + " `name_zh` String,\n" + + " `name_en` String,\n" + + " `description_zh` String,\n" + + " `description_en` String\n" + ")\n" + "PRIMARY KEY tag_name, value\n" + SQL_SOURCE_MYSQL + diff --git a/server/querier/app/prometheus/service/converters.go b/server/querier/app/prometheus/service/converters.go index efcc4b06e21..2fffd7836c4 100644 --- a/server/querier/app/prometheus/service/converters.go +++ b/server/querier/app/prometheus/service/converters.go @@ -1422,10 +1422,6 @@ func formatEnumTag(tagName string) (string, bool) { enumFile := strings.TrimSuffix(tagName, "_0") enumFile = strings.TrimSuffix(enumFile, "_1") _, exists := tagdescription.TAG_ENUMS[enumFile] - if !exists { - enumFile = fmt.Sprintf("%s.%s", enumFile, config.Cfg.Language) - _, exists = tagdescription.TAG_ENUMS[enumFile] - } if exists { return fmt.Sprintf("Enum(%s)", tagName), exists } diff --git a/server/querier/app/prometheus/service/converters_test.go b/server/querier/app/prometheus/service/converters_test.go index 0c29e3bc5d4..711f1f30ac9 100644 --- a/server/querier/app/prometheus/service/converters_test.go +++ b/server/querier/app/prometheus/service/converters_test.go @@ -422,9 +422,9 @@ func TestParsePromQLTag(t *testing.T) { orgID: "1", } p.getExternalTagFromCache = executor.convertExternalTagToQuerierAllowTag - tagdescription.TAG_ENUMS["l7_protocol"] = []*tagdescription.TagEnum{{Value: 20, DisplayName: "HTTP"}} - tagdescription.TAG_ENUMS["auto_service_type.ch"] = []*tagdescription.TagEnum{{Value: 1, DisplayName: "云服务器"}} - tagdescription.TAG_ENUMS["auto_service_type.en"] = []*tagdescription.TagEnum{{Value: 1, DisplayName: "Cloud Host"}} + tagdescription.TAG_ENUMS["l7_protocol"] = []*tagdescription.TagEnum{{Value: 20, DisplayNameEN: "HTTP"}} + tagdescription.TAG_ENUMS["auto_service_type.ch"] = []*tagdescription.TagEnum{{Value: 1, DisplayNameZH: "云服务器"}} + tagdescription.TAG_ENUMS["auto_service_type.en"] = []*tagdescription.TagEnum{{Value: 1, DisplayNameEN: "Cloud Host"}} // Test cases generation // Test cases for prefixType prefixTag diff --git a/server/querier/db_descriptions/clickhouse/tag/enum/instance_type.ch b/server/querier/db_descriptions/clickhouse/tag/enum/instance_type.ch deleted file mode 100644 index caa3a149fa4..00000000000 --- a/server/querier/db_descriptions/clickhouse/tag/enum/instance_type.ch +++ /dev/null @@ -1,13 +0,0 @@ -# Value , DisplayName , Description -1 , 云服务器 , -5 , 路由器 , -6 , 宿主机 , -9 , DHCP网关 , -10 , 容器POD , -11 , 容器服务 , -12 , Redis , -13 , RDS , -14 , 容器节点 , -15 , 负载均衡器 , -16 , NAT网关 , -120 , 进程 , diff --git a/server/querier/db_descriptions/clickhouse/tag/enum/instance_type.en b/server/querier/db_descriptions/clickhouse/tag/enum/instance_type.en deleted file mode 100644 index 7d31021e6e2..00000000000 --- a/server/querier/db_descriptions/clickhouse/tag/enum/instance_type.en +++ /dev/null @@ -1,13 +0,0 @@ -# Value , DisplayName , Description -1 , Cloud Host , -5 , Router , -6 , VM Hypervisor , -9 , DHCP Gateway , -10 , K8s POD , -11 , K8s Service , -12 , Redis , -13 , RDS , -14 , K8s Node , -15 , Load Balancer , -16 , NAT Gateway , -120 , Process , diff --git a/server/querier/db_descriptions/clickhouse/tag/enum/resource_gl2_type.en b/server/querier/db_descriptions/clickhouse/tag/enum/resource_gl2_type.en index a89345187e0..8973702604f 100644 --- a/server/querier/db_descriptions/clickhouse/tag/enum/resource_gl2_type.en +++ b/server/querier/db_descriptions/clickhouse/tag/enum/resource_gl2_type.en @@ -14,4 +14,3 @@ 102 , Service , 120 , Process , 255 , IP , - diff --git a/server/querier/engine/clickhouse/clickhouse.go b/server/querier/engine/clickhouse/clickhouse.go index 0669d046a9b..8dfd7225baf 100644 --- a/server/querier/engine/clickhouse/clickhouse.go +++ b/server/querier/engine/clickhouse/clickhouse.go @@ -103,6 +103,7 @@ type CHEngine struct { IsDerivative bool DerivativeGroupBy []string ORGID string + Language string } func init() { @@ -119,6 +120,7 @@ func (e *CHEngine) ExecuteQuery(args *common.QuerierParams) (*common.Result, map sql := args.Sql e.Context = args.Context e.NoPreWhere = args.NoPreWhere + e.Language = args.Language e.ORGID = common.DEFAULT_ORG_ID if args.ORGID != "" { e.ORGID = args.ORGID @@ -319,7 +321,7 @@ func ShowTagTypeMetrics(tagDescriptions, result *common.Result, db, table string } } else { nameMetric := []interface{}{ - name, true, displayName, displayName, displayName, "", "", "", metrics.METRICS_TYPE_NAME_MAP["tag"], + name, true, displayName, displayNameZH, displayNameEN, "", "", "", metrics.METRICS_TYPE_NAME_MAP["tag"], "Tag", metrics.METRICS_OPERATORS, permissions, table, "", "", "", } result.Values = append(result.Values, nameMetric) @@ -402,6 +404,22 @@ func formatMetricByLanguage(language string, values []interface{}) { } } +func formatEnumTagByLanguage(language string, values []interface{}) { + for _, value := range values { + displaynameZH := value.([]interface{})[4].(string) + displaynameEN := value.([]interface{})[5].(string) + descriptionZH := value.([]interface{})[11].(string) + descriptionEN := value.([]interface{})[12].(string) + if language == chCommon.LanguageEN { + value.([]interface{})[3] = displaynameEN + value.([]interface{})[10] = descriptionEN + } else { + value.([]interface{})[3] = displaynameZH + value.([]interface{})[10] = descriptionZH + } + } +} + func (e *CHEngine) ParseShowSql(sql string, args *common.QuerierParams, DebugInfo *client.DebugInfo) (*common.Result, []string, bool, error) { var visibilityFilterRegexp *regexp.Regexp sqlSplit := strings.Fields(sql) @@ -475,7 +493,7 @@ func (e *CHEngine) ParseShowSql(sql string, args *common.QuerierParams, DebugInf } return result, []string{}, true, err case 4: // show tag X values from Y X, Y not nil - result, sqlList, err := tagdescription.GetTagValues(e.DB, table, sql, args.QueryCacheTTL, args.ORGID, args.UseQueryCache) + result, sqlList, err := tagdescription.GetTagValues(e.DB, table, sql, args.QueryCacheTTL, args.ORGID, args.Language, args.UseQueryCache) e.DB = "flow_tag" return result, sqlList, true, err case 5: // show tags ... @@ -510,9 +528,12 @@ func (e *CHEngine) ParseShowSql(sql string, args *common.QuerierParams, DebugInf return nil, sqlList, true, err case 9: result, err := tagdescription.GetEnumTags(e.DB, table, sql) + if args.Language != "" { + formatTagByLanguage(args.Language, result.Values) + } return result, []string{}, true, err case 10: - sqlList, err := tagdescription.GetEnumTagAllValues(e.DB, table, sql) + sqlList, err := tagdescription.GetEnumTagAllValues(e.DB, table, sql, args.Language) return nil, sqlList, true, err } return nil, []string{}, true, fmt.Errorf("parse show sql error, sql: '%s' not support", sql) @@ -1819,7 +1840,7 @@ func (e *CHEngine) parseWhere(node sqlparser.Expr, w *Where, isCheck bool) (view } outfunc := function.Trans(e.Model) stmt := &WhereFunction{Function: outfunc, Value: sqlparser.String(node.Right)} - return stmt.Trans(node, w, e.AsTagMap, e.DB, e.Table) + return stmt.Trans(node, w, e) } case *sqlparser.FuncExpr: args := []string{} diff --git a/server/querier/engine/clickhouse/clickhouse_test.go b/server/querier/engine/clickhouse/clickhouse_test.go index 7763cef0073..5c5b7ab2018 100644 --- a/server/querier/engine/clickhouse/clickhouse_test.go +++ b/server/querier/engine/clickhouse/clickhouse_test.go @@ -213,7 +213,7 @@ var ( output: []string{"SELECT SUM(byte_tx) AS `max_byte` FROM flow_log.`l4_flow_log` ORDER BY length(tap_side) desc,`length(tap_side)` asc LIMIT 10000"}, }, { input: "select Enum(tap_side) from l7_flow_log limit 0, 50", - output: []string{"WITH dictGetOrDefault('flow_tag.string_enum_map', 'name', ('observation_point',observation_point), observation_point) AS `Enum(tap_side)` SELECT `Enum(tap_side)` FROM flow_log.`l7_flow_log` LIMIT 0, 50"}, + output: []string{"WITH dictGetOrDefault('flow_tag.string_enum_map', 'name_en', ('observation_point',observation_point), observation_point) AS `Enum(tap_side)` SELECT `Enum(tap_side)` FROM flow_log.`l7_flow_log` LIMIT 0, 50"}, }, { input: "select AAvg(`byte_tx`) AS `AAvg(byte_tx)`,icon_id(chost_0) as `xx`,region_0 from vtap_flow_edge_port where `time` >= 60 AND `time` <= 180 group by region_0 limit 1", output: []string{"SELECT `xx`, region_0, AVG(`_sum_byte_tx`) AS `AAvg(byte_tx)` FROM (WITH if(l3_device_type_0=1, dictGet('flow_tag.device_map', 'icon_id', (toUInt64(1),toUInt64(l3_device_id_0))), 0) AS `xx` SELECT `xx`, dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_0))) AS `region_0`, SUM(byte_tx) AS `_sum_byte_tx` FROM flow_metrics.`network_map` WHERE `time` >= 60 AND `time` <= 180 GROUP BY `xx`, dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_0))) AS `region_0`) GROUP BY `xx`, `region_0` LIMIT 1"}, @@ -228,10 +228,10 @@ var ( db: "flow_metrics", }, { input: "select request from l7_flow_log where Enum(tap_side)='xxx' limit 0, 50", - output: []string{"SELECT if(type IN [0, 2],1,0) AS `request` FROM flow_log.`l7_flow_log` PREWHERE (observation_point GLOBAL IN (SELECT value FROM flow_tag.string_enum_map WHERE name = 'xxx' and tag_name='observation_point') OR observation_point = 'xxx') LIMIT 0, 50"}, + output: []string{"SELECT if(type IN [0, 2],1,0) AS `request` FROM flow_log.`l7_flow_log` PREWHERE (observation_point GLOBAL IN (SELECT value FROM flow_tag.string_enum_map WHERE name_en = 'xxx' and tag_name='observation_point') OR observation_point = 'xxx') LIMIT 0, 50"}, }, { input: "select request from l7_flow_log where Enum(tap_side) like 'xxx' limit 0, 50", - output: []string{"SELECT if(type IN [0, 2],1,0) AS `request` FROM flow_log.`l7_flow_log` PREWHERE (observation_point GLOBAL IN (SELECT value FROM flow_tag.string_enum_map WHERE name ilike 'xxx' and tag_name='observation_point')) LIMIT 0, 50"}, + output: []string{"SELECT if(type IN [0, 2],1,0) AS `request` FROM flow_log.`l7_flow_log` PREWHERE (observation_point GLOBAL IN (SELECT value FROM flow_tag.string_enum_map WHERE name_en ilike 'xxx' and tag_name='observation_point')) LIMIT 0, 50"}, }, { input: "select Histogram(Sum(byte),10) AS histo from l4_flow_log", output: []string{"SELECT histogramIf(10)(assumeNotNull(`_sum_byte_tx+byte_rx`),`_sum_byte_tx+byte_rx`>0) AS `histo` FROM (SELECT SUM(byte_tx+byte_rx) AS `_sum_byte_tx+byte_rx` FROM flow_log.`l4_flow_log` LIMIT 10000)"}, @@ -317,7 +317,7 @@ var ( output: []string{"WITH if(l3_epc_id_0=-2,dictGet('flow_tag.device_map', 'icon_id', (toUInt64(63999),toUInt64(63999))),0) AS `client_icon_id` SELECT if(l3_epc_id_0=-2,'internet','') AS `client_node_type`, `client_icon_id`, if(l3_epc_id_1=-2,'internet','') AS `server_node_type`, if(l3_epc_id_0=-2,1,0) AS `is_internet_0`, if(l3_epc_id_1=-2,1,0) AS `is_internet_1` FROM flow_log.`l4_flow_log` GROUP BY `client_node_type`, `client_icon_id`, `server_node_type`, if(l3_epc_id_0=-2,1,0) AS `is_internet_0`, if(l3_epc_id_1=-2,1,0) AS `is_internet_1` LIMIT 1"}, }, { input: "select Enum(pod_group_type_0) ,pod_group_type_0 from l7_flow_log where Enum(pod_group_type_0)!='Deployment' limit 10", - output: []string{"WITH dictGetOrDefault('flow_tag.int_enum_map', 'name', ('pod_group_type',toUInt64(dictGet('flow_tag.pod_group_map', 'pod_group_type', (toUInt64(pod_group_id_0))))), dictGet('flow_tag.pod_group_map', 'pod_group_type', (toUInt64(pod_group_id_0)))) AS `Enum(pod_group_type_0)` SELECT `Enum(pod_group_type_0)`, dictGet('flow_tag.pod_group_map', 'pod_group_type', (toUInt64(pod_group_id_0))) AS `pod_group_type_0` FROM flow_log.`l7_flow_log` PREWHERE (not(toUInt64(dictGet('flow_tag.pod_group_map', 'pod_group_type', (toUInt64(pod_group_id_0)))) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE name = 'Deployment' and tag_name='pod_group_type') AND pod_group_id_0!=0)) LIMIT 10"}, + output: []string{"WITH dictGetOrDefault('flow_tag.int_enum_map', 'name_en', ('pod_group_type',toUInt64(dictGet('flow_tag.pod_group_map', 'pod_group_type', (toUInt64(pod_group_id_0))))), dictGet('flow_tag.pod_group_map', 'pod_group_type', (toUInt64(pod_group_id_0)))) AS `Enum(pod_group_type_0)` SELECT `Enum(pod_group_type_0)`, dictGet('flow_tag.pod_group_map', 'pod_group_type', (toUInt64(pod_group_id_0))) AS `pod_group_type_0` FROM flow_log.`l7_flow_log` PREWHERE (not(toUInt64(dictGet('flow_tag.pod_group_map', 'pod_group_type', (toUInt64(pod_group_id_0)))) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE name_en = 'Deployment' and tag_name='pod_group_type') AND pod_group_id_0!=0)) LIMIT 10"}, }, { name: "count_1", input: "select Count(row) as a from l7_flow_log having a > 0 ", @@ -350,7 +350,7 @@ var ( name: "topk_enum", db: "flow_log", input: "select TopK(protocol,2) from l4_flow_log limit 2", - output: []string{"SELECT arrayStringConcat(topKIf(2)(dictGetOrDefault('flow_tag.int_enum_map', 'name', ('protocol',toUInt64(protocol)), protocol), dictGetOrDefault('flow_tag.int_enum_map', 'name', ('protocol',toUInt64(protocol)), protocol) != ''), ',') AS `TopK_2(protocol)` FROM flow_log.`l4_flow_log` LIMIT 2"}, + output: []string{"SELECT arrayStringConcat(topKIf(2)(dictGetOrDefault('flow_tag.int_enum_map', 'name_en', ('protocol',toUInt64(protocol)), protocol), dictGetOrDefault('flow_tag.int_enum_map', 'name_en', ('protocol',toUInt64(protocol)), protocol) != ''), ',') AS `TopK_2(protocol)` FROM flow_log.`l4_flow_log` LIMIT 2"}, }, { name: "select_enum", db: "flow_log", @@ -459,7 +459,7 @@ var ( db: "flow_metrics", datasource: "1m", input: "WITH query1 AS (SELECT PerSecond(Avg(`request`)) AS `请求速率`, Avg(`server_error_ratio`) AS `服务端异常比例`, Avg(`rrt`) AS `响应时延`, node_type(region_0) AS `client_node_type`, icon_id(region_0) AS `client_icon_id`, region_id_0, region_0, Enum(tap_side), tap_side, is_internet_0, node_type(region_1) AS `server_node_type`, icon_id(region_1) AS `server_icon_id`, region_id_1, region_1, is_internet_1 FROM vtap_app_edge_port WHERE time>=1704338640 AND time<=1704339600 GROUP BY region_0, tap_side, is_internet_0, region_id_0, `client_node_type`, region_1, is_internet_1, region_id_1, `server_node_type` ORDER BY `请求速率` DESC LIMIT 50 OFFSET 0), query2 AS (SELECT Avg(`packet_tx`) AS `Avg(发送包数)`, node_type(region_0) AS `client_node_type`, icon_id(region_0) AS `client_icon_id`, region_id_0, region_0, Enum(tap_side), tap_side, is_internet_0, node_type(region_1) AS `server_node_type`, icon_id(region_1) AS `server_icon_id`, region_id_1, region_1, is_internet_1 FROM vtap_flow_edge_port WHERE time>=1704338640 AND time<=1704339600 GROUP BY region_0, tap_side, is_internet_0, region_id_0, `client_node_type`, region_1, is_internet_1, region_id_1, `server_node_type` LIMIT 50) SELECT query1.`请求速率` AS `请求速率`, query1.`服务端异常比例` AS `服务端异常比例`, query1.`响应时延` AS `响应时延`, query1.`client_node_type` AS `client_node_type`, query1.`client_icon_id` AS `client_icon_id`, query1.`region_id_0` AS `region_id_0`, query1.`region_0` AS `region_0`, query1.`Enum(tap_side)` AS `Enum(tap_side)`, query1.`tap_side` AS `tap_side`, query1.`is_internet_0` AS `is_internet_0`, query1.`server_node_type` AS `server_node_type`, query1.`server_icon_id` AS `server_icon_id`, query1.`region_id_1` AS `region_id_1`, query1.`region_1` AS `region_1`, query1.`is_internet_1` AS `is_internet_1`, query2.`Avg(发送包数)` AS `Avg(发送包数)` FROM query1 LEFT JOIN query2 ON query1.`region_0` = query2.`region_0` AND query1.`tap_side` = query2.`tap_side` AND query1.`is_internet_0` = query2.`is_internet_0` AND query1.`region_id_0` = query2.`region_id_0` AND query1.`client_node_type` = query2.`client_node_type` AND query1.`region_1` = query2.`region_1` AND query1.`is_internet_1` = query2.`is_internet_1` AND query1.`region_id_1` = query2.`region_id_1` AND query1.`server_node_type` = query2.`server_node_type`", - output: []string{"WITH query1 AS (WITH dictGet('flow_tag.region_map', 'icon_id', (toUInt64(region_id_0))) AS `client_icon_id`, dictGetOrDefault('flow_tag.string_enum_map', 'name', ('observation_point',observation_point), observation_point) AS `Enum(tap_side)`, dictGet('flow_tag.region_map', 'icon_id', (toUInt64(region_id_1))) AS `server_icon_id`, if(SUMIf(response, response>0)>0, divide(SUM(server_error), SUMIf(response, response>0)), null) AS `divide_0diveider_as_null_sum_server_error_sum_response_response>0`, if(SUMIf(rrt_count, rrt_count>0)>0, divide(SUM(rrt_sum), SUMIf(rrt_count, rrt_count>0)), null) AS `divide_0diveider_as_null_sum_rrt_sum_sum_rrt_count_rrt_count>0` SELECT 'region' AS `client_node_type`, `client_icon_id`, region_id_0, dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_0))) AS `region_0`, `Enum(tap_side)`, observation_point AS `tap_side`, if(l3_epc_id_0=-2,1,0) AS `is_internet_0`, 'region' AS `server_node_type`, `server_icon_id`, region_id_1, dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_1))) AS `region_1`, if(l3_epc_id_1=-2,1,0) AS `is_internet_1`, divide(sum(request)/(1020/60), 60) AS `请求速率`, `divide_0diveider_as_null_sum_server_error_sum_response_response>0`*100 AS `服务端异常比例`, `divide_0diveider_as_null_sum_rrt_sum_sum_rrt_count_rrt_count>0` AS `响应时延` FROM flow_metrics.`application_map.1m` PREWHERE `time` >= 1704338640 AND `time` <= 1704339600 GROUP BY dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_0))) AS `region_0`, observation_point AS `tap_side`, if(l3_epc_id_0=-2,1,0) AS `is_internet_0`, `region_id_0`, `client_node_type`, `client_icon_id`, dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_1))) AS `region_1`, if(l3_epc_id_1=-2,1,0) AS `is_internet_1`, `region_id_1`, `server_node_type`, `server_icon_id` ORDER BY `请求速率` desc LIMIT 0, 50), query2 AS (WITH dictGet('flow_tag.region_map', 'icon_id', (toUInt64(region_id_0))) AS `client_icon_id`, dictGetOrDefault('flow_tag.string_enum_map', 'name', ('observation_point',observation_point), observation_point) AS `Enum(tap_side)`, dictGet('flow_tag.region_map', 'icon_id', (toUInt64(region_id_1))) AS `server_icon_id` SELECT 'region' AS `client_node_type`, `client_icon_id`, region_id_0, dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_0))) AS `region_0`, `Enum(tap_side)`, observation_point AS `tap_side`, if(l3_epc_id_0=-2,1,0) AS `is_internet_0`, 'region' AS `server_node_type`, `server_icon_id`, region_id_1, dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_1))) AS `region_1`, if(l3_epc_id_1=-2,1,0) AS `is_internet_1`, sum(packet_tx)/(1020/60) AS `Avg(发送包数)` FROM flow_metrics.`network_map.1m` PREWHERE `time` >= 1704338640 AND `time` <= 1704339600 GROUP BY dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_0))) AS `region_0`, observation_point AS `tap_side`, if(l3_epc_id_0=-2,1,0) AS `is_internet_0`, `region_id_0`, `client_node_type`, `client_icon_id`, dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_1))) AS `region_1`, if(l3_epc_id_1=-2,1,0) AS `is_internet_1`, `region_id_1`, `server_node_type`, `server_icon_id` LIMIT 50) SELECT query1.`请求速率` AS `请求速率`, query1.`服务端异常比例` AS `服务端异常比例`, query1.`响应时延` AS `响应时延`, query1.`client_node_type` AS `client_node_type`, query1.`client_icon_id` AS `client_icon_id`, query1.`region_id_0` AS `region_id_0`, query1.`region_0` AS `region_0`, query1.`Enum(tap_side)` AS `Enum(tap_side)`, query1.`tap_side` AS `tap_side`, query1.`is_internet_0` AS `is_internet_0`, query1.`server_node_type` AS `server_node_type`, query1.`server_icon_id` AS `server_icon_id`, query1.`region_id_1` AS `region_id_1`, query1.`region_1` AS `region_1`, query1.`is_internet_1` AS `is_internet_1`, query2.`Avg(发送包数)` AS `Avg(发送包数)` FROM query1 LEFT JOIN query2 ON query1.`region_0` = query2.`region_0` AND query1.`tap_side` = query2.`tap_side` AND query1.`is_internet_0` = query2.`is_internet_0` AND query1.`region_id_0` = query2.`region_id_0` AND query1.`client_node_type` = query2.`client_node_type` AND query1.`region_1` = query2.`region_1` AND query1.`is_internet_1` = query2.`is_internet_1` AND query1.`region_id_1` = query2.`region_id_1` AND query1.`server_node_type` = query2.`server_node_type`"}, + output: []string{"WITH query1 AS (WITH dictGet('flow_tag.region_map', 'icon_id', (toUInt64(region_id_0))) AS `client_icon_id`, dictGetOrDefault('flow_tag.string_enum_map', 'name_en', ('observation_point',observation_point), observation_point) AS `Enum(tap_side)`, dictGet('flow_tag.region_map', 'icon_id', (toUInt64(region_id_1))) AS `server_icon_id`, if(SUMIf(response, response>0)>0, divide(SUM(server_error), SUMIf(response, response>0)), null) AS `divide_0diveider_as_null_sum_server_error_sum_response_response>0`, if(SUMIf(rrt_count, rrt_count>0)>0, divide(SUM(rrt_sum), SUMIf(rrt_count, rrt_count>0)), null) AS `divide_0diveider_as_null_sum_rrt_sum_sum_rrt_count_rrt_count>0` SELECT 'region' AS `client_node_type`, `client_icon_id`, region_id_0, dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_0))) AS `region_0`, `Enum(tap_side)`, observation_point AS `tap_side`, if(l3_epc_id_0=-2,1,0) AS `is_internet_0`, 'region' AS `server_node_type`, `server_icon_id`, region_id_1, dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_1))) AS `region_1`, if(l3_epc_id_1=-2,1,0) AS `is_internet_1`, divide(sum(request)/(1020/60), 60) AS `请求速率`, `divide_0diveider_as_null_sum_server_error_sum_response_response>0`*100 AS `服务端异常比例`, `divide_0diveider_as_null_sum_rrt_sum_sum_rrt_count_rrt_count>0` AS `响应时延` FROM flow_metrics.`application_map.1m` PREWHERE `time` >= 1704338640 AND `time` <= 1704339600 GROUP BY dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_0))) AS `region_0`, observation_point AS `tap_side`, if(l3_epc_id_0=-2,1,0) AS `is_internet_0`, `region_id_0`, `client_node_type`, `client_icon_id`, dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_1))) AS `region_1`, if(l3_epc_id_1=-2,1,0) AS `is_internet_1`, `region_id_1`, `server_node_type`, `server_icon_id` ORDER BY `请求速率` desc LIMIT 0, 50), query2 AS (WITH dictGet('flow_tag.region_map', 'icon_id', (toUInt64(region_id_0))) AS `client_icon_id`, dictGetOrDefault('flow_tag.string_enum_map', 'name_en', ('observation_point',observation_point), observation_point) AS `Enum(tap_side)`, dictGet('flow_tag.region_map', 'icon_id', (toUInt64(region_id_1))) AS `server_icon_id` SELECT 'region' AS `client_node_type`, `client_icon_id`, region_id_0, dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_0))) AS `region_0`, `Enum(tap_side)`, observation_point AS `tap_side`, if(l3_epc_id_0=-2,1,0) AS `is_internet_0`, 'region' AS `server_node_type`, `server_icon_id`, region_id_1, dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_1))) AS `region_1`, if(l3_epc_id_1=-2,1,0) AS `is_internet_1`, sum(packet_tx)/(1020/60) AS `Avg(发送包数)` FROM flow_metrics.`network_map.1m` PREWHERE `time` >= 1704338640 AND `time` <= 1704339600 GROUP BY dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_0))) AS `region_0`, observation_point AS `tap_side`, if(l3_epc_id_0=-2,1,0) AS `is_internet_0`, `region_id_0`, `client_node_type`, `client_icon_id`, dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_1))) AS `region_1`, if(l3_epc_id_1=-2,1,0) AS `is_internet_1`, `region_id_1`, `server_node_type`, `server_icon_id` LIMIT 50) SELECT query1.`请求速率` AS `请求速率`, query1.`服务端异常比例` AS `服务端异常比例`, query1.`响应时延` AS `响应时延`, query1.`client_node_type` AS `client_node_type`, query1.`client_icon_id` AS `client_icon_id`, query1.`region_id_0` AS `region_id_0`, query1.`region_0` AS `region_0`, query1.`Enum(tap_side)` AS `Enum(tap_side)`, query1.`tap_side` AS `tap_side`, query1.`is_internet_0` AS `is_internet_0`, query1.`server_node_type` AS `server_node_type`, query1.`server_icon_id` AS `server_icon_id`, query1.`region_id_1` AS `region_id_1`, query1.`region_1` AS `region_1`, query1.`is_internet_1` AS `is_internet_1`, query2.`Avg(发送包数)` AS `Avg(发送包数)` FROM query1 LEFT JOIN query2 ON query1.`region_0` = query2.`region_0` AND query1.`tap_side` = query2.`tap_side` AND query1.`is_internet_0` = query2.`is_internet_0` AND query1.`region_id_0` = query2.`region_id_0` AND query1.`client_node_type` = query2.`client_node_type` AND query1.`region_1` = query2.`region_1` AND query1.`is_internet_1` = query2.`is_internet_1` AND query1.`region_id_1` = query2.`region_id_1` AND query1.`server_node_type` = query2.`server_node_type`"}, }, { name: "test_slimit", db: "flow_metrics", @@ -616,7 +616,8 @@ func TestGetSql(t *testing.T) { if db == "" { db = "flow_log" } - e := CHEngine{DB: db} + // test language en + e := CHEngine{DB: db, Language: "en"} if pcase.datasource != "" { e.DataSource = pcase.datasource } diff --git a/server/querier/engine/clickhouse/filter.go b/server/querier/engine/clickhouse/filter.go index bac20dd08af..5374e25edb2 100644 --- a/server/querier/engine/clickhouse/filter.go +++ b/server/querier/engine/clickhouse/filter.go @@ -276,7 +276,7 @@ func TransIngressFilter(translator, regexpTranslator, op, value string) (filter return } -// The tag is not in the tagResourceMap +// The tag is not in the tagResourceMap default func TransTagFilter(whereTag, postAsTag, value, op, db, table, originFilter string, isRemoteRead bool, e *CHEngine) (filter string, err error) { tagItem := &tag.Tag{} ok := false @@ -423,17 +423,6 @@ func TransTagFilter(whereTag, postAsTag, value, op, db, table, originFilter stri filter = fmt.Sprintf(tagItem.WhereTranslator, nameNoPreffix, op, value) } } - } else if strings.HasPrefix(tagName, "Enum(") { - tagName = strings.TrimPrefix(tagName, "Enum(") - tagName = strings.TrimSuffix(tagName, ")") - tagItem, ok = tag.GetTag(tagName, db, table, "enum") - if ok { - if strings.Contains(op, "match") { - filter = fmt.Sprintf(tagItem.WhereRegexpTranslator, op, value) - } else { - filter = fmt.Sprintf(tagItem.WhereTranslator, op, value) - } - } } else { if strings.Contains(op, "match") { filter = fmt.Sprintf("%s(%s,%s)", op, postAsTag, value) @@ -558,10 +547,6 @@ func (t *WhereTag) Trans(expr sqlparser.Expr, w *Where, e *CHEngine) (view.Node, switch table { case "int_enum_map", "string_enum_map": tagItem, ok := tag.GetTag("enum_tag_id", db, table, "default") - tagName := strings.Trim(t.Tag, "`") - if strings.HasPrefix(tagName, "enum(") { - tagItem, ok = tag.GetTag("enum_tag_name", db, table, "default") - } if ok { switch strings.ToLower(op) { case "match": @@ -1317,7 +1302,10 @@ type WhereFunction struct { Value string } -func (f *WhereFunction) Trans(expr sqlparser.Expr, w *Where, asTagMap map[string]string, db, table string) (view.Node, error) { +func (f *WhereFunction) Trans(expr sqlparser.Expr, w *Where, e *CHEngine) (view.Node, error) { + db := e.DB + table := e.Table + language := e.Language opName := expr.(*sqlparser.ComparisonExpr).Operator op, opType := view.GetOperator(expr.(*sqlparser.ComparisonExpr).Operator) right := view.Expr{Value: ""} @@ -1331,6 +1319,18 @@ func (f *WhereFunction) Trans(expr sqlparser.Expr, w *Where, asTagMap map[string tagName = strings.Trim(tagName, "`") tagEnum := strings.TrimSuffix(tagName, "_0") tagEnum = strings.TrimSuffix(tagEnum, "_1") + nameColumn := "" + if language != "" { + nameColumn = "name_" + language + } else { + cfgLang := "" + if config.Cfg.Language == "en" { + cfgLang = "en" + } else { + cfgLang = "zh" + } + nameColumn = "name_" + cfgLang + } if db == "flow_tag" { if strings.ToLower(opName) == "like" || strings.ToLower(opName) == "not like" { f.Value = strings.ReplaceAll(f.Value, "*", "%") @@ -1369,17 +1369,17 @@ func (f *WhereFunction) Trans(expr sqlparser.Expr, w *Where, asTagMap map[string if ok { switch strings.ToLower(opName) { case "match": - filter = fmt.Sprintf(tagItem.WhereRegexpTranslator, "match", f.Value) + filter = fmt.Sprintf(tagItem.WhereRegexpTranslator, "match", nameColumn, f.Value) case "not match": - filter = "not(" + fmt.Sprintf(tagItem.WhereRegexpTranslator, "match", f.Value) + ")" + filter = "not(" + fmt.Sprintf(tagItem.WhereRegexpTranslator, "match", nameColumn, f.Value) + ")" case "not ilike": - filter = "not(" + fmt.Sprintf(tagItem.WhereTranslator, "ilike", f.Value) + ")" + filter = "not(" + fmt.Sprintf(tagItem.WhereTranslator, nameColumn, "ilike", f.Value) + ")" case "not in": - filter = "not(" + fmt.Sprintf(tagItem.WhereTranslator, "in", f.Value) + ")" + filter = "not(" + fmt.Sprintf(tagItem.WhereTranslator, nameColumn, "in", f.Value) + ")" case "!=": - filter = "not(" + fmt.Sprintf(tagItem.WhereTranslator, "=", f.Value) + ")" + filter = "not(" + fmt.Sprintf(tagItem.WhereTranslator, nameColumn, "=", f.Value) + ")" default: - filter = fmt.Sprintf(tagItem.WhereTranslator, opName, f.Value) + filter = fmt.Sprintf(tagItem.WhereTranslator, nameColumn, opName, f.Value) } } return &view.Expr{Value: "(" + filter + ")"}, nil @@ -1435,7 +1435,7 @@ func (f *WhereFunction) Trans(expr sqlparser.Expr, w *Where, asTagMap map[string } return &view.Expr{Value: "(" + whereFilter + ")"}, nil } - enumFileName := strings.TrimSuffix(tagDescription.EnumFile, "."+config.Cfg.Language) + enumFileName := tagDescription.EnumFile switch strings.ToLower(expr.(*sqlparser.ComparisonExpr).Operator) { case "=": //when enum function operator is '=' , add 'or tag = xxx' @@ -1445,15 +1445,15 @@ func (f *WhereFunction) Trans(expr sqlparser.Expr, w *Where, asTagMap map[string // when value type is int, add toUInt64() function if strings.Contains(tagName, "pod_group_type") { podGroupTag := strings.Replace(tagName, "pod_group_type", "pod_group_id", -1) - whereFilter = "(" + fmt.Sprintf(tagItem.WhereTranslator, "=", f.Value, enumFileName) + ") OR " + "dictGet('flow_tag.pod_group_map', 'pod_group_type', (toUInt64(" + podGroupTag + ")))" + " = " + "toUInt64(" + strconv.Itoa(intValue) + ")" + whereFilter = "(" + fmt.Sprintf(tagItem.WhereTranslator, nameColumn, "=", f.Value, enumFileName) + ") OR " + "dictGet('flow_tag.pod_group_map', 'pod_group_type', (toUInt64(" + podGroupTag + ")))" + " = " + "toUInt64(" + strconv.Itoa(intValue) + ")" } else { - whereFilter = fmt.Sprintf(tagItem.WhereTranslator, "=", f.Value, enumFileName) + " OR " + tagName + " = " + "toUInt64(" + strconv.Itoa(intValue) + ")" + whereFilter = fmt.Sprintf(tagItem.WhereTranslator, nameColumn, "=", f.Value, enumFileName) + " OR " + tagName + " = " + "toUInt64(" + strconv.Itoa(intValue) + ")" } } else { - whereFilter = fmt.Sprintf(tagItem.WhereTranslator, "=", f.Value, enumFileName) + whereFilter = fmt.Sprintf(tagItem.WhereTranslator, nameColumn, "=", f.Value, enumFileName) } } else { - whereFilter = fmt.Sprintf(tagItem.WhereTranslator, "=", f.Value, enumFileName) + " OR " + tagName + " = " + f.Value + whereFilter = fmt.Sprintf(tagItem.WhereTranslator, nameColumn, "=", f.Value, enumFileName) + " OR " + tagName + " = " + f.Value } case "!=": //when enum function operator is '!=', add 'and tag != xxx' @@ -1463,37 +1463,37 @@ func (f *WhereFunction) Trans(expr sqlparser.Expr, w *Where, asTagMap map[string // when value type is int, add toUInt64() function if strings.Contains(tagName, "pod_group_type") { podGroupTag := strings.Replace(tagName, "pod_group_type", "pod_group_id", -1) - whereFilter = "not(" + fmt.Sprintf(tagItem.WhereTranslator, "=", f.Value, enumFileName) + ") AND " + "dictGet('flow_tag.pod_group_map', 'pod_group_type', (toUInt64(" + podGroupTag + ")))" + " != " + "toUInt64(" + strconv.Itoa(intValue) + ")" + whereFilter = "not(" + fmt.Sprintf(tagItem.WhereTranslator, nameColumn, "=", f.Value, enumFileName) + ") AND " + "dictGet('flow_tag.pod_group_map', 'pod_group_type', (toUInt64(" + podGroupTag + ")))" + " != " + "toUInt64(" + strconv.Itoa(intValue) + ")" } else { - whereFilter = fmt.Sprintf(tagItem.WhereTranslator, opName, f.Value, enumFileName) + " AND " + tagName + " != " + "toUInt64(" + strconv.Itoa(intValue) + ")" + whereFilter = fmt.Sprintf(tagItem.WhereTranslator, nameColumn, opName, f.Value, enumFileName) + " AND " + tagName + " != " + "toUInt64(" + strconv.Itoa(intValue) + ")" } } else { if strings.Contains(tagName, "pod_group_type") { - whereFilter = "not(" + fmt.Sprintf(tagItem.WhereTranslator, "=", f.Value, enumFileName) + ")" + whereFilter = "not(" + fmt.Sprintf(tagItem.WhereTranslator, nameColumn, "=", f.Value, enumFileName) + ")" } else { - whereFilter = fmt.Sprintf(tagItem.WhereTranslator, opName, f.Value, enumFileName) + whereFilter = fmt.Sprintf(tagItem.WhereTranslator, nameColumn, opName, f.Value, enumFileName) } } } else { - whereFilter = fmt.Sprintf(tagItem.WhereTranslator, opName, f.Value, enumFileName) + " AND " + tagName + " != " + f.Value + whereFilter = fmt.Sprintf(tagItem.WhereTranslator, nameColumn, opName, f.Value, enumFileName) + " AND " + tagName + " != " + f.Value } case "not match": if strings.Contains(tagName, "pod_group_type") { - whereFilter = "not(" + fmt.Sprintf(tagItem.WhereRegexpTranslator, "match", f.Value, enumFileName) + ")" + whereFilter = "not(" + fmt.Sprintf(tagItem.WhereRegexpTranslator, "match", nameColumn, f.Value, enumFileName) + ")" } else { - whereFilter = fmt.Sprintf(tagItem.WhereRegexpTranslator, opName, f.Value, enumFileName) + whereFilter = fmt.Sprintf(tagItem.WhereRegexpTranslator, opName, nameColumn, f.Value, enumFileName) } case "not in": if strings.Contains(tagName, "pod_group_type") { - whereFilter = "not(" + fmt.Sprintf(tagItem.WhereTranslator, "in", f.Value, enumFileName) + ")" + whereFilter = "not(" + fmt.Sprintf(tagItem.WhereTranslator, nameColumn, "in", f.Value, enumFileName) + ")" } else { - whereFilter = fmt.Sprintf(tagItem.WhereTranslator, opName, f.Value, enumFileName) + whereFilter = fmt.Sprintf(tagItem.WhereTranslator, nameColumn, opName, f.Value, enumFileName) } default: if strings.Contains(opName, "match") { - whereFilter = fmt.Sprintf(tagItem.WhereRegexpTranslator, opName, f.Value, enumFileName) + whereFilter = fmt.Sprintf(tagItem.WhereRegexpTranslator, opName, nameColumn, f.Value, enumFileName) } else { - whereFilter = fmt.Sprintf(tagItem.WhereTranslator, opName, f.Value, enumFileName) + whereFilter = fmt.Sprintf(tagItem.WhereTranslator, nameColumn, opName, f.Value, enumFileName) } } return &view.Expr{Value: "(" + whereFilter + ")"}, nil diff --git a/server/querier/engine/clickhouse/function.go b/server/querier/engine/clickhouse/function.go index 285a3e587e2..3e8fd5c7e0c 100644 --- a/server/querier/engine/clickhouse/function.go +++ b/server/querier/engine/clickhouse/function.go @@ -190,11 +190,23 @@ func GetTopKTrans(name string, args []string, alias string, e *CHEngine) (Statem DB: db, Table: table, TagName: field, }] if getTagOK { + nameColumn := "" + if e.Language != "" { + nameColumn = "name_" + e.Language + } else { + cfgLang := "" + if config.Cfg.Language == "en" { + cfgLang = "en" + } else { + cfgLang = "zh" + } + nameColumn = "name_" + cfgLang + } if tagOK { - enumFileName := strings.TrimSuffix(tagDescription.EnumFile, "."+config.Cfg.Language) - dbFields[i] = fmt.Sprintf(tagDes.TagTranslator, enumFileName) + enumFileName := tagDescription.EnumFile + dbFields[i] = fmt.Sprintf(tagDes.TagTranslator, nameColumn, enumFileName) } else { - dbFields[i] = fmt.Sprintf(tagDes.TagTranslator, tagEnum) + dbFields[i] = fmt.Sprintf(tagDes.TagTranslator, nameColumn, tagEnum) } } @@ -422,9 +434,13 @@ func (f *AggFunction) FormatInnerTag(m *view.Model) (innerAlias string) { } // When using max, and min operators. The inner layer uses itself if slices.Contains([]string{view.FUNCTION_MAX, view.FUNCTION_MIN}, f.Name) { + field := f.Metrics.DBField + if f.Metrics.DBField == "time" { + field = "toUnixTimestamp(time)" + } innerFunction = view.DefaultFunction{ Name: f.Name, - Fields: []view.Node{&view.Field{Value: f.Metrics.DBField}}, + Fields: []view.Node{&view.Field{Value: field}}, IgnoreZero: true, } } @@ -589,6 +605,9 @@ func (f *AggFunction) Trans(m *view.Model) view.Node { if f.Name == view.FUNCTION_COUNT { field = "1" } + if slices.Contains([]string{view.FUNCTION_MAX, view.FUNCTION_MIN}, f.Name) && field == "time" { + field = "toUnixTimestamp(time)" + } outFunc.SetFields([]view.Node{&view.Field{Value: field}}) } outFunc.SetFlag(view.METRICS_FLAG_OUTER) @@ -916,30 +935,45 @@ func (f *TagFunction) Trans(m *view.Model) view.Node { } return f.getViewNode() case TAG_FUNCTION_ENUM: - var tagFilter string + var tagTranslator string tagEnum := strings.TrimSuffix(f.Args[0], "_0") tagEnum = strings.TrimSuffix(tagEnum, "_1") tagDes, getTagOK := tag.GetTag(f.Args[0], f.DB, f.Table, f.Name) tagDescription, tagOK := tag.TAG_DESCRIPTIONS[tag.TagDescriptionKey{ DB: f.DB, Table: f.Table, TagName: f.Args[0], }] + language := f.Engine.Language + nameColumn := "" + if language != "" { + nameColumn = "name_" + language + } else { + cfgLang := "" + if config.Cfg.Language == "en" { + cfgLang = "en" + } else { + cfgLang = "zh" + } + nameColumn = "name_" + cfgLang + } if getTagOK { if tagOK { - enumFileName := strings.TrimSuffix(tagDescription.EnumFile, "."+config.Cfg.Language) + enumFileName := tagDescription.EnumFile if tagEnum == "app_service" || tagEnum == "app_instance" { enumFileName = tagEnum + tagTranslator = fmt.Sprintf(tagDes.TagTranslator, enumFileName) + } else { + tagTranslator = fmt.Sprintf(tagDes.TagTranslator, nameColumn, enumFileName) } - tagFilter = fmt.Sprintf(tagDes.TagTranslator, enumFileName) } else { - tagFilter = fmt.Sprintf(tagDes.TagTranslator, tagEnum) + tagTranslator = fmt.Sprintf(tagDes.TagTranslator, nameColumn, tagEnum) } } else { - tagFilter = fmt.Sprintf("Enum(%s)", f.Args[0]) + tagTranslator = fmt.Sprintf("Enum(%s)", f.Args[0]) } if f.Alias == "" { f.Alias = fmt.Sprintf("Enum(%s)", f.Args[0]) } - f.Withs = []view.Node{&view.With{Value: tagFilter, Alias: f.Alias}} + f.Withs = []view.Node{&view.With{Value: tagTranslator, Alias: f.Alias}} return f.getViewNode() } values := make([]string, len(fields)) diff --git a/server/querier/engine/clickhouse/metrics/metrics.go b/server/querier/engine/clickhouse/metrics/metrics.go index 75c4a16255d..1889f7e95fa 100644 --- a/server/querier/engine/clickhouse/metrics/metrics.go +++ b/server/querier/engine/clickhouse/metrics/metrics.go @@ -224,13 +224,19 @@ func GetTagTypeMetrics(tagDescriptions *common.Result, newAllMetrics map[string] func GetMetrics(field, db, table, orgID string) (*Metrics, bool) { newAllMetrics := map[string]*Metrics{} field = strings.Trim(field, "`") - // time is not a metric // flow_tag database has no metrics // trace_tree table has no metrics // span_with_trace_id table has no metrics - if field == "time" || db == ckcommon.DB_NAME_FLOW_TAG || slices.Contains([]string{ckcommon.TABLE_NAME_TRACE_TREE, ckcommon.TABLE_NAME_SPAN_WITH_TRACE_ID}, table) { + if db == ckcommon.DB_NAME_FLOW_TAG || slices.Contains([]string{ckcommon.TABLE_NAME_TRACE_TREE, ckcommon.TABLE_NAME_SPAN_WITH_TRACE_ID}, table) { return nil, false } + if field == "time" { + metric := NewMetrics( + 0, "time", field, "时间", field, "", "", "", METRICS_TYPE_NAME_MAP["delay"], + "Tag", []bool{true, true, true}, "", table, "", "", "", "time", + ) + return metric, true + } if slices.Contains([]string{ckcommon.DB_NAME_EXT_METRICS, ckcommon.DB_NAME_DEEPFLOW_ADMIN, ckcommon.DB_NAME_DEEPFLOW_TENANT, ckcommon.DB_NAME_APPLICATION_LOG}, db) || table == "l7_flow_log" { fieldSplit := strings.Split(field, ".") if len(fieldSplit) > 1 { diff --git a/server/querier/engine/clickhouse/tag/description.go b/server/querier/engine/clickhouse/tag/description.go index d8056ce78e9..00f48b021e0 100644 --- a/server/querier/engine/clickhouse/tag/description.go +++ b/server/querier/engine/clickhouse/tag/description.go @@ -94,8 +94,6 @@ type TagDescription struct { DisplayNameEN string Type string EnumFile string - EnumFileZH string - EnumFileEN string Category string Description string DescriptionZH string @@ -109,7 +107,7 @@ type TagDescription struct { } func NewTagDescription( - name, clientName, serverName, displayName, displayNameZH, displayNameEN, tagType, enumFile, enumFileZH, enumFileEN, category string, + name, clientName, serverName, displayName, displayNameZH, displayNameEN, tagType, enumFile, category string, permissions []bool, description, descriptionZH, descriptionEN, relatedTag string, deprecated bool, notSupportedOperators []string, table string, ) *TagDescription { operators, ok := tagTypeToOperators[tagType] @@ -125,8 +123,6 @@ func NewTagDescription( DisplayNameEN: displayNameEN, Type: tagType, EnumFile: enumFile, - EnumFileZH: enumFileZH, - EnumFileEN: enumFileEN, Category: category, Operators: operators, Permissions: permissions, @@ -141,16 +137,22 @@ func NewTagDescription( } type TagEnum struct { - Value interface{} - DisplayName interface{} - Description interface{} + Value interface{} + DisplayNameZH interface{} + DisplayNameEN interface{} + DescriptionZH interface{} + DescriptionEN interface{} + TagType interface{} } -func NewTagEnum(value, displayName, description interface{}) *TagEnum { +func NewTagEnum(value, displayNameZH, displayNameEN, descriptionZH, descriptionEN, tagType interface{}) *TagEnum { return &TagEnum{ - Value: value, - DisplayName: displayName, - Description: description, + Value: value, + DisplayNameZH: displayNameZH, + DisplayNameEN: displayNameEN, + DescriptionZH: descriptionZH, + DescriptionEN: descriptionEN, + TagType: tagType, } } @@ -219,9 +221,6 @@ func LoadTagDescriptions(tagData map[string]interface{}) error { TAG_DESCRIPTION_KEYS = append(TAG_DESCRIPTION_KEYS, key) enumFile := tag[4].(string) - enumFile = tag[4].(string) + "." + config.Cfg.Language - enumFileZH := tag[4].(string) + ".ch" - enumFileEN := tag[4].(string) + ".en" displayName := tagLanguage[1].(string) displayNameZH := tagLanguageZH[1].(string) displayNameEN := tagLanguageEN[1].(string) @@ -230,7 +229,7 @@ func LoadTagDescriptions(tagData map[string]interface{}) error { desEN := tagLanguageEN[2].(string) description := NewTagDescription( tag[0].(string), tag[1].(string), tag[2].(string), displayName, displayNameZH, displayNameEN, - tag[3].(string), enumFile, enumFileZH, enumFileEN, tag[5].(string), permissions, des, desZH, desEN, "", deprecated, notSupportedOperators, table, + tag[3].(string), enumFile, tag[5].(string), permissions, des, desZH, desEN, "", deprecated, notSupportedOperators, table, ) TAG_DESCRIPTIONS[key] = description enumFileToTagType[enumFile] = tag[3].(string) @@ -241,33 +240,73 @@ func LoadTagDescriptions(tagData map[string]interface{}) error { // 生成tag enum值 tagEnumData, ok := tagData["enum"] if ok { + tagMap := map[string][][6]interface{}{} for tagEnumFile, enumData := range tagEnumData.(map[string]interface{}) { - tagEnums := []*TagEnum{} - tagIntEnums := []*TagEnum{} - tagStringEnums := []*TagEnum{} - // 根据tagEnumFile获取tagTypeToOperators - tagType, _ := enumFileToTagType[tagEnumFile] - if tagType == "" { - tagType, _ = enumFileToTagType[tagEnumFile+"."+config.Cfg.Language] + tagName := strings.TrimSuffix(tagEnumFile, ".ch") + tagName = strings.TrimSuffix(tagName, ".en") + values, ok := tagMap[tagName] + if !ok { + valuesLen := len(enumData.([][]interface{})) + values = make([][6]interface{}, valuesLen) } - for _, enumValue := range enumData.([][]interface{}) { + // 根据tagEnumFile获取tagTypeToOperators + tagType, _ := enumFileToTagType[tagName] + for i, enumValue := range enumData.([][]interface{}) { // 如果是int/int_enum，则将value转为interface if tagType == "int" || tagType == "int_enum" || tagType == "bit_enum" { value, _ := strconv.Atoi(enumValue[0].(string)) - tagIntEnums = append(tagIntEnums, NewTagEnum(enumValue[0], enumValue[1], enumValue[2])) - tagEnums = append(tagEnums, NewTagEnum(value, enumValue[1], enumValue[2])) + values[i][0] = value + if strings.HasSuffix(tagEnumFile, ".en") { + values[i][2] = enumValue[1] + values[i][4] = enumValue[2] + } else if strings.HasSuffix(tagEnumFile, ".ch") { + values[i][1] = enumValue[1] + values[i][3] = enumValue[2] + } else { + values[i][1] = enumValue[1] + values[i][2] = enumValue[1] + values[i][3] = enumValue[2] + values[i][4] = enumValue[2] + } } else if tagType == "string_enum" { - tagStringEnums = append(tagEnums, NewTagEnum(enumValue[0], enumValue[1], enumValue[2])) - tagEnums = append(tagEnums, NewTagEnum(enumValue[0], enumValue[1], enumValue[2])) + values[i][0] = enumValue[0] + if strings.HasSuffix(tagEnumFile, ".en") { + values[i][2] = enumValue[1] + values[i][4] = enumValue[2] + } else if strings.HasSuffix(tagEnumFile, ".ch") { + values[i][1] = enumValue[1] + values[i][3] = enumValue[2] + } else { + values[i][1] = enumValue[1] + values[i][2] = enumValue[1] + values[i][3] = enumValue[2] + values[i][4] = enumValue[2] + } + } + values[i][5] = tagType + } + tagMap[tagName] = values + } + for tagName, values := range tagMap { + tagEnums := []*TagEnum{} + tagIntEnums := []*TagEnum{} + tagStringEnums := []*TagEnum{} + for _, datas := range values { + tagType, _ := datas[5].(string) + if tagType == "string_enum" { + tagStringEnums = append(tagStringEnums, NewTagEnum(datas[0], datas[1], datas[2], datas[3], datas[4], datas[5])) + } else { + tagIntEnums = append(tagIntEnums, NewTagEnum(datas[0], datas[1], datas[2], datas[3], datas[4], datas[5])) } + tagEnums = append(tagEnums, NewTagEnum(datas[0], datas[1], datas[2], datas[3], datas[4], datas[5])) } - if len(tagIntEnums) != 0 { - TAG_INT_ENUMS[tagEnumFile] = tagIntEnums + if len(tagIntEnums) > 0 { + TAG_INT_ENUMS[tagName] = tagIntEnums } - if len(tagStringEnums) != 0 { - TAG_STRING_ENUMS[tagEnumFile] = tagStringEnums + if len(tagStringEnums) > 0 { + TAG_STRING_ENUMS[tagName] = tagStringEnums } - TAG_ENUMS[tagEnumFile] = tagEnums + TAG_ENUMS[tagName] = tagEnums } } else { return errors.New("get tag enum failed! ") @@ -1108,7 +1147,7 @@ func GetEnumTagValues(db, table, sql string) (map[string][]interface{}, error) { for key, tagValue := range TAG_INT_ENUMS { tagValues := []interface{}{} for _, value := range tagValue { - tagValues = append(tagValues, []interface{}{value.Value, value.DisplayName, value.Description}) + tagValues = append(tagValues, []interface{}{value.Value, value.DisplayNameZH, value.DisplayNameEN, value.DescriptionZH, value.DescriptionEN}) } response[key] = tagValues } @@ -1117,7 +1156,7 @@ func GetEnumTagValues(db, table, sql string) (map[string][]interface{}, error) { for key, tagValue := range TAG_STRING_ENUMS { tagValues := []interface{}{} for _, value := range tagValue { - tagValues = append(tagValues, []interface{}{value.Value, value.DisplayName, value.Description}) + tagValues = append(tagValues, []interface{}{value.Value, value.DisplayNameZH, value.DisplayNameEN, value.DescriptionZH, value.DescriptionEN}) } response[key] = tagValues } @@ -1129,7 +1168,7 @@ func GetEnumTags(db, table, sql string) (*common.Result, error) { response := &common.Result{ Columns: []interface{}{ "name", "client_name", "server_name", "display_name", "display_name_zh", "display_name_en", "type", "category", - "operators", "permissions", "description", "related_tag", "deprecated", "not_supported_operators", "table", + "operators", "permissions", "description", "description_zh", "description_en", "related_tag", "deprecated", "not_supported_operators", "table", }, Values: []interface{}{}, } @@ -1142,7 +1181,7 @@ func GetEnumTags(db, table, sql string) (*common.Result, error) { response.Values = append(response.Values, []interface{}{ tagDescription.Name, tagDescription.ClientName, tagDescription.ServerName, tagDescription.DisplayName, tagDescription.DisplayNameZH, tagDescription.DisplayNameEN, tagDescription.Type, - tagDescription.Category, tagDescription.Operators, tagDescription.Permissions, tagDescription.Description, tagDescription.RelatedTag, tagDescription.Deprecated, tagDescription.NotSupportedOperators, "", + tagDescription.Category, tagDescription.Operators, tagDescription.Permissions, tagDescription.Description, tagDescription.DescriptionZH, tagDescription.DescriptionEN, tagDescription.RelatedTag, tagDescription.Deprecated, tagDescription.NotSupportedOperators, "", }) } @@ -1151,7 +1190,7 @@ func GetEnumTags(db, table, sql string) (*common.Result, error) { return response, nil } -func GetEnumTagAllValues(db, table, sql string) ([]string, error) { +func GetEnumTagAllValues(db, table, sql, language string) ([]string, error) { sqlList := []string{} sqlSplit := strings.Fields(sql) tag := sqlSplit[2] @@ -1161,43 +1200,49 @@ func GetEnumTagAllValues(db, table, sql string) ([]string, error) { for _, tagDescription := range TAG_DESCRIPTIONS { if tagDescription.Name == tag { _, isEnumOK := TAG_ENUMS[tagDescription.EnumFile] - if !isEnumOK { - _, isEnumOK = TAG_ENUMS[strings.TrimSuffix(tagDescription.EnumFile, "."+config.Cfg.Language)] - } if !isEnumOK { return sqlList, errors.New(fmt.Sprintf("tag %s is not enum", tag)) } _, isStringEnumOK := TAG_STRING_ENUMS[tagDescription.EnumFile] - if !isStringEnumOK { - _, isStringEnumOK = TAG_STRING_ENUMS[strings.TrimSuffix(tagDescription.EnumFile, "."+config.Cfg.Language)] - } if isStringEnumOK { table = "string_enum_map" - tag_name = strings.TrimSuffix(tagDescription.EnumFile, "."+config.Cfg.Language) + tag_name = tagDescription.EnumFile if !slices.Contains(tag_names, "'"+tag_name+"'") { tag_names = append(tag_names, "'"+tag_name+"'") } } _, isIntEnumOK := TAG_INT_ENUMS[tagDescription.EnumFile] - if !isIntEnumOK { - _, isIntEnumOK = TAG_INT_ENUMS[strings.TrimSuffix(tagDescription.EnumFile, "."+config.Cfg.Language)] - } if isIntEnumOK { table = "int_enum_map" - tag_name = strings.TrimSuffix(tagDescription.EnumFile, "."+config.Cfg.Language) + tag_name = tagDescription.EnumFile if !slices.Contains(tag_names, "'"+tag_name+"'") { tag_names = append(tag_names, "'"+tag_name+"'") } } } } - sql = fmt.Sprintf("SELECT value,name AS display_name, description FROM %s WHERE tag_name IN (%s) GROUP BY value, display_name, description ORDER BY value ASC", table, strings.Join(tag_names, ",")) + nameColumn := "" + descriptionColumn := "" + if language != "" { + nameColumn = "name_" + language + descriptionColumn = "description_" + language + } else { + cfgLang := "" + if config.Cfg.Language == "en" { + cfgLang = "en" + } else { + cfgLang = "zh" + } + nameColumn = "name_" + cfgLang + descriptionColumn = "description_" + cfgLang + } + sql = fmt.Sprintf("SELECT value, %s AS display_name, %s AS description FROM %s WHERE tag_name IN (%s) GROUP BY value, display_name, description ORDER BY value ASC", nameColumn, descriptionColumn, table, strings.Join(tag_names, ",")) log.Debug(sql) sqlList = append(sqlList, sql) return sqlList, nil } -func GetTagValues(db, table, sql, queryCacheTTL, orgID string, useQueryCache bool) (*common.Result, []string, error) { +func GetTagValues(db, table, sql, queryCacheTTL, orgID, language string, useQueryCache bool) (*common.Result, []string, error) { var sqlList []string // 把`1m`的反引号去掉 table = strings.Trim(table, "`") @@ -1250,27 +1295,18 @@ func GetTagValues(db, table, sql, queryCacheTTL, orgID string, useQueryCache boo } // 根据tagEnumFile获取values _, isEnumOK := TAG_ENUMS[tagDescription.EnumFile] - if !isEnumOK { - _, isEnumOK = TAG_ENUMS[strings.TrimSuffix(tagDescription.EnumFile, "."+config.Cfg.Language)] - } if !isEnumOK { return GetTagResourceValues(db, table, sql) } _, isStringEnumOK := TAG_STRING_ENUMS[tagDescription.EnumFile] - if !isStringEnumOK { - _, isStringEnumOK = TAG_STRING_ENUMS[strings.TrimSuffix(tagDescription.EnumFile, "."+config.Cfg.Language)] - } if isStringEnumOK { table = "string_enum_map" - tag = strings.TrimSuffix(tagDescription.EnumFile, "."+config.Cfg.Language) + tag = tagDescription.EnumFile } _, isIntEnumOK := TAG_INT_ENUMS[tagDescription.EnumFile] - if !isIntEnumOK { - _, isIntEnumOK = TAG_INT_ENUMS[strings.TrimSuffix(tagDescription.EnumFile, "."+config.Cfg.Language)] - } if isIntEnumOK { table = "int_enum_map" - tag = strings.TrimSuffix(tagDescription.EnumFile, "."+config.Cfg.Language) + tag = tagDescription.EnumFile } var limitSql string @@ -1301,8 +1337,23 @@ func GetTagValues(db, table, sql, queryCacheTTL, orgID string, useQueryCache boo if strings.Contains(strings.ToLower(sql), "like") || strings.Contains(strings.ToLower(sql), "regexp") { orderBy = "length(display_name)" } + nameColumn := "" + descriptionColumn := "" + if language != "" { + nameColumn = "name_" + language + descriptionColumn = "description_" + language + } else { + cfgLang := "" + if config.Cfg.Language == "en" { + cfgLang = "en" + } else { + cfgLang = "zh" + } + nameColumn = "name_" + cfgLang + descriptionColumn = "description_" + cfgLang + } // querier will be called later, so there is no need to display the declaration db - sql = fmt.Sprintf("SELECT value,name AS display_name, description FROM %s WHERE tag_name='%s' %s GROUP BY value, display_name, description ORDER BY %s ASC %s", table, tag, whereSql, orderBy, limitSql) + sql = fmt.Sprintf("SELECT value, %s AS display_name, %s AS description FROM %s WHERE tag_name='%s' %s GROUP BY value, display_name, description ORDER BY %s ASC %s", nameColumn, descriptionColumn, table, tag, whereSql, orderBy, limitSql) log.Debug(sql) sqlList = append(sqlList, sql) return nil, sqlList, nil diff --git a/server/querier/engine/clickhouse/tag/translation.go b/server/querier/engine/clickhouse/tag/translation.go index e35160b7a35..da9c246926e 100644 --- a/server/querier/engine/clickhouse/tag/translation.go +++ b/server/querier/engine/clickhouse/tag/translation.go @@ -969,10 +969,10 @@ func GenerateTagResoureMap() map[string]map[string]*Tag { "", ), "enum": NewTag( - "dictGetOrDefault('flow_tag.int_enum_map', 'name', ('%s',toUInt64(dictGet('flow_tag.pod_group_map', 'pod_group_type', (toUInt64("+podGroupIDSuffix+"))))), dictGet('flow_tag.pod_group_map', 'pod_group_type', (toUInt64("+podGroupIDSuffix+"))))", + "dictGetOrDefault('flow_tag.int_enum_map', '%s', ('%s',toUInt64(dictGet('flow_tag.pod_group_map', 'pod_group_type', (toUInt64("+podGroupIDSuffix+"))))), dictGet('flow_tag.pod_group_map', 'pod_group_type', (toUInt64("+podGroupIDSuffix+"))))", "", - "toUInt64(dictGet('flow_tag.pod_group_map', 'pod_group_type', (toUInt64("+podGroupIDSuffix+")))) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE name %s %s and tag_name='%s') AND "+podGroupIDSuffix+"!=0", - "toUInt64(dictGet('flow_tag.pod_group_map', 'pod_group_type', (toUInt64("+podGroupIDSuffix+")))) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s(name,%s) and tag_name='%s') AND "+podGroupIDSuffix+"!=0", + "toUInt64(dictGet('flow_tag.pod_group_map', 'pod_group_type', (toUInt64("+podGroupIDSuffix+")))) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s %s %s and tag_name='%s') AND "+podGroupIDSuffix+"!=0", + "toUInt64(dictGet('flow_tag.pod_group_map', 'pod_group_type', (toUInt64("+podGroupIDSuffix+")))) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s(%s,%s) and tag_name='%s') AND "+podGroupIDSuffix+"!=0", ), } } @@ -980,10 +980,10 @@ func GenerateTagResoureMap() map[string]map[string]*Tag { for _, enumName := range INT_ENUM_TAG { tagResourceMap[enumName] = map[string]*Tag{ "enum": NewTag( - "dictGetOrDefault('flow_tag.int_enum_map', 'name', ('%s',toUInt64("+enumName+")), "+enumName+")", + "dictGetOrDefault('flow_tag.int_enum_map', '%s', ('%s',toUInt64("+enumName+")), "+enumName+")", "", - "toUInt64("+enumName+") GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE name %s %s and tag_name='%s')", - "toUInt64("+enumName+") GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s(name,%s) and tag_name='%s')", + "toUInt64("+enumName+") GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s %s %s and tag_name='%s')", + "toUInt64("+enumName+") GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s(%s,%s) and tag_name='%s')", ), } } @@ -999,18 +999,18 @@ func GenerateTagResoureMap() map[string]map[string]*Tag { _, ok := tagResourceMap[tagEnumNameSuffix] if ok { tagResourceMap[tagEnumNameSuffix]["enum"] = NewTag( - "dictGetOrDefault('flow_tag.int_enum_map', 'name', ('%s',toUInt64("+enumNameSuffix+")), "+enumNameSuffix+")", + "dictGetOrDefault('flow_tag.int_enum_map', '%s', ('%s',toUInt64("+enumNameSuffix+")), "+enumNameSuffix+")", "", - "toUInt64("+enumNameSuffix+") GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE name %s %s and tag_name='%s')", - "toUInt64("+enumNameSuffix+") GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s(name,%s) and tag_name='%s')", + "toUInt64("+enumNameSuffix+") GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s %s %s and tag_name='%s')", + "toUInt64("+enumNameSuffix+") GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s(%s,%s) and tag_name='%s')", ) } else { tagResourceMap[tagEnumNameSuffix] = map[string]*Tag{ "enum": NewTag( - "dictGetOrDefault('flow_tag.int_enum_map', 'name', ('%s',toUInt64("+enumNameSuffix+")), "+enumNameSuffix+")", + "dictGetOrDefault('flow_tag.int_enum_map', '%s', ('%s',toUInt64("+enumNameSuffix+")), "+enumNameSuffix+")", "", - "toUInt64("+enumNameSuffix+") GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE name %s %s and tag_name='%s')", - "toUInt64("+enumNameSuffix+") GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s(name,%s) and tag_name='%s')", + "toUInt64("+enumNameSuffix+") GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s %s %s and tag_name='%s')", + "toUInt64("+enumNameSuffix+") GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s(%s,%s) and tag_name='%s')", ), } } @@ -1020,28 +1020,28 @@ func GenerateTagResoureMap() map[string]map[string]*Tag { // nullable int_enum tag do not return default value tagResourceMap["span_kind"] = map[string]*Tag{ "enum": NewTag( - "if(isNull(span_kind), '', dictGetOrDefault('flow_tag.int_enum_map', 'name', ('%s',toUInt64(assumeNotNull(span_kind))), span_kind))", + "if(isNull(span_kind), '', dictGetOrDefault('flow_tag.int_enum_map', '%s', ('%s',toUInt64(assumeNotNull(span_kind))), span_kind))", "", - "toUInt64(span_kind) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE name %s %s and tag_name='%s')", - "toUInt64(span_kind) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s(name,%s) and tag_name='%s')", + "toUInt64(span_kind) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s %s %s and tag_name='%s')", + "toUInt64(span_kind) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s(%s,%s) and tag_name='%s')", )} for _, enumName := range STRING_ENUM_TAG { tagResourceMap[enumName] = map[string]*Tag{ "enum": NewTag( - "dictGetOrDefault('flow_tag.string_enum_map', 'name', ('%s',"+enumName+"), "+enumName+")", + "dictGetOrDefault('flow_tag.string_enum_map', '%s', ('%s',"+enumName+"), "+enumName+")", "", - enumName+" GLOBAL IN (SELECT value FROM flow_tag.string_enum_map WHERE name %s %s and tag_name='%s')", - enumName+" GLOBAL IN (SELECT value FROM flow_tag.string_enum_map WHERE %s(name,%s) and tag_name='%s')", + enumName+" GLOBAL IN (SELECT value FROM flow_tag.string_enum_map WHERE %s %s %s and tag_name='%s')", + enumName+" GLOBAL IN (SELECT value FROM flow_tag.string_enum_map WHERE %s(%s,%s) and tag_name='%s')", ), } } // tap_side & Enum(tap_side) tagResourceMap["tap_side"] = map[string]*Tag{ "enum": NewTag( - "dictGetOrDefault('flow_tag.string_enum_map', 'name', ('%s',observation_point), observation_point)", + "dictGetOrDefault('flow_tag.string_enum_map', '%s', ('%s',observation_point), observation_point)", "", - "observation_point GLOBAL IN (SELECT value FROM flow_tag.string_enum_map WHERE name %s %s and tag_name='%s')", - "observation_point GLOBAL IN (SELECT value FROM flow_tag.string_enum_map WHERE %s(name,%s) and tag_name='%s')", + "observation_point GLOBAL IN (SELECT value FROM flow_tag.string_enum_map WHERE %s %s %s and tag_name='%s')", + "observation_point GLOBAL IN (SELECT value FROM flow_tag.string_enum_map WHERE %s(%s,%s) and tag_name='%s')", ), "default": NewTag( "observation_point", @@ -1053,10 +1053,10 @@ func GenerateTagResoureMap() map[string]map[string]*Tag { // tap_port_type & Enum(tap_port_type) tagResourceMap["tap_port_type"] = map[string]*Tag{ "enum": NewTag( - "dictGetOrDefault('flow_tag.int_enum_map', 'name', ('%s',toUInt64(capture_nic_type)), capture_nic_type)", + "dictGetOrDefault('flow_tag.int_enum_map', '%s', ('%s',toUInt64(capture_nic_type)), capture_nic_type)", "", - "toUInt64(capture_nic_type) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE name %s %s and tag_name='%s')", - "toUInt64(capture_nic_type) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s(name,%s) and tag_name='%s')", + "toUInt64(capture_nic_type) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s %s %s and tag_name='%s')", + "toUInt64(capture_nic_type) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s(%s,%s) and tag_name='%s')", ), "default": NewTag( "capture_nic_type", @@ -1997,20 +1997,20 @@ func GenerateAlarmEventTagResoureMap() map[string]map[string]*Tag { // enum(event_level) tagResourceMap["event_level"] = map[string]*Tag{ "enum": NewTag( - "dictGetOrDefault('flow_tag.int_enum_map', 'name', ('%s',toUInt64(event_level)), event_level)", + "dictGetOrDefault('flow_tag.int_enum_map', '%s', ('%s',toUInt64(event_level)), event_level)", "", - "toUInt64(event_level) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE name %s %s and tag_name='%s')", - "toUInt64(event_level) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s(name,%s) and tag_name='%s')", + "toUInt64(event_level) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s %s %s and tag_name='%s')", + "toUInt64(event_level) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s(%s,%s) and tag_name='%s')", ), } //enum(policy_type) tagResourceMap["policy_type"] = map[string]*Tag{ "enum": NewTag( - "dictGetOrDefault('flow_tag.int_enum_map', 'name', ('%s',toUInt64(policy_type)), policy_type)", + "dictGetOrDefault('flow_tag.int_enum_map', '%s', ('%s',toUInt64(policy_type)), policy_type)", "", - "toUInt64(policy_type) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE name %s %s and tag_name='%s')", - "toUInt64(policy_type) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s(name,%s) and tag_name='%s')", + "toUInt64(policy_type) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s %s %s and tag_name='%s')", + "toUInt64(policy_type) GLOBAL IN (SELECT value FROM flow_tag.int_enum_map WHERE %s(%s,%s) and tag_name='%s')", ), } @@ -2095,14 +2095,14 @@ func GenerateFlowTagTagResoureMap() map[string]map[string]*Tag { "default": NewTag( "", "", - "name %s %s", - "%s (name, %s)", + "%s %s %s", + "%s (%s, %s)", ), "enum": NewTag( "", "", - "name %s %s", - "%s (name, %s)", + "%s %s %s", + "%s (%s, %s)", ), }