Skip to content

Commit

Permalink
feat: querier supports show enum tags by language
Browse files Browse the repository at this point in the history
  • Loading branch information
@xiaochaoren1
xiaochaoren1 committed Dec 11, 2024
1 parent 76b2131 commit 24e0ed8
Show file tree
Hide file tree
Showing 23 changed files with 438 additions and 289 deletions.
13 changes: 8 additions & 5 deletions server/controller/db/mysql/migrator/schema/rawsql/init.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2205,12 +2205,13 @@ CREATE TABLE IF NOT EXISTS mail_server (
)ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
TRUNCATE TABLE mail_server;


CREATE TABLE IF NOT EXISTS ch_string_enum (
tag_name VARCHAR(256) NOT NULL ,
value VARCHAR(256) NOT NULL,
name VARCHAR(256) ,
description VARCHAR(256) ,
name_zh VARCHAR(256) ,
name_en VARCHAR(256) ,
description_zh VARCHAR(256) ,
description_en VARCHAR(256) ,
updated_at DATETIME NOT NULL ON UPDATE CURRENT_TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (tag_name,value)
)ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
Expand All @@ -2219,8 +2220,10 @@ TRUNCATE TABLE ch_string_enum;
CREATE TABLE IF NOT EXISTS ch_int_enum (
tag_name VARCHAR(256) NOT NULL,
value INTEGER DEFAULT 0,
name VARCHAR(256) ,
description VARCHAR(256) ,
name_zh VARCHAR(256) ,
name_en VARCHAR(256) ,
description_zh VARCHAR(256) ,
description_en VARCHAR(256) ,
updated_at DATETIME NOT NULL ON UPDATE CURRENT_TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (tag_name,value)
)ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
Expand Down
29 changes: 29 additions & 0 deletions server/controller/db/mysql/migrator/schema/rawsql/issu/6.6.1.19.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
DROP TABLE IF EXISTS ch_string_enum;
DROP TABLE IF EXISTS ch_int_enum;

CREATE TABLE IF NOT EXISTS ch_string_enum (
tag_name VARCHAR(256) NOT NULL ,
value VARCHAR(256) NOT NULL,
name_zh VARCHAR(256) ,
name_en VARCHAR(256) ,
description_zh VARCHAR(256) ,
description_en VARCHAR(256) ,
updated_at DATETIME NOT NULL ON UPDATE CURRENT_TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (tag_name,value)
)ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;

CREATE TABLE IF NOT EXISTS ch_int_enum (
tag_name VARCHAR(256) NOT NULL,
value INTEGER DEFAULT 0,
name_zh VARCHAR(256) ,
name_en VARCHAR(256) ,
description_zh VARCHAR(256) ,
description_en VARCHAR(256) ,
updated_at DATETIME NOT NULL ON UPDATE CURRENT_TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (tag_name,value)
)ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;

-- update db_version to latest, remeber update DB_VERSION_EXPECT in migrate/init.go
UPDATE db_version SET version='6.6.1.19';
-- modify end

2 changes: 1 addition & 1 deletion server/controller/db/mysql/migrator/schema/version.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ package schema

const (
DB_VERSION_TABLE = "db_version"
DB_VERSION_EXPECTED = "6.6.1.18"
DB_VERSION_EXPECTED = "6.6.1.19"
)

const (
Expand Down
24 changes: 14 additions & 10 deletions server/controller/db/mysql/model/ch_model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -329,19 +329,23 @@ type ChPodServiceK8sLabels struct {
}

type ChStringEnum struct {
TagName string `gorm:"primaryKey;column:tag_name;type:varchar(256);default:null" json:"TAG_NAME"`
Value string `gorm:"primaryKey;column:value;type:varchar(256);default:null" json:"VALUE"`
Name string `gorm:"column:name;type:varchar(256);default:null" json:"NAME"`
Description string `gorm:"column:description;type:varchar(256);default:null" json:"DESCRIPTION"`
UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime:now,type:timestamp" json:"UPDATED_AT"`
TagName string `gorm:"primaryKey;column:tag_name;type:varchar(256);default:null" json:"TAG_NAME"`
Value string `gorm:"primaryKey;column:value;type:varchar(256);default:null" json:"VALUE"`
NameZH string `gorm:"column:name_zh;type:varchar(256);default:null" json:"NAME_ZH"`
NameEN string `gorm:"column:name_en;type:varchar(256);default:null" json:"NAME_EN"`
DescriptionZH string `gorm:"column:description_zh;type:varchar(256);default:null" json:"DESCRIPTION_ZH"`
DescriptionEN string `gorm:"column:description_en;type:varchar(256);default:null" json:"DESCRIPTION_EN"`
UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime:now,type:timestamp" json:"UPDATED_AT"`
}

type ChIntEnum struct {
TagName string `gorm:"primaryKey;column:tag_name;type:varchar(256);default:null" json:"TAG_NAME"`
Value int `gorm:"primaryKey;column:value;type:int;default:0" json:"VALUE"`
Name string `gorm:"column:name;type:varchar(256);default:null" json:"NAME"`
Description string `gorm:"column:description;type:varchar(256);default:null" json:"DESCRIPTION"`
UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime:now,type:timestamp" json:"UPDATED_AT"`
TagName string `gorm:"primaryKey;column:tag_name;type:varchar(256);default:null" json:"TAG_NAME"`
Value int `gorm:"primaryKey;column:value;type:int;default:0" json:"VALUE"`
NameZH string `gorm:"column:name_zh;type:varchar(256);default:null" json:"NAME_ZH"`
NameEN string `gorm:"column:name_en;type:varchar(256);default:null" json:"NAME_EN"`
DescriptionZH string `gorm:"column:description_zh;type:varchar(256);default:null" json:"DESCRIPTION_ZH"`
DescriptionEN string `gorm:"column:description_en;type:varchar(256);default:null" json:"DESCRIPTION_EN"`
UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime:now,type:timestamp" json:"UPDATED_AT"`
}

type ChNodeType struct {
Expand Down
8 changes: 7 additions & 1 deletion server/controller/http/service/resource/domain.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -872,7 +872,13 @@ func UpdateSubDomain(lcuuid string, db *mysql.DB, userInfo *httpcommon.UserInfo,
}

// pub to tagrecorder
teamIDInt := int(teamID.(float64))
teamIDInt := 0
teamIDFloat64, ok := teamID.(float64)
if ok {
teamIDInt = int(teamIDFloat64)
} else {
teamIDInt = domain.TeamID
}
if teamIDInt != subDomain.TeamID {
metadata := message.NewMetadata(db.ORGID, message.MetadataSubDomainID(subDomain.ID), message.MetadataTeamID(teamIDInt))
for _, s := range tagrecorder.GetSubscriberManager().GetSubscribers("sub_domain") {
Expand Down
43 changes: 23 additions & 20 deletions server/controller/tagrecorder/ch_int_enum.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,8 @@
package tagrecorder

import (
"strconv"
"strings"

"github.com/deepflowio/deepflow/server/controller/db/mysql"
mysqlmodel "github.com/deepflowio/deepflow/server/controller/db/mysql/model"
"github.com/deepflowio/deepflow/server/querier/config"
"github.com/deepflowio/deepflow/server/querier/engine/clickhouse/tag"
)

Expand Down Expand Up @@ -51,28 +47,29 @@ func (e *ChIntEnum) generateNewData(dbClient *mysql.DB) (map[IntEnumTagKey]mysql
}

for name, tagValues := range respMap {
tagName := strings.TrimSuffix(name, "."+config.Cfg.Language)
for _, valueAndName := range tagValues {
tagValue := valueAndName.([]interface{})[0]
tagDisplayName := valueAndName.([]interface{})[1]
tagDescription := valueAndName.([]interface{})[2]
tagValueInt, err := strconv.Atoi(tagValue.(string))
if err == nil {
tagDisplayNameZH := valueAndName.([]interface{})[1]
tagDisplayNameEN := valueAndName.([]interface{})[2]
tagDescriptionZH := valueAndName.([]interface{})[3]
tagDescriptionEN := valueAndName.([]interface{})[4]
tagValueInt, ok := tagValue.(int)
if ok {
key := IntEnumTagKey{
TagName: tagName,
TagName: name,
TagValue: tagValueInt,
}
keyToItem[key] = mysqlmodel.ChIntEnum{
TagName: tagName,
Value: tagValueInt,
Name: tagDisplayName.(string),
Description: tagDescription.(string),
TagName: name,
Value: tagValueInt,
NameZH: tagDisplayNameZH.(string),
NameEN: tagDisplayNameEN.(string),
DescriptionZH: tagDescriptionZH.(string),
DescriptionEN: tagDescriptionEN.(string),
}
}

}
}

return keyToItem, true
}

Expand All @@ -88,11 +85,17 @@ func (e *ChIntEnum) generateUpdateInfo(oldItem, newItem mysqlmodel.ChIntEnum) (m
if oldItem.Value != newItem.Value {
updateInfo["value"] = newItem.Value
}
if oldItem.Name != newItem.Name {
updateInfo["name"] = newItem.Name
if oldItem.NameZH != newItem.NameZH {
updateInfo["name_zh"] = newItem.NameZH
}
if oldItem.NameEN != newItem.NameEN {
updateInfo["name_en"] = newItem.NameEN
}
if oldItem.DescriptionZH != newItem.DescriptionZH {
updateInfo["description_zh"] = newItem.DescriptionZH
}
if oldItem.Description != newItem.Description {
updateInfo["description"] = newItem.Description
if oldItem.DescriptionEN != newItem.DescriptionEN {
updateInfo["description_en"] = newItem.DescriptionEN
}
if len(updateInfo) > 0 {
return updateInfo, true
Expand Down
37 changes: 21 additions & 16 deletions server/controller/tagrecorder/ch_string_enum.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,8 @@
package tagrecorder

import (
"strings"

"github.com/deepflowio/deepflow/server/controller/db/mysql"
mysqlmodel "github.com/deepflowio/deepflow/server/controller/db/mysql/model"
"github.com/deepflowio/deepflow/server/querier/config"
"github.com/deepflowio/deepflow/server/querier/engine/clickhouse/tag"
)

Expand Down Expand Up @@ -50,24 +47,26 @@ func (e *ChStringEnum) generateNewData(dbClient *mysql.DB) (map[StringEnumTagKey
}

for name, tagValues := range respMap {
tagName := strings.TrimSuffix(name, "."+config.Cfg.Language)
for _, valueAndName := range tagValues {
tagValue := valueAndName.([]interface{})[0]
tagDisplayName := valueAndName.([]interface{})[1]
tagDescription := valueAndName.([]interface{})[2]
tagDisplayNameZH := valueAndName.([]interface{})[1]
tagDisplayNameEN := valueAndName.([]interface{})[2]
tagDescriptionZH := valueAndName.([]interface{})[3]
tagDescriptionEN := valueAndName.([]interface{})[4]
key := StringEnumTagKey{
TagName: tagName,
TagName: name,
TagValue: tagValue.(string),
}
keyToItem[key] = mysqlmodel.ChStringEnum{
TagName: tagName,
Value: tagValue.(string),
Name: tagDisplayName.(string),
Description: tagDescription.(string),
TagName: name,
Value: tagValue.(string),
NameZH: tagDisplayNameZH.(string),
NameEN: tagDisplayNameEN.(string),
DescriptionZH: tagDescriptionZH.(string),
DescriptionEN: tagDescriptionEN.(string),
}
}
}

return keyToItem, true
}

Expand All @@ -83,11 +82,17 @@ func (e *ChStringEnum) generateUpdateInfo(oldItem, newItem mysqlmodel.ChStringEn
if oldItem.Value != newItem.Value {
updateInfo["value"] = newItem.Value
}
if oldItem.Name != newItem.Name {
updateInfo["name"] = newItem.Name
if oldItem.NameZH != newItem.NameZH {
updateInfo["name_zh"] = newItem.NameZH
}
if oldItem.NameEN != newItem.NameEN {
updateInfo["name_en"] = newItem.NameEN
}
if oldItem.DescriptionZH != newItem.DescriptionZH {
updateInfo["description_zh"] = newItem.DescriptionZH
}
if oldItem.Description != newItem.Description {
updateInfo["description"] = newItem.Description
if oldItem.DescriptionEN != newItem.DescriptionEN {
updateInfo["description_en"] = newItem.DescriptionEN
}
if len(updateInfo) > 0 {
return updateInfo, true
Expand Down
42 changes: 23 additions & 19 deletions server/controller/tagrecorder/check/ch_int_enum.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,7 @@
package tagrecorder

import (
"strconv"
"strings"

mysqlmodel "github.com/deepflowio/deepflow/server/controller/db/mysql/model"
"github.com/deepflowio/deepflow/server/querier/config"
"github.com/deepflowio/deepflow/server/querier/engine/clickhouse/tag"
)

Expand Down Expand Up @@ -50,25 +46,27 @@ func (e *ChIntEnum) generateNewData() (map[IntEnumTagKey]mysqlmodel.ChIntEnum, b
}

for name, tagValues := range respMap {
tagName := strings.TrimSuffix(name, "."+config.Cfg.Language)
for _, valueAndName := range tagValues {
tagValue := valueAndName.([]interface{})[0]
tagDisplayName := valueAndName.([]interface{})[1]
tagDescription := valueAndName.([]interface{})[2]
tagValueInt, err := strconv.Atoi(tagValue.(string))
if err == nil {
tagDisplayNameZH := valueAndName.([]interface{})[1]
tagDisplayNameEN := valueAndName.([]interface{})[2]
tagDescriptionZH := valueAndName.([]interface{})[3]
tagDescriptionEN := valueAndName.([]interface{})[4]
tagValueInt, ok := tagValue.(int)
if ok {
key := IntEnumTagKey{
TagName: tagName,
TagName: name,
TagValue: tagValueInt,
}
keyToItem[key] = mysqlmodel.ChIntEnum{
TagName: tagName,
Value: tagValueInt,
Name: tagDisplayName.(string),
Description: tagDescription.(string),
TagName: name,
Value: tagValueInt,
NameZH: tagDisplayNameZH.(string),
NameEN: tagDisplayNameEN.(string),
DescriptionZH: tagDescriptionZH.(string),
DescriptionEN: tagDescriptionEN.(string),
}
}

}
}

Expand All @@ -87,11 +85,17 @@ func (e *ChIntEnum) generateUpdateInfo(oldItem, newItem mysqlmodel.ChIntEnum) (m
if oldItem.Value != newItem.Value {
updateInfo["value"] = newItem.Value
}
if oldItem.Name != newItem.Name {
updateInfo["name"] = newItem.Name
if oldItem.NameZH != newItem.NameZH {
updateInfo["name_zh"] = newItem.NameZH
}
if oldItem.NameEN != newItem.NameEN {
updateInfo["name_en"] = newItem.NameEN
}
if oldItem.DescriptionZH != newItem.DescriptionZH {
updateInfo["description_zh"] = newItem.DescriptionZH
}
if oldItem.Description != newItem.Description {
updateInfo["description"] = newItem.Description
if oldItem.DescriptionEN != newItem.DescriptionEN {
updateInfo["description_en"] = newItem.DescriptionEN
}
if len(updateInfo) > 0 {
return updateInfo, true
Expand Down
Loading

0 comments on commit 24e0ed8

Please sign in to comment.