diff --git a/server/agent_config/migrator.go b/server/agent_config/migrator.go index 8f911a8848d..81de66999d1 100644 --- a/server/agent_config/migrator.go +++ b/server/agent_config/migrator.go @@ -87,7 +87,7 @@ func (m *Upgrader) lowerToHigher(lowerVerData interface{}, ancestor string, high lowers := m.higherVerToLowerVerKeys[higher] if len(lowers) > 1 { for _, lower := range lowers { - log.Warnf("%s has been upgraded to %s", lower, higher) // TODO return? + log.Warnf("%s has been upgraded to %s, please configure it manually", lower, higher) // TODO return? } } else { m.setNestedValue(higherVerData, higher, m.fmtLowerVersionValue(newAncestor, value)) @@ -199,6 +199,13 @@ func (m *Upgrader) fmtLowerVersionValue(longKey string, value interface{}) inter return value } } + // TODO + // 升级 static_config.ebpf.uprobe-process-name-regexs.golang-symbol 时，需要： + // 1. 将 inputs.proc.symbol_table.golang_specific.enabled 设置为 true + // 2. 新增一条 inputs.proc.process_matcher + // 升级 static_config.os-proc-sync-tagged-only 时，需要： + // 1. 将 inputs.proc.process_matcher 里所有的 only_with_tag 设置为 static_config.os-proc-sync-tagged-only + return m.convDictData(longKey, value) } @@ -552,10 +559,10 @@ func (p *MigrationToolData) fmtDictValKeyMap() { // "only_with_tag": "static_config.os-proc-sync-tagged-only", // TODO "ignore": "action", "rewrite_name": "rewrite-name", - // "enabled_features": []string{"static_config.ebpf.on-cpu-profile.regex", "static_config.ebpf.off-cpu-profile.regex"}, // 无法降级 + // "enabled_features": []string{"static_config.ebpf.on-cpu-profile.regex", "static_config.ebpf.off-cpu-profile.regex"}, // 无法升降级 }, - "inputs.cdpf.af_packet.bond_interfaces": { + "inputs.cbpf.af_packet.bond_interfaces": { "slave_interfaces": "tap-interfaces", }, "inputs.resources.kubernetes.api_resources": { diff --git a/server/agent_config/migrator_test.go b/server/agent_config/migrator_test.go index 76d3cae0ded..9c525ce9033 100644 --- a/server/agent_config/migrator_test.go +++ b/server/agent_config/migrator_test.go @@ -199,11 +199,34 @@ static_config: value: HTTP HTTP2: - field-name: HTTP2-1 +`), + wantErr: false, + }, + { + name: "case02", + args: args{ + bytes: []byte(`inputs: + cbpf: + af_packet: + bond_interfaces: + - slave_interfaces: + - eth0 + - eth1 +`), + }, + want: []byte(`static_config: + tap-interface-bond-groups: + - tap-interfaces: + - eth0 + - eth1 `), wantErr: false, }, } for _, tt := range tests { + if tt.name != "case02" { + continue + } t.Run(tt.name, func(t *testing.T) { toolData, err := NewMigrationToolData(nil) if err != nil { @@ -387,32 +410,31 @@ func TestGenerateUpgradeHigherToLower(t *testing.T) { name: "case02", args: args{}, want: map[string][]string{ - "inputs.proc.process_matcher": {"static_config.os-proc-regex"}, - "inputs.proc.tag_extraction.process_matcher.rewrite_name": {"static_config.os-proc-regex.rewrite-name"}, + "inputs.proc.process_matcher": {"static_config.os-proc-regex"}, + "inputs.cbpf.af_packet.bond_interfaces": {"static_config.tap-interface-bond-groups"}, }, wantErr: false, }, } for _, tt := range tests { + if tt.name != "case02" { + continue + } t.Run(tt.name, func(t *testing.T) { - fmt, err := NewMigrationToolData(nil) + fmtt, err := NewMigrationToolData(nil) if err != nil { t.Fatalf("Failed to create parser: %v", err) } - fmt.Format() if (err != nil) != tt.wantErr { t.Errorf("getHigherToLowers() error = \"%v\", wantErr \"%v\"", err, tt.wantErr) return } - if len(fmt.higherVerToLowerVerKeys) != len(tt.want) { - t.Errorf("getHigherToLowers() = \"%v\", want \"%v\"", fmt.higherVerToLowerVerKeys, tt.want) - } for k, v := range tt.want { - if len(v) != len(fmt.higherVerToLowerVerKeys[k]) { + if len(v) != len(fmtt.higherVerToLowerVerKeys[k]) { t.Errorf("field %s getHigherToLowers() = \"%v\", want \"%v\"", k, v, tt.want[k]) } for i := 0; i < len(v); i++ { - if v[i] != fmt.higherVerToLowerVerKeys[k][i] { + if v[i] != fmtt.higherVerToLowerVerKeys[k][i] { t.Errorf("field %s %d getHigherToLowers() = \"%v\", want \"%v\"", k, i, v[i], tt.want[k][i]) } } @@ -569,7 +591,7 @@ func TestLowerToHigher(t *testing.T) { name: "case02", args: args{ lowerToHigher: map[string]string{ - "static_config.tap-interface-bond-groups": "inputs.resources.kubernetes.api_resources", + "static_config.tap-interface-bond-groups": "inputs.cbpf.af_packet.bond_interfaces", }, dictValLowerKeyToHigher: map[string]map[string]interface{}{ "static_config.tap-interface-bond-groups": { @@ -593,9 +615,9 @@ func TestLowerToHigher(t *testing.T) { }, want: map[string]interface{}{ "inputs": map[string]interface{}{ - "resources": map[string]interface{}{ - "kubernetes": map[string]interface{}{ - "api_resources": []map[string]interface{}{ + "cbpf": map[string]interface{}{ + "af_packet": map[string]interface{}{ + "bond_interfaces": []map[string]interface{}{ { "slave_interfaces": []string{"eth0", "eth1"}, }, @@ -610,9 +632,9 @@ func TestLowerToHigher(t *testing.T) { }, } for _, tt := range tests { - if tt.name != "case02" { - continue - } + // if tt.name != "case02" { + // continue + // } t.Run(tt.name, func(t *testing.T) { migrator := &Upgrader{ MigrationToolData: MigrationToolData{ @@ -800,8 +822,54 @@ func TestHigherToLower(t *testing.T) { }, }, }, + { + name: "case04", + args: args{ + higherToLower: map[string][]string{ + "inputs.cbpf.af_packet.bond_interfaces": {"static_config.tap-interface-bond-groups"}, + }, + dictValHigherKeyToLower: map[string]map[string]interface{}{ + "inputs.cbpf.af_packet.bond_interfaces": { + "slave_interfaces": "tap-interfaces", + }, + }, + ancestor: "", + data: map[string]interface{}{ + "inputs": map[string]interface{}{ + "cbpf": map[string]interface{}{ + "af_packet": map[string]interface{}{ + "bond_interfaces": []map[string]interface{}{ + { + "slave_interfaces": []string{"eth0", "eth1"}, + }, + { + "slave_interfaces": []string{"eth2", "eth3"}, + }, + }, + }, + }, + }, + }, + result: map[string]interface{}{}, + }, + want: map[string]interface{}{ + "static_config": map[string]interface{}{ + "tap-interface-bond-groups": []map[string]interface{}{ + { + "tap-interfaces": []string{"eth0", "eth1"}, + }, + { + "tap-interfaces": []string{"eth2", "eth3"}, + }, + }, + }, + }, + }, } for _, tt := range tests { + if tt.name != "case04" { + continue + } t.Run(tt.name, func(t *testing.T) { migrator := &Downgrader{ MigrationToolData: MigrationToolData{ @@ -976,7 +1044,6 @@ func TestConvDictDataValue(t *testing.T) { t.Run(tt.name, func(t *testing.T) { dictConv := &dictDataConv{} got := dictConv.convDictDataValue(tt.args.data, tt.args.convMap, tt.args.longKey) - fmt.Printf("got: %v\n", got) if !reflect.DeepEqual(got, tt.want) { t.Errorf("convDictDataValue() = %v, want %v", got, tt.want) } diff --git a/server/agent_config/template_json.go b/server/agent_config/template_json.go index 203e6b68940..dca906421b0 100644 --- a/server/agent_config/template_json.go +++ b/server/agent_config/template_json.go @@ -554,7 +554,7 @@ func (f *DataFomatter) LoadMapData(data map[string]interface{}) error { return fmt.Errorf("unmarshal yaml to node error: %v", err) } - err = f.stringToDictValue(f.mapData, "", f.keyToComment) + err = f.fmtVal("", f.mapData, f.keyToComment, false) if err != nil { return fmt.Errorf("convert dict value to string error: %v", err) } @@ -567,14 +567,11 @@ func (f *DataFomatter) LoadMapData(data map[string]interface{}) error { } func (f *DataFomatter) mapToYAML() ([]byte, error) { - var buf strings.Builder - enc := yaml.NewEncoder(&buf) - enc.SetIndent(2) - err := enc.Encode(f.mapData) + str, err := f.dictToString(f.mapData) if err != nil { - return nil, err + return nil, fmt.Errorf("convert dict to string error: %v", err) } - return []byte(buf.String()), nil + return []byte(str), nil } func (f *DataFomatter) LoadYAMLData(yamlData []byte) error { @@ -606,17 +603,12 @@ func (f *DataFomatter) mapToJSON(keyToComment KeyToComment) ([]byte, error) { } func (f *DataFomatter) fmtMapValAndRefresh(keyToComment KeyToComment, dictValToStr bool) error { - var err error - if dictValToStr { - err = f.dictValueToString(f.mapData, "", keyToComment) - } else { - err = f.stringToDictValue(f.mapData, "", keyToComment) - } + err := f.fmtVal("", f.mapData, keyToComment, dictValToStr) if err != nil { return fmt.Errorf("convert dict value to string error: %v", err) } // refresh yamlNode if changes dict value - if _, ok := keyToComment["changesDictValue"]; ok { + if _, ok := keyToComment["valueChanged"]; ok { yamlBytes, err := yaml.Marshal(f.mapData) if err != nil { return fmt.Errorf("marshal map to yaml error: %v", err) @@ -631,54 +623,47 @@ func (f *DataFomatter) fmtMapValAndRefresh(keyToComment KeyToComment, dictValToS return nil } -func (f *DataFomatter) dictValueToString(data interface{}, ancestors string, keyToComment KeyToComment) error { +func (f *DataFomatter) fmtVal(ancestors string, data interface{}, keyToComment KeyToComment, dictValToStr bool) error { switch data := data.(type) { case map[string]interface{}: for key, value := range data { newAncestors := f.appendAncestor(ancestors, key) if !f.isKeyComment(key) { if f.isDictValue(keyToComment[newAncestors]) { - valueStr, err := f.dictToString(value) - if err != nil { - return fmt.Errorf("convert dict value to string error: %v, key: %s", err, newAncestors) - } - data[key] = valueStr - keyToComment["changesDictValue"] = make(map[string]interface{}) - } - f.dictValueToString(value, newAncestors, keyToComment) - } - } - default: - return nil - } - return nil -} - -func (f *DataFomatter) stringToDictValue(data interface{}, ancestors string, keyToComment KeyToComment) error { - switch data := data.(type) { - case map[string]interface{}: - for key, value := range data { - newAncestors := f.appendAncestor(ancestors, key) - if !f.isKeyComment(key) { - if f.isDictValue(keyToComment[newAncestors]) { - if strings.HasPrefix(strings.TrimSpace(value.(string)), "-") { - var valueMap map[string]interface{} - err := yaml.Unmarshal([]byte(key+":\n"+value.(string)), &valueMap) + if dictValToStr { + valueStr, err := f.dictToString(value) if err != nil { - return fmt.Errorf("unmarshal string to map error: %v, key: %s", err, newAncestors) + return fmt.Errorf("convert dict value to string error: %v, key: %s", err, newAncestors) } - data[key] = valueMap[key] + data[key] = valueStr } else { - var valueMap map[string]interface{} - err := yaml.Unmarshal([]byte(value.(string)), &valueMap) - if err != nil { - return fmt.Errorf("unmarshal string to map error: %v, key: %s", err, newAncestors) + if strings.HasPrefix(strings.TrimSpace(value.(string)), "-") { + var valueMap map[string]interface{} + err := yaml.Unmarshal([]byte(key+":\n"+value.(string)), &valueMap) + if err != nil { + return fmt.Errorf("unmarshal string to map error: %v, key: %s", err, newAncestors) + } + data[key] = valueMap[key] + } else { + var valueMap map[string]interface{} + err := yaml.Unmarshal([]byte(value.(string)), &valueMap) + if err != nil { + return fmt.Errorf("unmarshal string to map error: %v, key: %s", err, newAncestors) + } + data[key] = valueMap } - data[key] = valueMap } - keyToComment["changesDictValue"] = make(map[string]interface{}) + keyToComment["valueChanged"] = make(map[string]interface{}) + } else if f.isIntValue(keyToComment[newAncestors]) { + switch value := value.(type) { + case int: + data[key] = value + case float64: + data[key] = int(value) + keyToComment["valueChanged"] = make(map[string]interface{}) + } } - f.stringToDictValue(value, newAncestors, keyToComment) + f.fmtVal(newAncestors, value, keyToComment, dictValToStr) } } default: @@ -697,8 +682,14 @@ func (f *DataFomatter) formatJson(data interface{}) ([]byte, error) { } func (f *DataFomatter) dictToString(data interface{}) (string, error) { - bytes, err := yaml.Marshal(data) - return string(bytes), err + var buf strings.Builder + enc := yaml.NewEncoder(&buf) + enc.SetIndent(2) + err := enc.Encode(data) + if err != nil { + return "", err + } + return buf.String(), nil } func (f *DataFomatter) isKeyComment(key string) bool { @@ -719,3 +710,10 @@ func (f *DataFomatter) isDictValue(comment map[string]interface{}) bool { } return false } + +func (f *DataFomatter) isIntValue(comment map[string]interface{}) bool { + if _, ok := comment["type"]; ok { + return comment["type"] == "int" + } + return false +} diff --git a/server/agent_config/template_json_test.go b/server/agent_config/template_json_test.go index 6cc62691ae2..596cf861f39 100644 --- a/server/agent_config/template_json_test.go +++ b/server/agent_config/template_json_test.go @@ -696,7 +696,7 @@ outputs: if err := dataFmt.LoadYAMLData(tt.args.yamlData); err != nil { t.Fatalf("Failed to init yaml data: %v", err) } - err := dataFmt.dictValueToString(dataFmt.mapData, "", keyToComment) + err := dataFmt.fmtVal("", dataFmt.mapData, keyToComment, true) if (err != nil) != tt.wantErr { t.Errorf("DictValueToString() error = %v, wantErr %v", err, tt.wantErr) return @@ -796,7 +796,7 @@ outputs: } else { dataFmt.mapData = tt.args.mapData } - err := dataFmt.stringToDictValue(dataFmt.mapData, "", keyToComment) + err := dataFmt.fmtVal("", dataFmt.mapData, keyToComment, false) if (err != nil) != tt.wantErr { t.Errorf("stringToDictValue() error = %v, wantErr %v", err, tt.wantErr) return @@ -1213,14 +1213,34 @@ func TestConvertJSONToYAMLAndValidate(t *testing.T) { resources: kubernetes: kubernetes_namespace: "111" +`), + wantErr: false, + }, + { + name: "case08", + args: args{ + jsonData: map[string]interface{}{ + "inputs": map[string]interface{}{ + "cbpf": map[string]interface{}{ + "af_packet": map[string]interface{}{ + "bond_interfaces": "- slave_interfaces: [eth0, eth1]\r\n- slave_interfaces: [eth2, eth3]", + }, + }, + }, + }, + }, + want: []byte(`inputs: + cbpf: + af_packet: + bond_interfaces: "- slave_interfaces:\n - eth0\n - eth1\n- slave_interfaces:\n - eth2\n - eth3\n" `), wantErr: false, }, } for _, tt := range tests { - // if tt.name != "case02" { - // continue - // } + if tt.name != "case06" { + continue + } t.Run(tt.name, func(t *testing.T) { got, err := ConvertJSONToYAMLAndValidate(tt.args.jsonData) if (err != nil) != tt.wantErr { diff --git a/server/controller/common/const.go b/server/controller/common/const.go index 464fe930ed4..8ad55b17f1e 100644 --- a/server/controller/common/const.go +++ b/server/controller/common/const.go @@ -705,3 +705,5 @@ const ( const TRISOLARIS_NODE_TYPE_MASTER = "master" const CLICK_HOUSE_VERSION = "24" + +const TAP_TYPE_VALUE_CLOUD_NETWORK = 3 diff --git a/server/controller/http/service/agent_group_config.go b/server/controller/http/service/agent_group_config.go index e06388676f1..25ca519ecdf 100644 --- a/server/controller/http/service/agent_group_config.go +++ b/server/controller/http/service/agent_group_config.go @@ -61,13 +61,13 @@ func (a *AgentGroupConfig) GetAgentGroupConfigTemplateJson() ([]byte, error) { return nil, err } var tapTypes []model.TapType - if err := dbInfo.Select("id", "name").Find(&tapTypes).Error; err != nil { + if err := dbInfo.Where("value != ?", common.TAP_TYPE_VALUE_CLOUD_NETWORK).Select("value", "name").Find(&tapTypes).Error; err != nil { return nil, err } tapTypInfos := make([]map[string]interface{}, len(tapTypes)) for i, tapType := range tapTypes { tapTypInfos[i] = map[string]interface{}{ - strconv.Itoa(tapType.ID): map[string]interface{}{ + strconv.Itoa(tapType.Value): map[string]interface{}{ "ch": tapType.Name, "en": tapType.Name, }, @@ -144,7 +144,7 @@ func (a *AgentGroupConfig) GetAgentGroupConfigTemplateJson() ([]byte, error) { // TODO get from ck l7Protocols := []string{ "HTTP", "HTTP2", "Dubbo", "gRPC", "SOFARPC", "FastCGI", "bRPC", "Tars", "Some/IP", "MySQL", "PostgreSQL", - "Oracle", "Redis", "MongoDB", "Kafka", "MQTT", "AMQP", "OpenWire", "NATS", "Pulsar", "ZMTP", "DNS", "TLS", "Custom"} + "Oracle", "Redis", "MongoDB", "Memcached", "Kafka", "MQTT", "AMQP", "OpenWire", "NATS", "Pulsar", "ZMTP", "DNS", "TLS", "Custom"} l7ProtocolsYamlBytes, err := yaml.Marshal(l7Protocols) if err != nil { return nil, err @@ -241,7 +241,11 @@ func (a *AgentGroupConfig) CreateAgentGroupConfig(groupLcuuid string, data inter if err != nil { return nil, err } - log.Infof("create agent group config, group lcuuid: %s, data: %#v, data type: %d", groupLcuuid, data, dataType, dbInfo.LogPrefixORGID) + if dataType == DataTypeJSON { + log.Infof("create agent group config, group lcuuid: %s, data: %#v, data type: %d", groupLcuuid, data, dataType, dbInfo.LogPrefixORGID) + } else { + log.Infof("create agent group config, group lcuuid: %s, data: %s, data type: %d", groupLcuuid, data.(string), dataType, dbInfo.LogPrefixORGID) + } var agentGroup model.VTapGroup if err := dbInfo.Where("lcuuid = ?", groupLcuuid).First(&agentGroup).Error; err != nil { return nil, err @@ -263,8 +267,6 @@ func (a *AgentGroupConfig) CreateAgentGroupConfig(groupLcuuid string, data inter if err := dbInfo.Create(newConfig).Error; err != nil { return nil, err } - - return a.GetAgentGroupConfig(groupLcuuid, dataType) } return nil, err } @@ -306,6 +308,8 @@ func (a *AgentGroupConfig) compatibleWithOldVersion(dbInfo *mysql.DB, groupLcuui if err := dbInfo.Create(vtapGroupConfig).Error; err != nil { log.Errorf("failed to create agent group lcuuid %s old version yaml: %v", groupLcuuid, err) } + } else { + log.Errorf("failed to get agent group config (lcuuid %s): %v", groupLcuuid, err) } return } diff --git a/server/controller/recorder/statsd/statsd.go b/server/controller/recorder/statsd/statsd.go index f2140b2809b..cef6cc94749 100644 --- a/server/controller/recorder/statsd/statsd.go +++ b/server/controller/recorder/statsd/statsd.go @@ -74,7 +74,7 @@ func (r *DomainStatsd) GetMetadata() *common.Metadata { } func (r *DomainStatsd) Start() { - log.Infof("start statsd", r.md.Domain.Lcuuid, r.md.LogPrefixes) + log.Info("start statsd", r.md.LogPrefixes) err := stats.RegisterCountableWithModulePrefix( counterModulePrefix, counterModuleTypeCloudTaskCost, @@ -159,7 +159,7 @@ func (r *SubDomainStatsd) GetMetadata() *common.Metadata { } func (r *SubDomainStatsd) Start() { - log.Infof("start statsd", r.md.LogPrefixes) + log.Info("start statsd", r.md.LogPrefixes) err := stats.RegisterCountableWithModulePrefix( counterModulePrefix, counterModuleTypeResourceSyncDelay,