From 3c264b0b293c9abc3c25ce041dfea9da40ed0770 Mon Sep 17 00:00:00 2001 From: zhengya Date: Thu, 12 Dec 2024 12:13:15 +0800 Subject: [PATCH] fix: agent group api responses invalid int value --- server/agent_config/migrator.go | 11 ++- server/agent_config/template_json.go | 83 +++++++++---------- server/agent_config/template_json_test.go | 10 +-- .../http/service/agent_group_config.go | 2 +- server/controller/recorder/statsd/statsd.go | 4 +- 5 files changed, 56 insertions(+), 54 deletions(-) diff --git a/server/agent_config/migrator.go b/server/agent_config/migrator.go index 859bc8cb9d6..81de66999d1 100644 --- a/server/agent_config/migrator.go +++ b/server/agent_config/migrator.go @@ -87,7 +87,7 @@ func (m *Upgrader) lowerToHigher(lowerVerData interface{}, ancestor string, high lowers := m.higherVerToLowerVerKeys[higher] if len(lowers) > 1 { for _, lower := range lowers { - log.Warnf("%s has been upgraded to %s", lower, higher) // TODO return? + log.Warnf("%s has been upgraded to %s, please configure it manually", lower, higher) // TODO return? } } else { m.setNestedValue(higherVerData, higher, m.fmtLowerVersionValue(newAncestor, value)) @@ -199,6 +199,13 @@ func (m *Upgrader) fmtLowerVersionValue(longKey string, value interface{}) inter return value } } + // TODO + // 升级 static_config.ebpf.uprobe-process-name-regexs.golang-symbol 时，需要： + // 1. 将 inputs.proc.symbol_table.golang_specific.enabled 设置为 true + // 2. 新增一条 inputs.proc.process_matcher + // 升级 static_config.os-proc-sync-tagged-only 时，需要： + // 1. 将 inputs.proc.process_matcher 里所有的 only_with_tag 设置为 static_config.os-proc-sync-tagged-only + return m.convDictData(longKey, value) } @@ -552,7 +559,7 @@ func (p *MigrationToolData) fmtDictValKeyMap() { // "only_with_tag": "static_config.os-proc-sync-tagged-only", // TODO "ignore": "action", "rewrite_name": "rewrite-name", - // "enabled_features": []string{"static_config.ebpf.on-cpu-profile.regex", "static_config.ebpf.off-cpu-profile.regex"}, // 无法降级 + // "enabled_features": []string{"static_config.ebpf.on-cpu-profile.regex", "static_config.ebpf.off-cpu-profile.regex"}, // 无法升降级 }, "inputs.cbpf.af_packet.bond_interfaces": { diff --git a/server/agent_config/template_json.go b/server/agent_config/template_json.go index 9588f7b90b3..dca906421b0 100644 --- a/server/agent_config/template_json.go +++ b/server/agent_config/template_json.go @@ -554,7 +554,7 @@ func (f *DataFomatter) LoadMapData(data map[string]interface{}) error { return fmt.Errorf("unmarshal yaml to node error: %v", err) } - err = f.stringToDictValue(f.mapData, "", f.keyToComment) + err = f.fmtVal("", f.mapData, f.keyToComment, false) if err != nil { return fmt.Errorf("convert dict value to string error: %v", err) } @@ -603,17 +603,12 @@ func (f *DataFomatter) mapToJSON(keyToComment KeyToComment) ([]byte, error) { } func (f *DataFomatter) fmtMapValAndRefresh(keyToComment KeyToComment, dictValToStr bool) error { - var err error - if dictValToStr { - err = f.dictValueToString(f.mapData, "", keyToComment) - } else { - err = f.stringToDictValue(f.mapData, "", keyToComment) - } + err := f.fmtVal("", f.mapData, keyToComment, dictValToStr) if err != nil { return fmt.Errorf("convert dict value to string error: %v", err) } // refresh yamlNode if changes dict value - if _, ok := keyToComment["changesDictValue"]; ok { + if _, ok := keyToComment["valueChanged"]; ok { yamlBytes, err := yaml.Marshal(f.mapData) if err != nil { return fmt.Errorf("marshal map to yaml error: %v", err) @@ -628,54 +623,47 @@ func (f *DataFomatter) fmtMapValAndRefresh(keyToComment KeyToComment, dictValToS return nil } -func (f *DataFomatter) dictValueToString(data interface{}, ancestors string, keyToComment KeyToComment) error { - switch data := data.(type) { - case map[string]interface{}: - for key, value := range data { - newAncestors := f.appendAncestor(ancestors, key) - if !f.isKeyComment(key) { - if f.isDictValue(keyToComment[newAncestors]) { - valueStr, err := f.dictToString(value) - if err != nil { - return fmt.Errorf("convert dict value to string error: %v, key: %s", err, newAncestors) - } - data[key] = valueStr - keyToComment["changesDictValue"] = make(map[string]interface{}) - } - f.dictValueToString(value, newAncestors, keyToComment) - } - } - default: - return nil - } - return nil -} - -func (f *DataFomatter) stringToDictValue(data interface{}, ancestors string, keyToComment KeyToComment) error { +func (f *DataFomatter) fmtVal(ancestors string, data interface{}, keyToComment KeyToComment, dictValToStr bool) error { switch data := data.(type) { case map[string]interface{}: for key, value := range data { newAncestors := f.appendAncestor(ancestors, key) if !f.isKeyComment(key) { if f.isDictValue(keyToComment[newAncestors]) { - if strings.HasPrefix(strings.TrimSpace(value.(string)), "-") { - var valueMap map[string]interface{} - err := yaml.Unmarshal([]byte(key+":\n"+value.(string)), &valueMap) + if dictValToStr { + valueStr, err := f.dictToString(value) if err != nil { - return fmt.Errorf("unmarshal string to map error: %v, key: %s", err, newAncestors) + return fmt.Errorf("convert dict value to string error: %v, key: %s", err, newAncestors) } - data[key] = valueMap[key] + data[key] = valueStr } else { - var valueMap map[string]interface{} - err := yaml.Unmarshal([]byte(value.(string)), &valueMap) - if err != nil { - return fmt.Errorf("unmarshal string to map error: %v, key: %s", err, newAncestors) + if strings.HasPrefix(strings.TrimSpace(value.(string)), "-") { + var valueMap map[string]interface{} + err := yaml.Unmarshal([]byte(key+":\n"+value.(string)), &valueMap) + if err != nil { + return fmt.Errorf("unmarshal string to map error: %v, key: %s", err, newAncestors) + } + data[key] = valueMap[key] + } else { + var valueMap map[string]interface{} + err := yaml.Unmarshal([]byte(value.(string)), &valueMap) + if err != nil { + return fmt.Errorf("unmarshal string to map error: %v, key: %s", err, newAncestors) + } + data[key] = valueMap } - data[key] = valueMap } - keyToComment["changesDictValue"] = make(map[string]interface{}) + keyToComment["valueChanged"] = make(map[string]interface{}) + } else if f.isIntValue(keyToComment[newAncestors]) { + switch value := value.(type) { + case int: + data[key] = value + case float64: + data[key] = int(value) + keyToComment["valueChanged"] = make(map[string]interface{}) + } } - f.stringToDictValue(value, newAncestors, keyToComment) + f.fmtVal(newAncestors, value, keyToComment, dictValToStr) } } default: @@ -722,3 +710,10 @@ func (f *DataFomatter) isDictValue(comment map[string]interface{}) bool { } return false } + +func (f *DataFomatter) isIntValue(comment map[string]interface{}) bool { + if _, ok := comment["type"]; ok { + return comment["type"] == "int" + } + return false +} diff --git a/server/agent_config/template_json_test.go b/server/agent_config/template_json_test.go index 74538cf2158..596cf861f39 100644 --- a/server/agent_config/template_json_test.go +++ b/server/agent_config/template_json_test.go @@ -696,7 +696,7 @@ outputs: if err := dataFmt.LoadYAMLData(tt.args.yamlData); err != nil { t.Fatalf("Failed to init yaml data: %v", err) } - err := dataFmt.dictValueToString(dataFmt.mapData, "", keyToComment) + err := dataFmt.fmtVal("", dataFmt.mapData, keyToComment, true) if (err != nil) != tt.wantErr { t.Errorf("DictValueToString() error = %v, wantErr %v", err, tt.wantErr) return @@ -796,7 +796,7 @@ outputs: } else { dataFmt.mapData = tt.args.mapData } - err := dataFmt.stringToDictValue(dataFmt.mapData, "", keyToComment) + err := dataFmt.fmtVal("", dataFmt.mapData, keyToComment, false) if (err != nil) != tt.wantErr { t.Errorf("stringToDictValue() error = %v, wantErr %v", err, tt.wantErr) return @@ -1238,9 +1238,9 @@ func TestConvertJSONToYAMLAndValidate(t *testing.T) { }, } for _, tt := range tests { - // if tt.name != "case02" { - // continue - // } + if tt.name != "case06" { + continue + } t.Run(tt.name, func(t *testing.T) { got, err := ConvertJSONToYAMLAndValidate(tt.args.jsonData) if (err != nil) != tt.wantErr { diff --git a/server/controller/http/service/agent_group_config.go b/server/controller/http/service/agent_group_config.go index dc27cb43b17..25ca519ecdf 100644 --- a/server/controller/http/service/agent_group_config.go +++ b/server/controller/http/service/agent_group_config.go @@ -144,7 +144,7 @@ func (a *AgentGroupConfig) GetAgentGroupConfigTemplateJson() ([]byte, error) { // TODO get from ck l7Protocols := []string{ "HTTP", "HTTP2", "Dubbo", "gRPC", "SOFARPC", "FastCGI", "bRPC", "Tars", "Some/IP", "MySQL", "PostgreSQL", - "Oracle", "Redis", "MongoDB", "Kafka", "MQTT", "AMQP", "OpenWire", "NATS", "Pulsar", "ZMTP", "DNS", "TLS", "Custom"} + "Oracle", "Redis", "MongoDB", "Memcached", "Kafka", "MQTT", "AMQP", "OpenWire", "NATS", "Pulsar", "ZMTP", "DNS", "TLS", "Custom"} l7ProtocolsYamlBytes, err := yaml.Marshal(l7Protocols) if err != nil { return nil, err diff --git a/server/controller/recorder/statsd/statsd.go b/server/controller/recorder/statsd/statsd.go index f2140b2809b..cef6cc94749 100644 --- a/server/controller/recorder/statsd/statsd.go +++ b/server/controller/recorder/statsd/statsd.go @@ -74,7 +74,7 @@ func (r *DomainStatsd) GetMetadata() *common.Metadata { } func (r *DomainStatsd) Start() { - log.Infof("start statsd", r.md.Domain.Lcuuid, r.md.LogPrefixes) + log.Info("start statsd", r.md.LogPrefixes) err := stats.RegisterCountableWithModulePrefix( counterModulePrefix, counterModuleTypeCloudTaskCost, @@ -159,7 +159,7 @@ func (r *SubDomainStatsd) GetMetadata() *common.Metadata { } func (r *SubDomainStatsd) Start() { - log.Infof("start statsd", r.md.LogPrefixes) + log.Info("start statsd", r.md.LogPrefixes) err := stats.RegisterCountableWithModulePrefix( counterModulePrefix, counterModuleTypeResourceSyncDelay,