Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: fix ip filter error #8693

Merged
merged 1 commit into from
Dec 13, 2024
Merged

fix: fix ip filter error #8693

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions server/querier/engine/clickhouse/clickhouse_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ var (
output: []string{"WITH toStartOfInterval(time, toIntervalSecond(60)) + toIntervalSecond(arrayJoin([0]) * 60) AS `_toi` SELECT toUnixTimestamp(`_toi`) AS `toi`, divide(plus(SUM(byte_tx+byte_rx), 100), 60) AS `persecond_max_byte_100` FROM flow_log.`l4_flow_log` GROUP BY `toi` LIMIT 1"},
}, {
input: "select resource_gl0_0,ip_0 from l7_flow_log where ip_0='1.1.1.1' and resource_gl0_0='abc' and resource_gl0_0 regexp 'abc' and resource_gl0_id_0=2 group by resource_gl0_0,ip_0",
output: []string{"SELECT if(auto_instance_type_0 in (0,255),if(is_ipv4=1, IPv4NumToString(ip4_0), IPv6NumToString(ip6_0)),dictGet('flow_tag.device_map', 'name', (toUInt64(auto_instance_type_0),toUInt64(auto_instance_id_0)))) AS `resource_gl0_0`, if(is_ipv4=1, IPv4NumToString(ip4_0), IPv6NumToString(ip6_0)) AS `ip_0`, auto_instance_type_0 AS `resource_gl0_type_0` FROM flow_log.`l7_flow_log` PREWHERE (((if(is_ipv4=1, ip4_0 = toIPv4('1.1.1.1'), ip6_0 = toIPv6('1.1.1.1'))))) AND (if(auto_instance_type_0 in (0,255),if(is_ipv4=1, IPv4NumToString(ip4_0), IPv6NumToString(ip6_0)) = 'abc',(toUInt64(auto_instance_id_0),toUInt64(auto_instance_type_0)) GLOBAL IN (SELECT deviceid,devicetype FROM flow_tag.device_map WHERE name = 'abc' AND devicetype not in (101,102)))) AND (if(auto_instance_type_0 in (0,255),match(if(is_ipv4=1, IPv4NumToString(ip4_0), IPv6NumToString(ip6_0)),'abc'),(toUInt64(auto_instance_id_0),toUInt64(auto_instance_type_0)) GLOBAL IN (SELECT deviceid,devicetype FROM flow_tag.device_map WHERE match(name,'abc') AND devicetype not in (101,102)))) AND (if(auto_instance_type_0 in (0,255),subnet_id_0 = 2,auto_instance_id_0 = 2)) GROUP BY if(auto_instance_type_0 in (0,255),if(is_ipv4=1, IPv4NumToString(ip4_0), IPv6NumToString(ip6_0)),dictGet('flow_tag.device_map', 'name', (toUInt64(auto_instance_type_0),toUInt64(auto_instance_id_0)))) AS `resource_gl0_0`, `resource_gl0_type_0`, if(is_ipv4=1, IPv4NumToString(ip4_0), IPv6NumToString(ip6_0)) AS `ip_0` LIMIT 10000"},
output: []string{"SELECT if(auto_instance_type_0 in (0,255),if(is_ipv4=1, IPv4NumToString(ip4_0), IPv6NumToString(ip6_0)),dictGet('flow_tag.device_map', 'name', (toUInt64(auto_instance_type_0),toUInt64(auto_instance_id_0)))) AS `resource_gl0_0`, if(is_ipv4=1, IPv4NumToString(ip4_0), IPv6NumToString(ip6_0)) AS `ip_0`, auto_instance_type_0 AS `resource_gl0_type_0` FROM flow_log.`l7_flow_log` PREWHERE (((if(is_ipv4=1, ip4_0 = toIPv4OrNull('1.1.1.1'), ip6_0 = toIPv6OrNull('1.1.1.1'))))) AND (if(auto_instance_type_0 in (0,255),if(is_ipv4=1, IPv4NumToString(ip4_0), IPv6NumToString(ip6_0)) = 'abc',(toUInt64(auto_instance_id_0),toUInt64(auto_instance_type_0)) GLOBAL IN (SELECT deviceid,devicetype FROM flow_tag.device_map WHERE name = 'abc' AND devicetype not in (101,102)))) AND (if(auto_instance_type_0 in (0,255),match(if(is_ipv4=1, IPv4NumToString(ip4_0), IPv6NumToString(ip6_0)),'abc'),(toUInt64(auto_instance_id_0),toUInt64(auto_instance_type_0)) GLOBAL IN (SELECT deviceid,devicetype FROM flow_tag.device_map WHERE match(name,'abc') AND devicetype not in (101,102)))) AND (if(auto_instance_type_0 in (0,255),subnet_id_0 = 2,auto_instance_id_0 = 2)) GROUP BY if(auto_instance_type_0 in (0,255),if(is_ipv4=1, IPv4NumToString(ip4_0), IPv6NumToString(ip6_0)),dictGet('flow_tag.device_map', 'name', (toUInt64(auto_instance_type_0),toUInt64(auto_instance_id_0)))) AS `resource_gl0_0`, `resource_gl0_type_0`, if(is_ipv4=1, IPv4NumToString(ip4_0), IPv6NumToString(ip6_0)) AS `ip_0` LIMIT 10000"},
}, {
input: "select pod_service_0 from l7_flow_log where pod_service_0 !='xx' group by pod_service_0",
output: []string{"SELECT dictGet('flow_tag.device_map', 'name', (toUInt64(11),toUInt64(service_id_0))) AS `pod_service_0` FROM flow_log.`l7_flow_log` PREWHERE (not(toUInt64(service_id_0) GLOBAL IN (SELECT deviceid FROM flow_tag.device_map WHERE name = 'xx' AND devicetype=11))) AND (service_id_0!=0) GROUP BY dictGet('flow_tag.device_map', 'name', (toUInt64(11),toUInt64(service_id_0))) AS `pod_service_0` LIMIT 10000"},
Expand All @@ -166,7 +166,7 @@ var (
output: []string{"SELECT byte_tx+byte_rx AS `123` FROM flow_log.`l4_flow_log` PREWHERE 1 = 1 GROUP BY `123` ORDER BY `123` asc LIMIT 1"},
}, {
input: "select byte from l4_flow_log where ip>=('1.1.1.1/24','2.2.2.2') and ip<='::/24'",
output: []string{"SELECT byte_tx+byte_rx AS `byte` FROM flow_log.`l4_flow_log` PREWHERE (((if(is_ipv4=1, ip4 >= toIPv4('1.1.1.255'), ip6 >= toIPv6('1.1.1.255'))) OR (if(is_ipv4=1, ip4 >= toIPv4('2.2.2.2'), ip6 >= toIPv6('2.2.2.2'))))) AND (((if(is_ipv4=1, ip4 <= toIPv4('::'), ip6 <= toIPv6('::'))))) LIMIT 10000"},
output: []string{"SELECT byte_tx+byte_rx AS `byte` FROM flow_log.`l4_flow_log` PREWHERE (((if(is_ipv4=1, ip4 >= toIPv4OrNull('1.1.1.255'), ip6 >= toIPv6OrNull('1.1.1.255'))) OR (if(is_ipv4=1, ip4 >= toIPv4OrNull('2.2.2.2'), ip6 >= toIPv6OrNull('2.2.2.2'))))) AND (((if(is_ipv4=1, ip4 <= toIPv4OrNull('::'), ip6 <= toIPv6OrNull('::'))))) LIMIT 10000"},
}, {
input: "select `k8s.label.statefulset.kubernetes.io/pod-name_0` from l4_flow_log where `k8s.label.statefulset.kubernetes.io/pod-name_0`='opensource-loki-0' group by `k8s.label.statefulset.kubernetes.io/pod-name_0`",
output: []string{"SELECT if(dictGet('flow_tag.pod_service_k8s_label_map', 'value', (toUInt64(service_id_0),'statefulset.kubernetes.io/pod-name'))!='', dictGet('flow_tag.pod_service_k8s_label_map', 'value', (toUInt64(service_id_0),'statefulset.kubernetes.io/pod-name')), dictGet('flow_tag.pod_k8s_label_map', 'value', (toUInt64(pod_id_0),'statefulset.kubernetes.io/pod-name')) ) AS `k8s.label.statefulset.kubernetes.io/pod-name_0` FROM flow_log.`l4_flow_log` PREWHERE ((toUInt64(service_id_0) GLOBAL IN (SELECT id FROM flow_tag.pod_service_k8s_label_map WHERE value = 'opensource-loki-0' and key='statefulset.kubernetes.io/pod-name')) OR (toUInt64(pod_id_0) GLOBAL IN (SELECT id FROM flow_tag.pod_k8s_label_map WHERE value = 'opensource-loki-0' and key='statefulset.kubernetes.io/pod-name'))) AND (((toUInt64(service_id_0) GLOBAL IN (SELECT id FROM flow_tag.pod_service_k8s_label_map WHERE key='statefulset.kubernetes.io/pod-name')) OR (toUInt64(pod_id_0) GLOBAL IN (SELECT id FROM flow_tag.pod_k8s_label_map WHERE key='statefulset.kubernetes.io/pod-name')))) GROUP BY `k8s.label.statefulset.kubernetes.io/pod-name_0` LIMIT 10000"},
Expand Down
16 changes: 8 additions & 8 deletions server/querier/engine/clickhouse/tag/translation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -432,7 +432,7 @@ func GenerateTagResoureMap() map[string]map[string]*Tag {
"default": NewTag(
fmt.Sprintf("if(is_ipv4=1, IPv4NumToString(%s), IPv6NumToString(%s))", ip4Suffix, ip6Suffix),
"",
fmt.Sprintf("if(is_ipv4=1, %s %%s toIPv4(%%s), %s %%s toIPv6(%%s))", ip4Suffix, ip6Suffix),
fmt.Sprintf("if(is_ipv4=1, %s %%s toIPv4OrNull(%%s), %s %%s toIPv6OrNull(%%s))", ip4Suffix, ip6Suffix),
"",
), "mask": NewTag(
"if(is_ipv4, IPv4NumToString(bitAnd("+ip4Suffix+", %v)), IPv6NumToString(bitAnd("+ip6Suffix+", toFixedString(unhex('%s'), 16))))",
Expand Down Expand Up @@ -894,7 +894,7 @@ func GenerateTagResoureMap() map[string]map[string]*Tag {
"default": NewTag(
NatRealIP4Suffix,
"",
fmt.Sprintf("%s %%s toIPv4(%%s)", NatRealIP4Suffix),
fmt.Sprintf("%s %%s toIPv4OrNull(%%s)", NatRealIP4Suffix),
"",
)}
}
Expand All @@ -903,42 +903,42 @@ func GenerateTagResoureMap() map[string]map[string]*Tag {
"default": NewTag(
"",
"",
"(if(tunnel_is_ipv4=1, tunnel_tx_ip4_0 %s toIPv4(%s), tunnel_tx_ip6_0 %s toIPv6(%s)) OR if(tunnel_is_ipv4=1, tunnel_tx_ip4_1 %s toIPv4(%s), tunnel_tx_ip6_1 %s toIPv6(%s)))",
"(if(tunnel_is_ipv4=1, tunnel_tx_ip4_0 %s toIPv4OrNull(%s), tunnel_tx_ip6_0 %s toIPv6OrNull(%s)) OR if(tunnel_is_ipv4=1, tunnel_tx_ip4_1 %s toIPv4OrNull(%s), tunnel_tx_ip6_1 %s toIPv6OrNull(%s)))",
"",
)}
tagResourceMap["tunnel_tx_ip_0"] = map[string]*Tag{
"default": NewTag(
"if(tunnel_is_ipv4, IPv4NumToString(tunnel_tx_ip4_0), IPv6NumToString(tunnel_tx_ip6_0))",
"",
"if(tunnel_is_ipv4=1, tunnel_tx_ip4_0 %s toIPv4(%s), tunnel_tx_ip6_0 %s toIPv6(%s))",
"if(tunnel_is_ipv4=1, tunnel_tx_ip4_0 %s toIPv4OrNull(%s), tunnel_tx_ip6_0 %s toIPv6OrNull(%s))",
"",
)}
tagResourceMap["tunnel_tx_ip_1"] = map[string]*Tag{
"default": NewTag(
"if(tunnel_is_ipv4, IPv4NumToString(tunnel_tx_ip4_1), IPv6NumToString(tunnel_tx_ip6_1))",
"",
"if(tunnel_is_ipv4=1, tunnel_tx_ip4_1 %s toIPv4(%s), tunnel_tx_ip6_1 %s toIPv6(%s))",
"if(tunnel_is_ipv4=1, tunnel_tx_ip4_1 %s toIPv4OrNull(%s), tunnel_tx_ip6_1 %s toIPv6OrNull(%s))",
"",
)}
tagResourceMap["tunnel_rx_ip"] = map[string]*Tag{
"default": NewTag(
"",
"",
"(if(tunnel_is_ipv4=1, tunnel_rx_ip4_0 %s toIPv4(%s), tunnel_rx_ip6_0 %s toIPv6(%s)) OR if(tunnel_is_ipv4=1, tunnel_rx_ip4_1 %s toIPv4(%s), tunnel_rx_ip6_1 %s toIPv6(%s)))",
"(if(tunnel_is_ipv4=1, tunnel_rx_ip4_0 %s toIPv4OrNull(%s), tunnel_rx_ip6_0 %s toIPv6OrNull(%s)) OR if(tunnel_is_ipv4=1, tunnel_rx_ip4_1 %s toIPv4OrNull(%s), tunnel_rx_ip6_1 %s toIPv6OrNull(%s)))",
"",
)}
tagResourceMap["tunnel_rx_ip_0"] = map[string]*Tag{
"default": NewTag(
"if(tunnel_is_ipv4, IPv4NumToString(tunnel_rx_ip4_0), IPv6NumToString(tunnel_rx_ip6_0))",
"",
"if(tunnel_is_ipv4=1, tunnel_rx_ip4_0 %s toIPv4(%s), tunnel_rx_ip6_0 %s toIPv6(%s))",
"if(tunnel_is_ipv4=1, tunnel_rx_ip4_0 %s toIPv4OrNull(%s), tunnel_rx_ip6_0 %s toIPv6OrNull(%s))",
"",
)}
tagResourceMap["tunnel_rx_ip_1"] = map[string]*Tag{
"default": NewTag(
"if(tunnel_is_ipv4, IPv4NumToString(tunnel_rx_ip4_1), IPv6NumToString(tunnel_rx_ip6_1))",
"",
"if(tunnel_is_ipv4=1, tunnel_rx_ip4_1 %s toIPv4(%s), tunnel_rx_ip6_1 %s toIPv6(%s))",
"if(tunnel_is_ipv4=1, tunnel_rx_ip4_1 %s toIPv4OrNull(%s), tunnel_rx_ip6_1 %s toIPv6OrNull(%s))",
"",
)}
// 开始时间
Expand Down
Loading