Skip to content

Commit

Permalink
fix: querier group by icon_id error
Browse files Browse the repository at this point in the history
  • Loading branch information
@xiaochaoren1 @SongZhen0704
xiaochaoren1 authored and SongZhen0704 committed Dec 23, 2024
1 parent 3601b21 commit 6b98cb8
Show file tree
Hide file tree
Showing 6 changed files with 27 additions and 5 deletions.
2 changes: 1 addition & 1 deletion server/querier/engine/clickhouse/clickhouse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1343,7 +1343,7 @@ func (e *CHEngine) TransDerivativeGroupBy(groups sqlparser.GroupBy) error {
colName, ok := group.(*sqlparser.ColName)
if ok {
groupTag := sqlparser.String(colName)
if !strings.Contains(groupTag, "time") {
if !strings.Contains(groupTag, "time") && !strings.Contains(groupTag, "node_type") && !strings.Contains(groupTag, "icon_id") {
groupSlice = append(groupSlice, groupTag)
}
}
Expand Down
4 changes: 2 additions & 2 deletions server/querier/engine/clickhouse/clickhouse_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -476,8 +476,8 @@ var (
name: "test_chost_hostname_ip",
db: "flow_metrics",
datasource: "1m",
input: "SELECT region_0 as region_0, chost_hostname_id_0, chost_ip_id_0, chost_hostname_0, chost_ip_0, node_type(chost_ip_0), node_type(chost_hostname_0) as `client_node_type`, icon_id(chost_hostname_0) as `client_icon_id` FROM `vtap_flow_edge_port` WHERE time>=1705040184 AND time<=1705045184 AND chost_hostname_0 != 'a' AND chost_hostname_id_0 != 1 AND chost_ip_id_0 != 2 GROUP BY region_0, chost_hostname_id_0, chost_ip_id_0, chost_hostname_0, chost_ip_0, `client_node_type` limit 5",
output: []string{"WITH if(l3_device_type_0=1, dictGet('flow_tag.device_map', 'icon_id', (toUInt64(1),toUInt64(l3_device_id_0))), 0) AS `client_icon_id` SELECT dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_0))) AS `region_0`, if(l3_device_type_0=1,l3_device_id_0, 0) AS `chost_hostname_id_0`, if(l3_device_type_0=1,l3_device_id_0, 0) AS `chost_ip_id_0`, if(l3_device_type_0=1, dictGet('flow_tag.device_map', 'hostname', (toUInt64(1),toUInt64(l3_device_id_0))), '') AS `chost_hostname_0`, if(l3_device_type_0=1, dictGet('flow_tag.device_map', 'ip', (toUInt64(1),toUInt64(l3_device_id_0))), '') AS `chost_ip_0`, 'chost', 'chost' AS `client_node_type`, `client_icon_id` FROM flow_metrics.`network_map.1m` PREWHERE `time` >= 1705040184 AND `time` <= 1705045184 AND (not(toUInt64(l3_device_id_0) GLOBAL IN (SELECT deviceid FROM flow_tag.device_map WHERE hostname = 'a' AND devicetype=1) AND l3_device_type_0=1)) AND (not(l3_device_id_0 = 1 AND l3_device_type_0=1)) AND (not(l3_device_id_0 = 2 AND l3_device_type_0=1)) AND (l3_device_id_0!=0 AND l3_device_type_0=1) AND (l3_device_id_0!=0 AND l3_device_type_0=1) AND (l3_device_id_0!=0 AND l3_device_type_0=1) AND (l3_device_id_0!=0 AND l3_device_type_0=1) GROUP BY dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_0))) AS `region_0`, if(l3_device_type_0=1,l3_device_id_0, 0) AS `chost_hostname_id_0`, if(l3_device_type_0=1,l3_device_id_0, 0) AS `chost_ip_id_0`, if(l3_device_type_0=1, dictGet('flow_tag.device_map', 'hostname', (toUInt64(1),toUInt64(l3_device_id_0))), '') AS `chost_hostname_0`, if(l3_device_type_0=1, dictGet('flow_tag.device_map', 'ip', (toUInt64(1),toUInt64(l3_device_id_0))), '') AS `chost_ip_0` LIMIT 5"},
input: "SELECT region_0 as region_0, chost_hostname_id_0, chost_ip_id_0, chost_hostname_0, chost_ip_0, node_type(chost_hostname_0) as `client_node_type`, icon_id(chost_hostname_0) as `client_icon_id` FROM `vtap_flow_edge_port` WHERE time>=1705040184 AND time<=1705045184 AND chost_hostname_0 != 'a' AND chost_hostname_id_0 != 1 AND chost_ip_id_0 != 2 GROUP BY region_0, chost_hostname_id_0, chost_ip_id_0, chost_hostname_0, chost_ip_0, `client_node_type` limit 5",
output: []string{"WITH if(l3_device_type_0=1, dictGet('flow_tag.device_map', 'icon_id', (toUInt64(1),toUInt64(l3_device_id_0))), 0) AS `client_icon_id` SELECT dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_0))) AS `region_0`, if(l3_device_type_0=1,l3_device_id_0, 0) AS `chost_hostname_id_0`, if(l3_device_type_0=1,l3_device_id_0, 0) AS `chost_ip_id_0`, if(l3_device_type_0=1, dictGet('flow_tag.device_map', 'hostname', (toUInt64(1),toUInt64(l3_device_id_0))), '') AS `chost_hostname_0`, if(l3_device_type_0=1, dictGet('flow_tag.device_map', 'ip', (toUInt64(1),toUInt64(l3_device_id_0))), '') AS `chost_ip_0`, 'chost' AS `client_node_type`, `client_icon_id` FROM flow_metrics.`network_map.1m` PREWHERE `time` >= 1705040184 AND `time` <= 1705045184 AND (not(toUInt64(l3_device_id_0) GLOBAL IN (SELECT deviceid FROM flow_tag.device_map WHERE hostname = 'a' AND devicetype=1) AND l3_device_type_0=1)) AND (not(l3_device_id_0 = 1 AND l3_device_type_0=1)) AND (not(l3_device_id_0 = 2 AND l3_device_type_0=1)) AND (l3_device_id_0!=0 AND l3_device_type_0=1) AND (l3_device_id_0!=0 AND l3_device_type_0=1) AND (l3_device_id_0!=0 AND l3_device_type_0=1) AND (l3_device_id_0!=0 AND l3_device_type_0=1) GROUP BY `client_icon_id`, dictGet('flow_tag.region_map', 'name', (toUInt64(region_id_0))) AS `region_0`, if(l3_device_type_0=1,l3_device_id_0, 0) AS `chost_hostname_id_0`, if(l3_device_type_0=1,l3_device_id_0, 0) AS `chost_ip_id_0`, if(l3_device_type_0=1, dictGet('flow_tag.device_map', 'hostname', (toUInt64(1),toUInt64(l3_device_id_0))), '') AS `chost_hostname_0`, if(l3_device_type_0=1, dictGet('flow_tag.device_map', 'ip', (toUInt64(1),toUInt64(l3_device_id_0))), '') AS `chost_ip_0` LIMIT 5"},
}, {
name: "test_pod_node_hostname_ip",
db: "flow_metrics",
Expand Down
1 change: 0 additions & 1 deletion server/querier/engine/clickhouse/filter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -587,7 +587,6 @@ func (t *WhereTag) Trans(expr sqlparser.Expr, w *Where, e *CHEngine) (view.Node,
checkTag := strings.TrimSuffix(t.Tag, "_id")
if slices.Contains(chCommon.SHOW_TAG_VALUE_MAP[table], checkTag) {
if strings.HasSuffix(t.Tag, "_id") {

if checkTag == strings.TrimSuffix(table, "_map") || checkTag == common.CHOST_HOSTNAME || checkTag == common.CHOST_IP {
tagItem, ok := tag.GetTag("value", db, table, "default")
if ok {
Expand Down
6 changes: 6 additions & 0 deletions server/querier/engine/clickhouse/function.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1042,6 +1042,12 @@ func (f *TagFunction) Format(m *view.Model) {
resourceNameSuffix := resourceStr + suffix
if f.Args[0] == resourceNameSuffix {
m.AddGroup(&view.Group{Value: fmt.Sprintf("`%s`", strings.Trim(f.Alias, "`"))})
} else {
for resource, _ := range tag.HOSTNAME_IP_DEVICE_MAP {
if slices.Contains([]string{common.CHOST_HOSTNAME, common.CHOST_IP}, resource) && f.Args[0] == resource+suffix {
m.AddGroup(&view.Group{Value: fmt.Sprintf("`%s`", strings.Trim(f.Alias, "`"))})
}
}
}
}
}
Expand Down
1 change: 0 additions & 1 deletion server/querier/engine/clickhouse/view/function.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,6 @@ func GetFunc(name string) Function {
default:
return &DefaultFunction{Name: name}
}
return nil
}

type Function interface {
Expand Down
18 changes: 18 additions & 0 deletions server/querier/engine/clickhouse/view/view.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package view

import (
"bytes"
"slices"
"strings"

"github.com/deepflowio/deepflow/server/querier/common"
Expand Down Expand Up @@ -260,6 +261,23 @@ func (v *View) trans() {
groupsLevelInner = append(groupsLevelInner, group)
}
}
// The inner tag should be in the outer group
groupList := []string{}
for _, group := range groupsLevelMetrics {
groupList = append(groupList, group.(*Group).Value)
}
for _, node := range v.Model.Tags.tags {
switch tag := node.(type) {
case *Tag:
if tag.Flag == NODE_FLAG_METRICS {
// outer group
if tag.Alias != "" && !slices.Contains(groupList, tag.Alias) {
groupsLevelMetrics = append(groupsLevelMetrics, &Group{Value: tag.Alias})
}
}
}
}

if v.Model.MetricsLevelFlag == MODEL_METRICS_LEVEL_FLAG_UNLAY {
// 计算层不拆层
// 里层tag+外层metric
Expand Down

0 comments on commit 6b98cb8

Please sign in to comment.