NETOBSERV-1927 scopes config

config/sample-config.yaml

@@ -982,6 +982,126 @@ frontend:
       name: Flow RTT
       component: number
       hint: Specify a TCP smoothed Round Trip Time in nanoseconds.
+  scopes:
+    - id: cluster
+      name: Cluster
+      shortName: Cl
+      description: Cluster name or identifier
+      labels:
+        - K8S_ClusterName
+      feature: multiCluster
+      filter: cluster_name
+      stepInto: zone
+    - id: zone
+      name: Zone
+      shortName: AZ
+      description: Availability zone
+      labels:
+        - SrcK8S_Zone
+        - DstK8S_Zone
+      feature: zones
+      groups:
+        - clusters
+      filters:
+        - src_zone
+        - dst_zone
+      stepInto: host
+    - id: host
+      name: Node
+      description: Node on which the resources are running
+      labels: 
+        - SrcK8S_HostName
+        - DstK8S_HostName
+      groups:
+        - clusters
+        - zones
+        - clusters+zones
+      filters:
+        - src_host_name
+        - dst_host_name
+      stepInto: resource
+    - id: namespace
+      name: Namespace
+      shortName: NS
+      description: Resource namespace
+      labels: 
+        - SrcK8S_Namespace
+        - DstK8S_Namespace
+      groups:
+        - clusters
+        - clusters+zones
+        - clusters+hosts
+        - zones
+        - zones+hosts
+        - hosts
+      filters:
+        - src_namespace
+        - dst_namespace
+      stepInto: owner
+    - id: owner
+      name: Owner
+      shortName: Own
+      description: Controller owner, such as a Deployment
+      labels: 
+        - SrcK8S_OwnerName
+        - SrcK8S_OwnerType
+        - DstK8S_OwnerName
+        - DstK8S_OwnerType
+        - SrcK8S_Namespace
+        - DstK8S_Namespace
+      groups:
+        - clusters
+        - clusters+zones
+        - clusters+hosts
+        - clusters+namespaces
+        - zones
+        - zones+hosts
+        - zones+namespaces
+        - hosts
+        - hosts+namespaces
+        - namespaces
+      filters:
+        - src_owner_name
+        - dst_owner_name
+      stepInto: resource
+    - id: resource
+      name: Resource
+      shortName: Res
+      description: Base resource, such as a Pod, a Service or a Node
+      labels: 
+        - SrcK8S_Name
+        - SrcK8S_Type
+        - SrcK8S_OwnerName
+        - SrcK8S_OwnerType
+        - SrcK8S_Namespace
+        - SrcAddr
+        - SrcK8S_HostName
+        - DstK8S_Name
+        - DstK8S_Type
+        - DstK8S_OwnerName
+        - DstK8S_OwnerType
+        - DstK8S_Namespace
+        - DstAddr
+        - DstK8S_HostName
+      groups:
+        - clusters
+        - clusters+zones
+        - clusters+hosts
+        - clusters+namespaces
+        - clusters+owners
+        - zones
+        - zones+hosts
+        - zones+namespaces
+        - zones+owners
+        - hosts
+        - hosts+namespaces
+        - hosts+owners
+        - namespaces
+        - namespaces+owners
+        - owners
+      filters:
+        - src_resource
+        - dst_resource
   fields:
     - name: TimeFlowStartMs
       type: number

pkg/config/config.go

@@ -77,7 +77,7 @@ type Column struct {
 	Filter     string   `yaml:"filter,omitempty" json:"filter,omitempty"`
 	Default    bool     `yaml:"default,omitempty" json:"default,omitempty"`
 	Width      int      `yaml:"width,omitempty" json:"width,omitempty"`
-	Feature    string   `yaml:"feature" json:"feature"`
+	Feature    string   `yaml:"feature,omitempty" json:"feature,omitempty"`
 }
 
 type Filter struct {
@@ -92,6 +92,19 @@ type Filter struct {
 	Placeholder            string `yaml:"placeholder,omitempty" json:"placeholder,omitempty"`
 }
 
+type Scope struct {
+	ID          string   `yaml:"id" json:"id"`
+	Name        string   `yaml:"name" json:"name"`
+	ShortName   string   `yaml:"shortName" json:"shortName"`
+	Description string   `yaml:"description" json:"description"`
+	Labels      []string `yaml:"labels" json:"labels"`
+	Feature     string   `yaml:"feature,omitempty" json:"feature,omitempty"`
+	Groups      []string `yaml:"groups,omitempty" json:"groups,omitempty"`
+	Filter      string   `yaml:"filter,omitempty" json:"filter,omitempty"`
+	Filters     []string `yaml:"filters,omitempty" json:"filters,omitempty"`
+	StepInto    string   `yaml:"stepInto,omitempty" json:"stepInto,omitempty"`
+}
+
 type QuickFilter struct {
 	Name   string            `yaml:"name" json:"name"`
 	Filter map[string]string `yaml:"filter" json:"filter"`
@@ -120,6 +133,7 @@ type Frontend struct {
 	Panels          []string      `yaml:"panels" json:"panels"`
 	Columns         []Column      `yaml:"columns" json:"columns"`
 	Filters         []Filter      `yaml:"filters" json:"filters"`
+	Scopes          []Scope       `yaml:"scopes" json:"scopes"`
 	QuickFilters    []QuickFilter `yaml:"quickFilters" json:"quickFilters"`
 	AlertNamespaces []string      `yaml:"alertNamespaces" json:"alertNamespaces"`
 	Sampling        int           `yaml:"sampling" json:"sampling"`
@@ -166,7 +180,13 @@ func ReadFile(version, date, filename string) (*Config, error) {
 				{ID: "SrcAddr", Name: "IP", Group: "Source", Field: "SrcAddr", Default: true, Width: 15},
 				{ID: "DstAddr", Name: "IP", Group: "Destination", Field: "DstAddr", Default: true, Width: 15},
 			},
-			Filters:      []Filter{},
+			Filters: []Filter{},
+			Scopes: []Scope{
+				{ID: "host", Name: "Node", Labels: []string{"SrcK8S_HostName", "DstK8S_HostName"}},
+				{ID: "namespace", Name: "Namespace", Labels: []string{"SrcK8S_Namespace", "DstK8S_Namespace"}},
+				{ID: "owner", Name: "Owner", Labels: []string{"SrcK8S_OwnerName", "SrcK8S_OwnerType", "DstK8S_OwnerName", "DstK8S_OwnerType", "SrcK8S_Namespace", "DstK8S_Namespace"}},
+				{ID: "resource", Name: "Resource", Labels: []string{"SrcK8S_Name", "SrcK8S_Type", "SrcK8S_OwnerName", "SrcK8S_OwnerType", "SrcK8S_Namespace", "SrcAddr", "SrcK8S_HostName", "DstK8S_Name", "DstK8S_Type", "DstK8S_OwnerName", "DstK8S_OwnerType", "DstK8S_Namespace", "DstAddr", "DstK8S_HostName"}},
+			},
 			QuickFilters: []QuickFilter{},
 			Features:     []string{},
 			Deduper: Deduper{
@@ -299,3 +319,16 @@ func (c *Config) GetAuthChecker() (auth.Checker, error) {
 	}
 	return auth.NewChecker(checkType, client.NewInCluster)
 }
+
+func (c *Frontend) GetAggregateKeyLabels() map[string][]string {
+	keyLabels := map[string][]string{
+		"app":          {"app"},
+		"droppedState": {"PktDropLatestState"},
+		"droppedCause": {"PktDropLatestDropCause"},
+		"dnsRCode":     {"DnsFlagsResponseCode"},
+	}
+	for i := range c.Scopes {
+		keyLabels[c.Scopes[i].ID] = c.Scopes[i].Labels
+	}
+	return keyLabels
+}

pkg/handler/lokiclientmock/loki_client_mock.go

@@ -37,7 +37,7 @@ func (o *LokiClientMock) Get(url string) ([]byte, int, error) {
 			} else if strings.Contains(url, "by(K8S_ClusterName)") {
 				path += "_cluster.json"
 			} else if strings.Contains(url, "by(SrcK8S_Zone,DstK8S_Zone)") {
-				path += "zone.json"
+				path += "_zone.json"
 			} else if strings.Contains(url, "by(SrcK8S_HostName,DstK8S_HostName)") {
 				path += "_host.json"
 			} else if strings.Contains(url, "by(SrcK8S_Namespace,DstK8S_Namespace)") {

pkg/handler/topology.go

@@ -139,7 +139,7 @@ func (h *Handlers) extractTopologyQueryParams(params url.Values, ds constants.Da
 			namespace,
 			func(filters filters.SingleQuery) bool {
 				// Do not expand if this is managed from prometheus
-				sr, _ := getEligiblePromMetric(h.PromInventory, filters, &in, namespace != "")
+				sr, _ := getEligiblePromMetric(h.Cfg.Frontend.GetAggregateKeyLabels(), h.PromInventory, filters, &in, namespace != "")
 				return sr != nil && len(sr.Found) > 0
 			},
 		)
@@ -276,12 +276,12 @@ func buildTopologyQuery(
 	qr *v1.Range,
 	isDev bool,
 ) (string, *prometheus.Query, int, error) {
-	search, unsupportedReason := getEligiblePromMetric(promInventory, filters, in, isDev)
+	search, unsupportedReason := getEligiblePromMetric(cfg.Frontend.GetAggregateKeyLabels(), promInventory, filters, in, isDev)
 	if unsupportedReason != "" {
 		hlog.Debugf("Unsupported Prometheus query; reason: %s.", unsupportedReason)
 	} else if search != nil && len(search.Found) > 0 {
 		// Success, we can use Prometheus
-		qb := prometheus.NewQuery(in, qr, filters, search.Found)
+		qb := prometheus.NewQuery(cfg.Frontend.GetAggregateKeyLabels(), in, qr, filters, search.Found)
 		q := qb.Build()
 		return "", &q, http.StatusOK, nil
 	}
@@ -310,7 +310,7 @@ func buildTopologyQuery(
 			"this request could not be performed with Prometheus metrics%s: it requires installing and enabling Loki", reason)
 	}
 
-	qb, err := loki.NewTopologyQuery(&cfg.Loki, in)
+	qb, err := loki.NewTopologyQuery(&cfg.Loki, cfg.Frontend.GetAggregateKeyLabels(), in)
 	if err != nil {
 		return "", nil, http.StatusBadRequest, err
 	}
@@ -321,7 +321,7 @@ func buildTopologyQuery(
 	return EncodeQuery(qb.Build()), nil, http.StatusOK, nil
 }
 
-func getEligiblePromMetric(promInventory *prometheus.Inventory, filters filters.SingleQuery, in *loki.TopologyInput, isDev bool) (*prometheus.SearchResult, string) {
+func getEligiblePromMetric(kl map[string][]string, promInventory *prometheus.Inventory, filters filters.SingleQuery, in *loki.TopologyInput, isDev bool) (*prometheus.SearchResult, string) {
 	if in.DataSource != constants.DataSourceAuto && in.DataSource != constants.DataSourceProm {
 		return nil, ""
 	}
@@ -332,7 +332,7 @@ func getEligiblePromMetric(promInventory *prometheus.Inventory, filters filters.
 		return nil, fmt.Sprintf("RecordType not managed: %s", in.RecordType)
 	}
 
-	labelsNeeded, _ := prometheus.GetLabelsAndFilter(in.Aggregate, in.Groups)
+	labelsNeeded, _ := prometheus.GetLabelsAndFilter(kl, in.Aggregate, in.Groups)
 	fromFilters, unsupportedReason := prometheus.FiltersToLabels(filters)
 	if unsupportedReason != "" {
 		return nil, unsupportedReason

pkg/loki/topology_query.go

@@ -13,28 +13,6 @@ const (
 	topologyDefaultLimit = "100"
 )
 
-var (
-	aggregateKeyLabels = map[string][]string{
-		"app":          {"app"},
-		"droppedState": {"PktDropLatestState"},
-		"droppedCause": {"PktDropLatestDropCause"},
-		"dnsRCode":     {"DnsFlagsResponseCode"},
-		"cluster":      {"K8S_ClusterName"},
-		"zone":         {"SrcK8S_Zone", "DstK8S_Zone"},
-		"host":         {"SrcK8S_HostName", "DstK8S_HostName"},
-		"namespace":    {"SrcK8S_Namespace", "DstK8S_Namespace"},
-		"owner":        {"SrcK8S_OwnerName", "SrcK8S_OwnerType", "DstK8S_OwnerName", "DstK8S_OwnerType", "SrcK8S_Namespace", "DstK8S_Namespace"},
-		"resource":     {"SrcK8S_Name", "SrcK8S_Type", "SrcK8S_OwnerName", "SrcK8S_OwnerType", "SrcK8S_Namespace", "SrcAddr", "SrcK8S_HostName", "DstK8S_Name", "DstK8S_Type", "DstK8S_OwnerName", "DstK8S_OwnerType", "DstK8S_Namespace", "DstAddr", "DstK8S_HostName"},
-	}
-	groupKeyLabels = map[string][]string{
-		"clusters":   {"K8S_ClusterName"},
-		"zones":      {"SrcK8S_Zone", "DstK8S_Zone"},
-		"hosts":      {"SrcK8S_HostName", "DstK8S_HostName"},
-		"namespaces": {"SrcK8S_Namespace", "DstK8S_Namespace"},
-		"owners":     {"SrcK8S_OwnerName", "SrcK8S_OwnerType", "DstK8S_OwnerName", "DstK8S_OwnerType"},
-	}
-)
-
 type TopologyInput struct {
 	Start          string
 	End            string
@@ -53,10 +31,11 @@ type TopologyInput struct {
 
 type TopologyQueryBuilder struct {
 	*FlowQueryBuilder
-	topology *TopologyInput
+	topology           *TopologyInput
+	aggregateKeyLabels map[string][]string
 }
 
-func NewTopologyQuery(cfg *config.Loki, in *TopologyInput) (*TopologyQueryBuilder, error) {
+func NewTopologyQuery(cfg *config.Loki, kl map[string][]string, in *TopologyInput) (*TopologyQueryBuilder, error) {
 	var dedup bool
 	var rt constants.RecordType
 	if slices.Contains(constants.AnyConnectionType, string(in.RecordType)) {
@@ -69,20 +48,21 @@ func NewTopologyQuery(cfg *config.Loki, in *TopologyInput) (*TopologyQueryBuilde
 
 	fqb := NewFlowQueryBuilder(cfg, in.Start, in.End, in.Top, dedup, rt, in.PacketLoss)
 	return &TopologyQueryBuilder{
-		FlowQueryBuilder: fqb,
-		topology:         in,
+		FlowQueryBuilder:   fqb,
+		topology:           in,
+		aggregateKeyLabels: kl,
 	}, nil
 }
 
-func GetLabelsAndFilter(aggregate, groups string) ([]string, string) {
+func GetLabelsAndFilter(kl map[string][]string, aggregate, groups string) ([]string, string) {
 	var fields []string
 	var filter string
-	if fields = aggregateKeyLabels[aggregate]; fields == nil {
+	if fields = kl[aggregate]; fields == nil {
 		fields = []string{aggregate}
 		filter = aggregate
 	}
 	if groups != "" {
-		for gr, labels := range groupKeyLabels {
+		for gr, labels := range kl {
 			if strings.Contains(groups, gr) {
 				for _, label := range labels {
 					if !slices.Contains(fields, label) {
@@ -142,7 +122,7 @@ func (q *TopologyQueryBuilder) Build() string {
 		top = topologyDefaultLimit
 	}
 
-	labels, extraFilter := GetLabelsAndFilter(q.topology.Aggregate, q.topology.Groups)
+	labels, extraFilter := GetLabelsAndFilter(q.aggregateKeyLabels, q.topology.Aggregate, q.topology.Groups)
 	if q.config.IsLabel(extraFilter) {
 		extraFilter = ""
 	}

pkg/loki/topology_query_test.go

@@ -14,6 +14,19 @@ var lokiConfig = config.Loki{
 	Labels: []string{"SrcK8S_Namespace", "SrcK8S_OwnerName", "DstK8S_Namespace", "DstK8S_OwnerName", "FlowDirection"},
 }
 
+var aggregateKeyLabels = map[string][]string{
+	"app":          {"app"},
+	"droppedState": {"PktDropLatestState"},
+	"droppedCause": {"PktDropLatestDropCause"},
+	"dnsRCode":     {"DnsFlagsResponseCode"},
+	"cluster":      {"K8S_ClusterName"},
+	"zone":         {"SrcK8S_Zone", "DstK8S_Zone"},
+	"host":         {"SrcK8S_HostName", "DstK8S_HostName"},
+	"namespace":    {"SrcK8S_Namespace", "DstK8S_Namespace"},
+	"owner":        {"SrcK8S_OwnerName", "SrcK8S_OwnerType", "DstK8S_OwnerName", "DstK8S_OwnerType", "SrcK8S_Namespace", "DstK8S_Namespace"},
+	"resource":     {"SrcK8S_Name", "SrcK8S_Type", "SrcK8S_OwnerName", "SrcK8S_OwnerType", "SrcK8S_Namespace", "SrcAddr", "SrcK8S_HostName", "DstK8S_Name", "DstK8S_Type", "DstK8S_OwnerName", "DstK8S_OwnerType", "DstK8S_Namespace", "DstAddr", "DstK8S_HostName"},
+}
+
 func TestBuildTopologyQuery_SimpleAggregate(t *testing.T) {
 	in := TopologyInput{
 		Start:          "(start)",
@@ -28,7 +41,7 @@ func TestBuildTopologyQuery_SimpleAggregate(t *testing.T) {
 		Aggregate:      "namespace",
 		DedupMark:      true,
 	}
-	q, err := NewTopologyQuery(&lokiConfig, &in)
+	q, err := NewTopologyQuery(&lokiConfig, aggregateKeyLabels, &in)
 	require.NoError(t, err)
 	result := q.Build()
 	assert.Equal(
@@ -54,7 +67,7 @@ func TestBuildTopologyQuery_GroupsAndAggregate(t *testing.T) {
 		Groups:         "hosts",
 		DedupMark:      true,
 	}
-	q, err := NewTopologyQuery(&lokiConfig, &in)
+	q, err := NewTopologyQuery(&lokiConfig, aggregateKeyLabels, &in)
 	require.NoError(t, err)
 	result := q.Build()
 	assert.Equal(
@@ -79,7 +92,7 @@ func TestBuildTopologyQuery_CustomAggregate(t *testing.T) {
 		Aggregate:      "SomeField",
 		DedupMark:      true,
 	}
-	q, err := NewTopologyQuery(&lokiConfig, &in)
+	q, err := NewTopologyQuery(&lokiConfig, aggregateKeyLabels, &in)
 	require.NoError(t, err)
 	result := q.Build()
 	assert.Equal(
@@ -104,7 +117,7 @@ func TestBuildTopologyQuery_CustomLabelAggregate(t *testing.T) {
 		Aggregate:      "FlowDirection",
 		DedupMark:      true,
 	}
-	q, err := NewTopologyQuery(&lokiConfig, &in)
+	q, err := NewTopologyQuery(&lokiConfig, aggregateKeyLabels, &in)
 	require.NoError(t, err)
 	result := q.Build()
 	assert.Equal(