Add namespace as a resource type in public-api (#760)

* Add namespace as a resource type in public-api

The cli and public-api only supported deployments as a resource type.

This change adds support for namespace as a resource type in the cli and
public-api. This also change includes:
- cli statsummary now prints `-`'s when objects are not in the mesh
- cli statsummary prints `No resources found.` when applicable
- removed `out-` from cli statsummary flags, and analagous proto changes
- switched public-api to use native prometheus label types
- misc error handling and logging fixes

Part of #627

Signed-off-by: Andrew Seigner <siggy@buoyant.io>

* Refactor filter and groupby label formulation

Signed-off-by: Kevin Lingerfelt <kl@buoyant.io>

* Rename stat_summary.go to stat.go in cli

Signed-off-by: Kevin Lingerfelt <kl@buoyant.io>

* Update rbac privileges for namespace stats

Signed-off-by: Kevin Lingerfelt <kl@buoyant.io>

cli/cmd/stat_summary.go → cli/cmd/stat.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"sort"
 	"strings"
 	"text/tabwriter"
@@ -13,13 +14,14 @@ import (
 	"github.com/prometheus/common/log"
 	"github.com/runconduit/conduit/controller/api/util"
 	pb "github.com/runconduit/conduit/controller/gen/public"
+	"github.com/runconduit/conduit/pkg/k8s"
 	"github.com/spf13/cobra"
 	"k8s.io/api/core/v1"
 )
 
 var timeWindow, namespace, resourceType, resourceName string
-var outToNamespace, outToType, outToName string
-var outFromNamespace, outFromType, outFromName string
+var toNamespace, toType, toName string
+var fromNamespace, fromType, fromName string
 var allNamespaces bool
 
 var statCmd = &cobra.Command{
@@ -29,17 +31,24 @@ var statCmd = &cobra.Command{
 
 Valid resource types include:
 
-	* deployment
+	* deployments
+	* namespaces
 
 This command will hide resources that have completed, such as pods that are in the Succeeded or Failed phases.
 If no resource name is specified, displays stats about all resources of the specified RESOURCETYPE`,
 	Example: `  # Get all deployments in the test namespace.
   conduit stat deployments -n test
 
   # Get the hello1 deployment in the test namespace.
-  conduit stat deployments hello1 -n test`,
+  conduit stat deployments hello1 -n test
+
+  # Get the test namespace.
+  conduit stat namespaces test
+
+  # Get all namespaces.
+  conduit stat --all-namespaces=true namespaces`,
 	Args:      cobra.RangeArgs(1, 2),
-	ValidArgs: []string{"deployment"},
+	ValidArgs: []string{k8s.KubernetesDeployments, k8s.KubernetesNamespaces},
 	RunE: func(cmd *cobra.Command, args []string) error {
 		switch len(args) {
 		case 1:
@@ -71,12 +80,12 @@ func init() {
 	RootCmd.AddCommand(statCmd)
 	statCmd.PersistentFlags().StringVarP(&namespace, "namespace", "n", "default", "Namespace of the specified resource")
 	statCmd.PersistentFlags().StringVarP(&timeWindow, "time-window", "t", "1m", "Stat window (one of: \"10s\", \"1m\", \"10m\", \"1h\")")
-	statCmd.PersistentFlags().StringVar(&outToName, "out-to", "", "If present, restricts outbound stats to the specified resource name")
-	statCmd.PersistentFlags().StringVar(&outToNamespace, "out-to-namespace", "", "Sets the namespace used to lookup the \"--out-to\" resource; by default the current \"--namespace\" is used")
-	statCmd.PersistentFlags().StringVar(&outToType, "out-to-resource", "", "If present, restricts outbound stats to the specified resource type")
-	statCmd.PersistentFlags().StringVar(&outFromName, "out-from", "", "If present, restricts outbound stats to the specified resource name")
-	statCmd.PersistentFlags().StringVar(&outFromNamespace, "out-from-namespace", "", "Sets the namespace used to lookup the \"--out-from\" resource; by default the current \"--namespace\" is used")
-	statCmd.PersistentFlags().StringVar(&outFromType, "out-from-resource", "", "If present, restricts outbound stats to the specified resource type")
+	statCmd.PersistentFlags().StringVar(&toName, "to", "", "If present, restricts outbound stats to the specified resource name")
+	statCmd.PersistentFlags().StringVar(&toNamespace, "to-namespace", "", "Sets the namespace used to lookup the \"--to\" resource; by default the current \"--namespace\" is used")
+	statCmd.PersistentFlags().StringVar(&toType, "to-resource", "", "Sets the resource type used to lookup the \"--to\" resource; by default the RESOURCETYPE is used")
+	statCmd.PersistentFlags().StringVar(&fromName, "from", "", "If present, restricts outbound stats from the specified resource name")
+	statCmd.PersistentFlags().StringVar(&fromNamespace, "from-namespace", "", "Sets the namespace used from lookup the \"--from\" resource; by default the current \"--namespace\" is used")
+	statCmd.PersistentFlags().StringVar(&fromType, "from-resource", "", "Sets the resource type used to lookup the \"--from\" resource; by default the RESOURCETYPE is used")
 	statCmd.PersistentFlags().BoolVar(&allNamespaces, "all-namespaces", false, "If present, returns stats across all namespaces, ignoring the \"--namespace\" flag")
 }
 
@@ -110,15 +119,19 @@ func renderStats(resp *pb.StatSummaryResponse) string {
 
 const padding = 3
 
-type row struct {
-	meshed      string
+type rowStats struct {
 	requestRate float64
 	successRate float64
 	latencyP50  uint64
 	latencyP95  uint64
 	latencyP99  uint64
 }
 
+type row struct {
+	meshed string
+	*rowStats
+}
+
 func writeStatsToBuffer(resp *pb.StatSummaryResponse, w *tabwriter.Writer) {
 	nameHeader := "NAME"
 	maxNameLength := len(nameHeader)
@@ -147,15 +160,22 @@ func writeStatsToBuffer(resp *pb.StatSummaryResponse, w *tabwriter.Writer) {
 			}
 
 			if r.Stats != nil {
-				stats[key].requestRate = getRequestRate(*r)
-				stats[key].successRate = getSuccessRate(*r)
-				stats[key].latencyP50 = r.Stats.LatencyMsP50
-				stats[key].latencyP95 = r.Stats.LatencyMsP95
-				stats[key].latencyP99 = r.Stats.LatencyMsP99
+				stats[key].rowStats = &rowStats{
+					requestRate: getRequestRate(*r),
+					successRate: getSuccessRate(*r),
+					latencyP50:  r.Stats.LatencyMsP50,
+					latencyP95:  r.Stats.LatencyMsP95,
+					latencyP99:  r.Stats.LatencyMsP99,
+				}
 			}
 		}
 	}
 
+	if len(stats) == 0 {
+		fmt.Fprintln(os.Stderr, "No traffic found.")
+		os.Exit(0)
+	}
+
 	headers := make([]string, 0)
 	if allNamespaces {
 		headers = append(headers,
@@ -170,6 +190,7 @@ func writeStatsToBuffer(resp *pb.StatSummaryResponse, w *tabwriter.Writer) {
 		"LATENCY_P95",
 		"LATENCY_P99\t", // trailing \t is required to format last column
 	}...)
+
 	fmt.Fprintln(w, strings.Join(headers, "\t"))
 
 	sortedKeys := sortStatsKeys(stats)
@@ -179,23 +200,32 @@ func writeStatsToBuffer(resp *pb.StatSummaryResponse, w *tabwriter.Writer) {
 		name := parts[1]
 		values := make([]interface{}, 0)
 		templateString := "%s\t%s\t%.2f%%\t%.1frps\t%dms\t%dms\t%dms\t\n"
+		templateStringEmpty := "%s\t%s\t-\t-\t-\t-\t-\t\n"
 
 		if allNamespaces {
 			values = append(values,
 				namespace+strings.Repeat(" ", maxNamespaceLength-len(namespace)))
 			templateString = "%s\t" + templateString
+			templateStringEmpty = "%s\t" + templateStringEmpty
 		}
 		values = append(values, []interface{}{
 			name + strings.Repeat(" ", maxNameLength-len(name)),
 			stats[key].meshed,
-			stats[key].successRate * 100,
-			stats[key].requestRate,
-			stats[key].latencyP50,
-			stats[key].latencyP95,
-			stats[key].latencyP99,
 		}...)
 
-		fmt.Fprintf(w, templateString, values...)
+		if stats[key].rowStats != nil {
+			values = append(values, []interface{}{
+				stats[key].successRate * 100,
+				stats[key].requestRate,
+				stats[key].latencyP50,
+				stats[key].latencyP95,
+				stats[key].latencyP99,
+			}...)
+
+			fmt.Fprintf(w, templateString, values...)
+		} else {
+			fmt.Fprintf(w, templateStringEmpty, values...)
+		}
 	}
 }
 
@@ -208,16 +238,16 @@ func buildStatSummaryRequest() (*pb.StatSummaryRequest, error) {
 	}
 
 	requestParams := util.StatSummaryRequestParams{
-		TimeWindow:       timeWindow,
-		ResourceName:     resourceName,
-		ResourceType:     resourceType,
-		Namespace:        targetNamespace,
-		OutToName:        outToName,
-		OutToType:        outToType,
-		OutToNamespace:   outToNamespace,
-		OutFromName:      outFromName,
-		OutFromType:      outFromType,
-		OutFromNamespace: outFromNamespace,
+		TimeWindow:    timeWindow,
+		ResourceName:  resourceName,
+		ResourceType:  resourceType,
+		Namespace:     targetNamespace,
+		ToName:        toName,
+		ToType:        toType,
+		ToNamespace:   toNamespace,
+		FromName:      fromName,
+		FromType:      fromType,
+		FromNamespace: fromNamespace,
 	}
 
 	return util.BuildStatSummaryRequest(requestParams)

cli/cmd/testdata/install_default.golden

@@ -23,7 +23,7 @@ rules:
   resources: ["deployments", "replicasets"]
   verbs: ["list", "get", "watch"]
 - apiGroups: [""]
-  resources: ["pods", "endpoints", "services"]
+  resources: ["pods", "endpoints", "services", "namespaces"]
   verbs: ["list", "get", "watch"]
 
 ---

cli/cmd/testdata/install_output.golden

@@ -23,7 +23,7 @@ rules:
   resources: ["deployments", "replicasets"]
   verbs: ["list", "get", "watch"]
 - apiGroups: [""]
-  resources: ["pods", "endpoints", "services"]
+  resources: ["pods", "endpoints", "services", "namespaces"]
   verbs: ["list", "get", "watch"]
 
 ---

cli/install/template.go

@@ -26,7 +26,7 @@ rules:
   resources: ["deployments", "replicasets"]
   verbs: ["list", "get", "watch"]
 - apiGroups: [""]
-  resources: ["pods", "endpoints", "services"]
+  resources: ["pods", "endpoints", "services", "namespaces"]
   verbs: ["list", "get", "watch"]
 
 ---

controller/api/public/client.go

@@ -79,7 +79,7 @@ func (c *grpcOverHttpClient) Tap(ctx context.Context, req *pb.TapRequest, _ ...g
 func (c *grpcOverHttpClient) apiRequest(ctx context.Context, endpoint string, req proto.Message, protoResponse proto.Message) error {
 	url := c.endpointNameToPublicApiUrl(endpoint)
 
-	log.Debugf("Making gRPC-over-HTTP call to [%s]", url.String())
+	log.Debugf("Making gRPC-over-HTTP call to [%s] [%+v]", url.String(), req)
 	httpRsp, err := c.post(ctx, url, req)
 	if err != nil {
 		return err

controller/api/public/grpc_server.go

@@ -24,6 +24,7 @@ type (
 	grpcServer struct {
 		prometheusAPI       promv1.API
 		tapClient           tapPb.TapClient
+		namespaceLister     corelisters.NamespaceLister
 		deployLister        applisters.DeploymentLister
 		replicaSetLister    applisters.ReplicaSetLister
 		podLister           corelisters.PodLister
@@ -43,6 +44,7 @@ const (
 func newGrpcServer(
 	promAPI promv1.API,
 	tapClient tapPb.TapClient,
+	namespaceLister corelisters.NamespaceLister,
 	deployLister applisters.DeploymentLister,
 	replicaSetLister applisters.ReplicaSetLister,
 	podLister corelisters.PodLister,
@@ -52,6 +54,7 @@ func newGrpcServer(
 	return &grpcServer{
 		prometheusAPI:       promAPI,
 		tapClient:           tapClient,
+		namespaceLister:     namespaceLister,
 		deployLister:        deployLister,
 		replicaSetLister:    replicaSetLister,
 		podLister:           podLister,

controller/api/public/grpc_server_test.go

@@ -166,13 +166,15 @@ spec:
 			clientSet := fake.NewSimpleClientset(k8sObjs...)
 			sharedInformers := informers.NewSharedInformerFactory(clientSet, 10*time.Minute)
 
+			namespaceInformer := sharedInformers.Core().V1().Namespaces()
 			deployInformer := sharedInformers.Apps().V1beta2().Deployments()
 			replicaSetInformer := sharedInformers.Apps().V1beta2().ReplicaSets()
 			podInformer := sharedInformers.Core().V1().Pods()
 
 			fakeGrpcServer := newGrpcServer(
 				&MockProm{Res: exp.promRes},
 				tap.NewTapClient(nil),
+				namespaceInformer.Lister(),
 				deployInformer.Lister(),
 				replicaSetInformer.Lister(),
 				podInformer.Lister(),
@@ -183,6 +185,7 @@ spec:
 			sharedInformers.Start(stopCh)
 			if !cache.WaitForCacheSync(
 				stopCh,
+				namespaceInformer.Informer().HasSynced,
 				deployInformer.Informer().HasSynced,
 				replicaSetInformer.Informer().HasSynced,
 				podInformer.Informer().HasSynced,

controller/api/public/http_server.go

@@ -199,6 +199,7 @@ func NewServer(
 	addr string,
 	prometheusClient promApi.Client,
 	tapClient tapPb.TapClient,
+	namespaceLister corelisters.NamespaceLister,
 	deployLister applisters.DeploymentLister,
 	replicaSetLister applisters.ReplicaSetLister,
 	podLister corelisters.PodLister,
@@ -209,6 +210,7 @@ func NewServer(
 		grpcServer: newGrpcServer(
 			promv1.NewAPI(prometheusClient),
 			tapClient,
+			namespaceLister,
 			deployLister,
 			replicaSetLister,
 			podLister,