Allow specified resource types and names for kube-objects collector (#…

…188)

* allow specified resource types and names for kubeobjects collector
* add tests to cover various kube-objects combinations

README.md

@@ -98,7 +98,7 @@ secretGenerator:
 #   behavior: replace
 #   literals:
 #   - DIAGNOSTIC_CONTAINERLOGS_LIST=kube-system # space-separated namespaces
-#   - DIAGNOSTIC_KUBEOBJECTS_LIST=kube-system/pod kube-system/service kube-system/deployment # space-separated namespace/kind pairs
+#   - DIAGNOSTIC_KUBEOBJECTS_LIST=kube-system/pod kube-system/service kube-system/deployment # space-separated list of namespace/resource-type[/resource]
 #   - DIAGNOSTIC_NODELOGS_LIST_LINUX="/var/log/azure/cluster-provision.log /var/log/cloud-init.log" # space-separated log file locations
 #   - DIAGNOSTIC_NODELOGS_LIST_WINDOWS="C:\AzureData\CustomDataSetupScript.log" # space-separated log file locations
 ```

pkg/collector/kubeobjects_collector.go

@@ -2,27 +2,35 @@ package collector
 
 import (
 	"fmt"
+	"log"
 	"strings"
 
 	"github.com/Azure/aks-periscope/pkg/utils"
-	"k8s.io/client-go/kubernetes"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/discovery"
+	memory "k8s.io/client-go/discovery/cached"
 	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/restmapper"
 	"k8s.io/kubectl/pkg/describe"
 )
 
 // KubeObjectsCollector defines a KubeObjects Collector struct
 type KubeObjectsCollector struct {
-	data        map[string]string
-	kubeconfig  *restclient.Config
-	runtimeInfo *utils.RuntimeInfo
+	data          map[string]string
+	kubeconfig    *restclient.Config
+	commandRunner *utils.KubeCommandRunner
+	runtimeInfo   *utils.RuntimeInfo
 }
 
 // NewKubeObjectsCollector is a constructor
 func NewKubeObjectsCollector(config *restclient.Config, runtimeInfo *utils.RuntimeInfo) *KubeObjectsCollector {
 	return &KubeObjectsCollector{
-		data:        make(map[string]string),
-		kubeconfig:  config,
-		runtimeInfo: runtimeInfo,
+		data:          make(map[string]string),
+		kubeconfig:    config,
+		commandRunner: utils.NewKubeCommandRunner(config),
+		runtimeInfo:   runtimeInfo,
 	}
 }
 
@@ -36,42 +44,83 @@ func (collector *KubeObjectsCollector) CheckSupported() error {
 
 // Collect implements the interface method
 func (collector *KubeObjectsCollector) Collect() error {
-	// Creates the clientset
-	clientset, err := kubernetes.NewForConfig(collector.kubeconfig)
+	// Create a discovery client for querying resource metadata
+	discoveryClient, err := discovery.NewDiscoveryClientForConfig(collector.kubeconfig)
 	if err != nil {
-		return fmt.Errorf("getting access to K8S failed: %w", err)
+		return fmt.Errorf("error creating discovery client: %w", err)
 	}
 
+	// Create a RESTMapper to handle the mapping between GroupKind and GroupVersionResource
+	mapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(discoveryClient))
+
 	for _, kubernetesObject := range collector.runtimeInfo.KubernetesObjects {
 		kubernetesObjectParts := strings.Split(kubernetesObject, "/")
-		nameSpace := kubernetesObjectParts[0]
-		objectType := kubernetesObjectParts[1]
+		if len(kubernetesObjectParts) < 2 {
+			log.Printf("Invalid kube-objects value: %s", kubernetesObject)
+			continue
+		}
+
+		namespace := kubernetesObjectParts[0]
+		groupResource := schema.ParseGroupResource(kubernetesObjectParts[1])
 
-		// List the pods in the given namespace
-		podList, err := utils.GetPods(clientset, nameSpace)
+		groupVersionKind, err := mapper.KindFor(groupResource.WithVersion(""))
 		if err != nil {
-			return fmt.Errorf("getting pods failed: %w", err)
+			log.Printf("Unable to determine Kind for resource %s: %v", groupResource.String(), err)
+			continue
+		}
+
+		describer, ok := describe.DescriberFor(groupVersionKind.GroupKind(), collector.kubeconfig)
+		if !ok {
+			log.Printf("Unable to create Describer for Kind %s", groupVersionKind.String())
+			continue
 		}
 
-		for _, pod := range podList.Items {
-			d := describe.PodDescriber{
-				Interface: clientset,
+		// Get the resources within the namespace to describe
+		var resourceNames []string
+		if len(kubernetesObjectParts) > 2 {
+			resourceNames = []string{kubernetesObjectParts[2]}
+		} else {
+			resourceNames, err = collector.getResourcesInNamespace(mapper, &groupResource, namespace)
+			if err != nil {
+				log.Printf("Unable to get %s resources in %s: %v", groupResource.String(), namespace, err)
+				continue
 			}
+		}
 
-			output, err := d.Describe(pod.Namespace, pod.Name, describe.DescriberSettings{
-				ShowEvents: true,
-			})
+		for _, resourceName := range resourceNames {
+			output, err := describer.Describe(namespace, resourceName, describe.DescriberSettings{ShowEvents: true})
 			if err != nil {
-				return fmt.Errorf("getting description failed: %w", err)
+				log.Printf("Error describing %s %s in namespace %s: %v", groupVersionKind.String(), resourceName, namespace, err)
+				continue
 			}
 
-			collector.data[pod.Namespace+"_"+objectType+"_"+pod.Name] = output
+			key := fmt.Sprintf("%s_%s_%s", namespace, groupResource.String(), resourceName)
+			collector.data[key] = output
 		}
 	}
 
 	return nil
 }
 
+func (collector *KubeObjectsCollector) getResourcesInNamespace(mapper meta.RESTMapper, groupResource *schema.GroupResource, namespace string) ([]string, error) {
+	groupVersionResource, err := mapper.ResourceFor(groupResource.WithVersion(""))
+	if err != nil {
+		return []string{}, fmt.Errorf("error determining Version for resource %s: %v", groupResource.String(), err)
+	}
+
+	resources, err := collector.commandRunner.GetUnstructuredList(&groupVersionResource, namespace, &metav1.ListOptions{})
+	if err != nil {
+		return []string{}, fmt.Errorf("error listing %s: %v", groupVersionResource.String(), err)
+	}
+
+	resourceNames := make([]string, len(resources.Items))
+	for i, resource := range resources.Items {
+		resourceNames[i] = resource.GetName()
+	}
+
+	return resourceNames, nil
+}
+
 func (collector *KubeObjectsCollector) GetData() map[string]string {
 	return collector.data
 }

pkg/collector/kubeobjects_collector_test.go

@@ -1,10 +1,15 @@
 package collector
 
 import (
+	"context"
+	"fmt"
+	"regexp"
 	"testing"
 
 	"github.com/Azure/aks-periscope/pkg/test"
 	"github.com/Azure/aks-periscope/pkg/utils"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/rest"
 )
 
 func TestKubeObjectsCollectorGetName(t *testing.T) {
@@ -25,39 +30,205 @@ func TestKubeObjectsCollectorCheckSupported(t *testing.T) {
 	}
 }
 
-func TestKubeObjectsCollectorCollect(t *testing.T) {
-	tests := []struct {
-		name    string
-		want    int
-		wantErr bool
-	}{
-		{
-			name:    "get kube objects logs",
-			want:    1,
-			wantErr: false,
-		},
+var defaultKubeObjects = []string{"kube-system/pod", "kube-system/service", "kube-system/deployment"}
+
+func getDefaultKubeObjectResults(fixture *test.ClusterFixture) (map[string]*regexp.Regexp, error) {
+	results := map[string]*regexp.Regexp{}
+
+	podList, err := fixture.AdminAccess.Clientset.CoreV1().Pods("kube-system").List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("error listing pods: %w", err)
+	}
+
+	for _, pod := range podList.Items {
+		key := fmt.Sprintf("kube-system_pod_%s", pod.Name)
+		results[key] = regexp.MustCompile(fmt.Sprintf(`^Name:\s+%s\n(.*\n)*Containers:\n(.*\n)*Conditions:\n(.*\n)*Events:`, pod.Name))
 	}
 
+	svcList, err := fixture.AdminAccess.Clientset.CoreV1().Services("kube-system").List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("error listing services: %w", err)
+	}
+
+	for _, svc := range svcList.Items {
+		key := fmt.Sprintf("kube-system_service_%s", svc.Name)
+		results[key] = regexp.MustCompile(fmt.Sprintf(`^Name:\s+%s\n(.*\n)*Type:(.*\n)*IP:(.*\n)*Endpoints:(.*\n)*Events:`, svc.Name))
+	}
+
+	deployList, err := fixture.AdminAccess.Clientset.AppsV1().Deployments("kube-system").List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("error listing deployments: %w", err)
+	}
+
+	for _, deploy := range deployList.Items {
+		key := fmt.Sprintf("kube-system_deployment_%s", deploy.Name)
+		results[key] = regexp.MustCompile(fmt.Sprintf(`^Name:\s+%s\n(.*\n)*Pod Template:\n(.*\n)*Conditions:\n(.*\n)*Events:`, deploy.Name))
+	}
+
+	return results, nil
+}
+
+func getNodeNames(fixture *test.ClusterFixture) ([]string, error) {
+	nodeList, err := fixture.AdminAccess.Clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("error listing nodes: %w", err)
+	}
+
+	nodeNames := make([]string, len(nodeList.Items))
+	for i, node := range nodeList.Items {
+		nodeNames[i] = node.Name
+	}
+
+	return nodeNames, nil
+}
+
+func getNodeResults(nodeNames []string) map[string]*regexp.Regexp {
+	results := map[string]*regexp.Regexp{}
+	for _, nodeName := range nodeNames {
+		key := fmt.Sprintf("_nodes_%s", nodeName)
+		results[key] = regexp.MustCompile(fmt.Sprintf(`^Name:\s+%s\n(.*\n)*Conditions:\n(.*\n)*System Info:\n(.*\n)*Events:\n`, nodeName))
+	}
+
+	return results
+}
+
+func TestKubeObjectsCollectorCollect(t *testing.T) {
 	fixture, _ := test.GetClusterFixture()
 
-	runtimeInfo := &utils.RuntimeInfo{
-		KubernetesObjects: []string{"kube-system/pod", "kube-system/service", "kube-system/deployment"},
+	testNamespace, err := fixture.CreateTestNamespace("kubeobjectstest")
+	if err != nil {
+		t.Fatalf("Error creating test namespace %s: %v", testNamespace, err)
+	}
+
+	deployResourcesCommand := fmt.Sprintf("kubectl apply -n %s -f /resources/kube-objects/test-resources.yaml", testNamespace)
+	_, err = fixture.CommandRunner.Run(deployResourcesCommand, fixture.AdminAccess.GetKubeConfigBinding())
+	if err != nil {
+		t.Fatalf("Error deploying test resources into %s namespace: %v", testNamespace, err)
 	}
 
-	c := NewKubeObjectsCollector(fixture.PeriscopeAccess.ClientConfig, runtimeInfo)
+	defaultResults, err := getDefaultKubeObjectResults(fixture)
+	if err != nil {
+		t.Fatalf("Error determining expected results for default configuration: %v", err)
+	}
+
+	nodeNames, err := getNodeNames(fixture)
+	if err != nil {
+		t.Fatalf("Error getting node names: %v", err)
+	}
+
+	tests := []struct {
+		name             string
+		requestedObjects []string
+		config           *rest.Config
+		wantErr          bool
+		want             map[string]*regexp.Regexp
+	}{
+		{
+			name:             "bad kubeconfig",
+			requestedObjects: defaultKubeObjects,
+			config:           &rest.Config{Host: string([]byte{0})},
+			wantErr:          true,
+			want:             nil,
+		},
+		{
+			name:             "too few kubeobject parts should be skipped",
+			requestedObjects: []string{"kube-system"},
+			config:           fixture.PeriscopeAccess.ClientConfig,
+			wantErr:          false,
+			want:             map[string]*regexp.Regexp{},
+		},
+		{
+			name:             "unknown resource type should be skipped",
+			requestedObjects: []string{"kube-system/notaresource"},
+			config:           fixture.PeriscopeAccess.ClientConfig,
+			wantErr:          false,
+			want:             map[string]*regexp.Regexp{},
+		},
+		{
+			name:             "undescribable resource type should be skipped",
+			requestedObjects: []string{"kube-system/events"},
+			config:           fixture.PeriscopeAccess.ClientConfig,
+			wantErr:          false,
+			want:             map[string]*regexp.Regexp{},
+		},
+		{
+			name:             "missing resource should be skipped",
+			requestedObjects: []string{"kube-system/pod/notexisting"},
+			config:           fixture.PeriscopeAccess.ClientConfig,
+			wantErr:          false,
+			want:             map[string]*regexp.Regexp{},
+		},
+		{
+			name:             "unknown namespace should be skipped",
+			requestedObjects: []string{"notanamespace/pod"},
+			config:           fixture.PeriscopeAccess.ClientConfig,
+			wantErr:          false,
+			want:             map[string]*regexp.Regexp{},
+		},
+		{
+			name:             "non-namespaced resource type can be described",
+			requestedObjects: []string{"/nodes"},
+			config:           fixture.PeriscopeAccess.ClientConfig,
+			wantErr:          false,
+			want:             getNodeResults(nodeNames),
+		},
+		{
+			name:             "single non-namespaced resource can be described",
+			requestedObjects: []string{fmt.Sprintf("/nodes/%s", nodeNames[0])},
+			config:           fixture.PeriscopeAccess.ClientConfig,
+			wantErr:          false,
+			want:             getNodeResults([]string{nodeNames[0]}),
+		},
+		{
+			name:             "specified resources",
+			requestedObjects: []string{fmt.Sprintf("%s/configmaps", testNamespace)},
+			config:           fixture.PeriscopeAccess.ClientConfig,
+			wantErr:          false,
+			want: map[string]*regexp.Regexp{
+				fmt.Sprintf("%s_configmaps_kube-root-ca.crt", testNamespace): regexp.MustCompile(`^Name:\s+kube-root-ca.crt\n(.*\n)*Data`),
+				fmt.Sprintf("%s_configmaps_test-configmap-1", testNamespace): regexp.MustCompile(`^Name:\s+test-configmap-1\n(.*\n)*Data`),
+				fmt.Sprintf("%s_configmaps_test-configmap-2", testNamespace): regexp.MustCompile(`^Name:\s+test-configmap-2\n(.*\n)*Data`),
+				fmt.Sprintf("%s_configmaps_test-configmap-3", testNamespace): regexp.MustCompile(`^Name:\s+test-configmap-3\n(.*\n)*Data`),
+			},
+		},
+		{
+			name:             "single resource",
+			requestedObjects: []string{fmt.Sprintf("%s/configmaps/test-configmap-2", testNamespace)},
+			config:           fixture.PeriscopeAccess.ClientConfig,
+			wantErr:          false,
+			want: map[string]*regexp.Regexp{
+				fmt.Sprintf("%s_configmaps_test-configmap-2", testNamespace): regexp.MustCompile(`^Name:\s+test-configmap-2\n(.*\n)*Data`),
+			},
+		},
+		{
+			name:             "default kubeobjects",
+			requestedObjects: defaultKubeObjects,
+			config:           fixture.PeriscopeAccess.ClientConfig,
+			wantErr:          false,
+			want:             defaultResults,
+		},
+	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			runtimeInfo := &utils.RuntimeInfo{
+				KubernetesObjects: tt.requestedObjects,
+			}
+
+			c := NewKubeObjectsCollector(tt.config, runtimeInfo)
+
 			err := c.Collect()
 
-			if (err != nil) != tt.wantErr {
-				t.Errorf("Collect() error = %v, wantErr %v", err, tt.wantErr)
+			if tt.wantErr {
+				if err == nil {
+					t.Error("expected error but none found")
+				}
+				return
 			}
-			raw := c.GetData()
 
-			if len(raw) < tt.want {
-				t.Errorf("len(GetData()) = %v, want %v", len(raw), tt.want)
-			}
+			data := c.GetData()
+
+			compareCollectorData(t, tt.want, data)
 		})
 	}
 }