From 146833a36495acd454add9da5dd395a0d466288a Mon Sep 17 00:00:00 2001 From: Nitya Dhanushkodi Date: Tue, 1 Dec 2020 19:23:29 -0800 Subject: [PATCH] Log envoy config for mesh gateways on test failures (#718) If the mesh gateway tests fail, the acceptance tests will now write two files -envoy-configdump.json and envoy-clusters.json which will have information from the envoy admin /config_dump endpoint and /clusters endpoint. Currently we write the outputs of `kubectl logs POD` and `kubectl describe POD` for the acceptance tests, but while debugging EKS acceptance test failures, it was hard to know from our existing logs whether the mesh gateway itself is unreachable or if the consul servers behind it are unhealthy. We'd be able to tell this from looking at the envoy config dump and cluster config to see what envoy's view of the consul servers are. --- test/acceptance/framework/k8s/debug.go | 34 +++++++++++++++++++++++--- test/acceptance/go.sum | 1 - 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/test/acceptance/framework/k8s/debug.go b/test/acceptance/framework/k8s/debug.go index 8078130ab6..5613db0b11 100644 --- a/test/acceptance/framework/k8s/debug.go +++ b/test/acceptance/framework/k8s/debug.go @@ -31,25 +31,51 @@ func WritePodsDebugInfoIfFailed(t *testing.T, kubectlOptions *k8s.KubectlOptions testDebugDirectory := filepath.Join(debugDirectory, t.Name(), contextName) require.NoError(t, os.MkdirAll(testDebugDirectory, 0755)) - logger.Logf(t, "dumping logs and pod info for %s to %s", labelSelector, testDebugDirectory) + logger.Logf(t, "dumping logs, pod info, and envoy config for %s to %s", labelSelector, testDebugDirectory) + + // Describe and get logs for any pods. pods, err := client.CoreV1().Pods(kubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: labelSelector}) require.NoError(t, err) for _, pod := range pods.Items { // Get logs for each pod, passing the discard logger to make sure secrets aren't printed to test logs. logs, err := RunKubectlAndGetOutputWithLoggerE(t, kubectlOptions, terratestLogger.Discard, "logs", "--all-containers=true", pod.Name) - - // Write logs or err to file name .log - logFilename := filepath.Join(testDebugDirectory, fmt.Sprintf("%s.log", pod.Name)) if err != nil { logs = fmt.Sprintf("Error getting logs: %s: %s", err, logs) } + + // Write logs or err to file name .log + logFilename := filepath.Join(testDebugDirectory, fmt.Sprintf("%s.log", pod.Name)) require.NoError(t, ioutil.WriteFile(logFilename, []byte(logs), 0600)) // Describe pod and write it to a file. writeResourceInfoToFile(t, pod.Name, "pod", testDebugDirectory, kubectlOptions) } + // Get envoy configuration from the mesh gateways, if there are any. + meshGatewayPods, err := client.CoreV1().Pods(kubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "component=mesh-gateway"}) + require.NoError(t, err) + + for _, mpod := range meshGatewayPods.Items { + // Get configdump from mesh gateway, passing the discard logger since we only need these logs written to the file (below). + configDump, err := RunKubectlAndGetOutputWithLoggerE(t, kubectlOptions, terratestLogger.Discard, "exec", mpod.Name, "-c", "lifecycle-sidecar", "--", "curl", "-s", "localhost:19000/config_dump?format=json") + if err != nil { + configDump = fmt.Sprintf("Error getting config_dump: %s: %s", err, configDump) + } + // Get cluster config from mesh gateway, passing the discard logger since we only need these logs written to the file (below). + clusters, err := RunKubectlAndGetOutputWithLoggerE(t, kubectlOptions, terratestLogger.Discard, "exec", mpod.Name, "-c", "lifecycle-sidecar", "--", "curl", "-s", "localhost:19000/clusters?format=json") + if err != nil { + clusters = fmt.Sprintf("Error getting clusters: %s: %s", err, clusters) + } + + // Write config/clusters or err to file name -envoy-[configdump/clusters].json + configDumpFilename := filepath.Join(testDebugDirectory, fmt.Sprintf("%s-envoy-configdump.json", mpod.Name)) + clustersFilename := filepath.Join(testDebugDirectory, fmt.Sprintf("%s-envoy-clusters.json", mpod.Name)) + require.NoError(t, ioutil.WriteFile(configDumpFilename, []byte(configDump), 0600)) + require.NoError(t, ioutil.WriteFile(clustersFilename, []byte(clusters), 0600)) + + } + // Describe any stateful sets. statefulSets, err := client.AppsV1().StatefulSets(kubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: labelSelector}) for _, statefulSet := range statefulSets.Items { diff --git a/test/acceptance/go.sum b/test/acceptance/go.sum index 68977d7a5a..30333b37bb 100644 --- a/test/acceptance/go.sum +++ b/test/acceptance/go.sum @@ -231,7 +231,6 @@ github.com/gruntwork-io/gruntwork-cli v0.7.0 h1:YgSAmfCj9c61H+zuvHwKfYUwlMhu5arn github.com/gruntwork-io/gruntwork-cli v0.7.0/go.mod h1:jp6Z7NcLF2avpY8v71fBx6hds9eOFPELSuD/VPv7w00= github.com/gruntwork-io/terratest v0.30.15 h1:/OAnHKEERSsy03wYi8dyEwnpYy2XV8wpzwq0c4BHvoc= github.com/gruntwork-io/terratest v0.30.15/go.mod h1:vl/YEB2AEqVZOv9zg0nlBX+fgfqcqNcpqNfjvhRFLXo= -github.com/hashicorp/consul-helm v0.25.0 h1:aXYK733Z7lIWdiVNczND+bBwrheGNtp6wvLe7N/8TgU= github.com/hashicorp/consul/api v1.4.1-0.20201015173526-812fe06d6c64 h1:8/zXC/mxIXKRqcRACIu/G016QKE/XWWBXsuNnJO3O9s= github.com/hashicorp/consul/api v1.4.1-0.20201015173526-812fe06d6c64/go.mod h1:duA0cMzNQkvdlj8Ayz6LmyaMUta7iVW8qY2w2bic/Iw= github.com/hashicorp/consul/sdk v0.6.0 h1:FfhMEkwvQl57CildXJyGHnwGGM4HMODGyfjGwNM1Vdw=