Log envoy config for mesh gateways on test failures (hashicorp#718)

If the mesh gateway tests fail, the acceptance tests will now write two
files <PODNAME>-envoy-configdump.json and <PODNAME>envoy-clusters.json
which will have information from the envoy admin /config_dump endpoint
and /clusters endpoint.

Currently we write the outputs of `kubectl logs POD` and `kubectl describe
POD` for the acceptance tests, but while debugging EKS acceptance test
failures, it was hard to know from our existing logs whether the mesh
gateway itself is unreachable or if the consul servers behind it are
unhealthy. We'd be able to tell this from looking at the envoy config
dump and cluster config to see what envoy's view of the consul servers
are.

test/acceptance/framework/k8s/debug.go

@@ -31,25 +31,51 @@ func WritePodsDebugInfoIfFailed(t *testing.T, kubectlOptions *k8s.KubectlOptions
 		testDebugDirectory := filepath.Join(debugDirectory, t.Name(), contextName)
 		require.NoError(t, os.MkdirAll(testDebugDirectory, 0755))
 
-		logger.Logf(t, "dumping logs and pod info for %s to %s", labelSelector, testDebugDirectory)
+		logger.Logf(t, "dumping logs, pod info, and envoy config for %s to %s", labelSelector, testDebugDirectory)
+
+		// Describe and get logs for any pods.
 		pods, err := client.CoreV1().Pods(kubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: labelSelector})
 		require.NoError(t, err)
 
 		for _, pod := range pods.Items {
 			// Get logs for each pod, passing the discard logger to make sure secrets aren't printed to test logs.
 			logs, err := RunKubectlAndGetOutputWithLoggerE(t, kubectlOptions, terratestLogger.Discard, "logs", "--all-containers=true", pod.Name)
-
-			// Write logs or err to file name <pod.Name>.log
-			logFilename := filepath.Join(testDebugDirectory, fmt.Sprintf("%s.log", pod.Name))
 			if err != nil {
 				logs = fmt.Sprintf("Error getting logs: %s: %s", err, logs)
 			}
+
+			// Write logs or err to file name <pod.Name>.log
+			logFilename := filepath.Join(testDebugDirectory, fmt.Sprintf("%s.log", pod.Name))
 			require.NoError(t, ioutil.WriteFile(logFilename, []byte(logs), 0600))
 
 			// Describe pod and write it to a file.
 			writeResourceInfoToFile(t, pod.Name, "pod", testDebugDirectory, kubectlOptions)
 		}
 
+		// Get envoy configuration from the mesh gateways, if there are any.
+		meshGatewayPods, err := client.CoreV1().Pods(kubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "component=mesh-gateway"})
+		require.NoError(t, err)
+
+		for _, mpod := range meshGatewayPods.Items {
+			// Get configdump from mesh gateway, passing the discard logger since we only need these logs written to the file (below).
+			configDump, err := RunKubectlAndGetOutputWithLoggerE(t, kubectlOptions, terratestLogger.Discard, "exec", mpod.Name, "-c", "lifecycle-sidecar", "--", "curl", "-s", "localhost:19000/config_dump?format=json")
+			if err != nil {
+				configDump = fmt.Sprintf("Error getting config_dump: %s: %s", err, configDump)
+			}
+			// Get cluster config from mesh gateway, passing the discard logger since we only need these logs written to the file (below).
+			clusters, err := RunKubectlAndGetOutputWithLoggerE(t, kubectlOptions, terratestLogger.Discard, "exec", mpod.Name, "-c", "lifecycle-sidecar", "--", "curl", "-s", "localhost:19000/clusters?format=json")
+			if err != nil {
+				clusters = fmt.Sprintf("Error getting clusters: %s: %s", err, clusters)
+			}
+
+			// Write config/clusters or err to file name <pod.Name>-envoy-[configdump/clusters].json
+			configDumpFilename := filepath.Join(testDebugDirectory, fmt.Sprintf("%s-envoy-configdump.json", mpod.Name))
+			clustersFilename := filepath.Join(testDebugDirectory, fmt.Sprintf("%s-envoy-clusters.json", mpod.Name))
+			require.NoError(t, ioutil.WriteFile(configDumpFilename, []byte(configDump), 0600))
+			require.NoError(t, ioutil.WriteFile(clustersFilename, []byte(clusters), 0600))
+
+		}
+
 		// Describe any stateful sets.
 		statefulSets, err := client.AppsV1().StatefulSets(kubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: labelSelector})
 		for _, statefulSet := range statefulSets.Items {

test/acceptance/go.sum

@@ -231,7 +231,6 @@ github.com/gruntwork-io/gruntwork-cli v0.7.0 h1:YgSAmfCj9c61H+zuvHwKfYUwlMhu5arn
 github.com/gruntwork-io/gruntwork-cli v0.7.0/go.mod h1:jp6Z7NcLF2avpY8v71fBx6hds9eOFPELSuD/VPv7w00=
 github.com/gruntwork-io/terratest v0.30.15 h1:/OAnHKEERSsy03wYi8dyEwnpYy2XV8wpzwq0c4BHvoc=
 github.com/gruntwork-io/terratest v0.30.15/go.mod h1:vl/YEB2AEqVZOv9zg0nlBX+fgfqcqNcpqNfjvhRFLXo=
-github.com/hashicorp/consul-helm v0.25.0 h1:aXYK733Z7lIWdiVNczND+bBwrheGNtp6wvLe7N/8TgU=
 github.com/hashicorp/consul/api v1.4.1-0.20201015173526-812fe06d6c64 h1:8/zXC/mxIXKRqcRACIu/G016QKE/XWWBXsuNnJO3O9s=
 github.com/hashicorp/consul/api v1.4.1-0.20201015173526-812fe06d6c64/go.mod h1:duA0cMzNQkvdlj8Ayz6LmyaMUta7iVW8qY2w2bic/Iw=
 github.com/hashicorp/consul/sdk v0.6.0 h1:FfhMEkwvQl57CildXJyGHnwGGM4HMODGyfjGwNM1Vdw=