From c37caccdd98e993bfa42e4b25fb99fcb3c548bac Mon Sep 17 00:00:00 2001 From: Shashank Ram <21697719+shashankram@users.noreply.github.com> Date: Wed, 16 Dec 2020 16:53:59 -0800 Subject: [PATCH] tests/e2e: add connectivity test with osm-controller restart (#2212) This change adds an e2e test to test the connectivity between client and server before/during/after osm-controller restarts. Previously this was resulting in 503s due to issue #2131 which has been fixed. Resolves #2146 and tests #2145. Signed-off-by: Shashank Ram --- tests/e2e/e2e_controller_restart_test.go | 127 +++++++++++++++++++++++ tests/framework/common.go | 27 +++++ 2 files changed, 154 insertions(+) create mode 100644 tests/e2e/e2e_controller_restart_test.go diff --git a/tests/e2e/e2e_controller_restart_test.go b/tests/e2e/e2e_controller_restart_test.go new file mode 100644 index 0000000000..79eb80d64d --- /dev/null +++ b/tests/e2e/e2e_controller_restart_test.go @@ -0,0 +1,127 @@ +package e2e + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + . "github.com/openservicemesh/osm/tests/framework" +) + +var _ = OSMDescribe("Test HTTP traffic from 1 pod client -> 1 pod server before and after osm-controller restart", + OSMDescribeInfo{ + Tier: 2, + Bucket: 1, + }, + func() { + Context("SimpleClientServer traffic test involving osm-controller restart: HTTP", func() { + testHTTPTrafficWithControllerRestart() + }) + }) + +func testHTTPTrafficWithControllerRestart() { + { + const sourceName = "client" + const destName = "server" + var ns = []string{sourceName, destName} + + It("Tests HTTP traffic for client pod -> server pod", func() { + // Install OSM + Expect(Td.InstallOSM(Td.GetOSMInstallOpts())).To(Succeed()) + + // Create Test NS + for _, n := range ns { + Expect(Td.CreateNs(n, nil)).To(Succeed()) + Expect(Td.AddNsToMesh(true, n)).To(Succeed()) + } + + // Get simple pod definitions for the HTTP server + svcAccDef, podDef, svcDef := Td.SimplePodApp( + SimplePodAppDef{ + Name: destName, + Namespace: destName, + Image: "kennethreitz/httpbin", + Ports: []int{80}, + }) + + _, err := Td.CreateServiceAccount(destName, &svcAccDef) + Expect(err).NotTo(HaveOccurred()) + _, err = Td.CreatePod(destName, podDef) + Expect(err).NotTo(HaveOccurred()) + dstSvc, err := Td.CreateService(destName, svcDef) + Expect(err).NotTo(HaveOccurred()) + + // Expect it to be up and running in it's receiver namespace + Expect(Td.WaitForPodsRunningReady(destName, 90*time.Second, 1)).To(Succeed()) + + srcPod := setupSource(sourceName, false /* no service for client */) + + By("Creating SMI policies") + // Deploy allow rule client->server + httpRG, trafficTarget := Td.CreateSimpleAllowPolicy( + SimpleAllowPolicy{ + RouteGroupName: "routes", + TrafficTargetName: "test-target", + + SourceNamespace: sourceName, + SourceSVCAccountName: sourceName, + + DestinationNamespace: destName, + DestinationSvcAccountName: destName, + }) + + // Configs have to be put into a monitored NS + _, err = Td.CreateHTTPRouteGroup(sourceName, httpRG) + Expect(err).NotTo(HaveOccurred()) + _, err = Td.CreateTrafficTarget(sourceName, trafficTarget) + Expect(err).NotTo(HaveOccurred()) + + // All ready. Expect client to reach server + clientToServer := HTTPRequestDef{ + SourceNs: sourceName, + SourcePod: srcPod.Name, + SourceContainer: sourceName, + + Destination: fmt.Sprintf("%s.%s", dstSvc.Name, dstSvc.Namespace), + } + + srcToDestStr := fmt.Sprintf("%s -> %s", + fmt.Sprintf("%s/%s", sourceName, srcPod.Name), + clientToServer.Destination) + + cond := Td.WaitForRepeatedSuccess(func() bool { + result := Td.HTTPRequest(clientToServer) + + if result.Err != nil || result.StatusCode != 200 { + Td.T.Logf("> (%s) HTTP Req failed %d %v", + srcToDestStr, result.StatusCode, result.Err) + return false + } + Td.T.Logf("> (%s) HTTP Req succeeded: %d", srcToDestStr, result.StatusCode) + return true + }, 5, 90*time.Second) + + Expect(cond).To(BeTrue(), "Failed testing HTTP traffic from source pod %s to destination service %s", srcPod.Name, dstSvc.Name) + + // Restart osm-controller + By("Restarting OSM controller") + Expect(Td.RestartOSMController(Td.GetOSMInstallOpts())).To(Succeed()) + + // Expect client to reach server + cond = Td.WaitForRepeatedSuccess(func() bool { + result := Td.HTTPRequest(clientToServer) + + if result.Err != nil || result.StatusCode != 200 { + Td.T.Logf("> (%s) HTTP Req failed %d %v", + srcToDestStr, result.StatusCode, result.Err) + return false + } + Td.T.Logf("> (%s) HTTP Req succeeded: %d", srcToDestStr, result.StatusCode) + return true + }, 20, 40*time.Second) + Expect(cond).To(BeTrue(), "Failed testing HTTP traffic from source pod %s to destination service %s after osm-controller restart", srcPod.Name, dstSvc.Name) + }) + } +} diff --git a/tests/framework/common.go b/tests/framework/common.go index a062dfebee..0377893ed4 100644 --- a/tests/framework/common.go +++ b/tests/framework/common.go @@ -45,6 +45,7 @@ import ( "sigs.k8s.io/kind/pkg/cluster/nodeutils" "github.com/openservicemesh/osm/pkg/cli" + "github.com/openservicemesh/osm/pkg/constants" "github.com/openservicemesh/osm/pkg/utils" ) @@ -534,6 +535,32 @@ func (td *OsmTestData) InstallOSM(instOpts InstallOSMOpts) error { return nil } +// RestartOSMController restarts the osm-controller pod in the installed controller's namespace +func (td *OsmTestData) RestartOSMController(instOpts InstallOSMOpts) error { + labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{"app": constants.OSMControllerName}} + listOptions := metav1.ListOptions{ + LabelSelector: labels.Set(labelSelector.MatchLabels).String(), + } + + controllerPods, err := td.Client.CoreV1().Pods(instOpts.ControlPlaneNS).List(context.TODO(), listOptions) + if err != nil { + return errors.Wrap(err, "error fetching controller pod") + } + if len(controllerPods.Items) != 1 { + return errors.Errorf("expected 1 osm-controller pod, got %d", len(controllerPods.Items)) + } + + pod := controllerPods.Items[0] + + // Delete the pod and let k8s spin it up again + err = td.Client.CoreV1().Pods(instOpts.ControlPlaneNS).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{}) + if err != nil { + return errors.Wrap(err, "erorr deleting osm-controller pod") + } + + return nil +} + // GetConfigMap is a wrapper to get a config map by name in a particular namespace func (td *OsmTestData) GetConfigMap(name, namespace string) (*corev1.ConfigMap, error) { configmap, err := td.Client.CoreV1().ConfigMaps(namespace).Get(context.Background(), name, metav1.GetOptions{})