tests/e2e: add NoInstall option (openservicemesh#1894)

* tests/e2e: add NoInstall option

Adds new install option `NoInstall`.

NoInstall means to run the tests assuming there is already
a running instance of OSM in namespace `osmNamespace`, part
of the test suite flags.

OSM will therefore not be installed/cleaned up between tests,
all other resources used by the tests (clients, other namespaces,
etc) will get cleaned though.

Since a single install is assumed, it assumes inmutability of the
install, so tests that _require_ or test specific parts of the
installation will be skipped.

Dynamic configuration (configmap) is however supported, and tests
that would usually call Install will instead set/reset the dynamic
config map values that the install command would originally set.

[] rebase

.github/workflows/main.yml

@@ -139,10 +139,10 @@ jobs:
         run: make docker-build-osm-controller docker-build-init build-osm
       - name: Run PR tests
         if: ${{ github.event_name == 'pull_request' }}
-        run: go test ./tests/e2e -test.v -ginkgo.v -ginkgo.progress -kindCluster -test.timeout 0 -test.failfast -ginkgo.focus='\[Tier 1\]'
+        run: go test ./tests/e2e -test.v -ginkgo.v -ginkgo.progress -installType=KindCluster -test.timeout 0 -test.failfast -ginkgo.failFast -ginkgo.focus='\[Tier 1\]'
       - name: Run tests for push to main
         if: ${{ github.event_name == 'push' }}
-        run: go test ./tests/e2e -test.v -ginkgo.v -ginkgo.progress -kindCluster -test.timeout 0 -test.failfast
+        run: go test ./tests/e2e -test.v -ginkgo.v -ginkgo.progress -installType=KindCluster -test.timeout 0 -test.failfast -ginkgo.failFast
 
   integration-tresor:
     name: Integration Test with Tresor, SMI traffic policies, and egress disabled

tests/e2e/common.go

@@ -14,11 +14,12 @@ import (
 	"strings"
 	"time"
 
+	. "github.com/onsi/ginkgo"
+
 	"github.com/docker/docker/client"
 	"github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha2"
 	cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
 	certman "github.com/jetstack/cert-manager/pkg/client/clientset/versioned"
-	. "github.com/onsi/ginkgo"
 	"github.com/pkg/errors"
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/chart/loader"
@@ -45,14 +46,27 @@ import (
 const (
 	// contant, default name for the Registry Secret
 	registrySecretName = "acr-creds"
-	// constant, default name for the mesh
-	defaultMeshName = "osm-system"
-	// default image tag
-	defaultImageTag = "latest"
 	// test tag prefix, for NS labeling
 	osmTest = "osmTest"
 )
 
+var (
+	// default name for the mesh
+	defaultOsmNamespace = "osm-system"
+	// default image tag
+	defaultImageTag = "latest"
+	// default cert manager
+	defaultCertManager = "tresor"
+	// default deploy Prometheus
+	defaultDeployPrometheus = false
+	// default deploy Grafana
+	defaultDeployGrafana = false
+	// default deploy Jaeger
+	defaultDeployJaeger = false
+	// default deploy Fluentbit
+	defaultDeployFluentbit = false
+)
+
 func DescribeTierN(tier uint) func(string, func()) bool {
 	return func(name string, body func()) bool {
 		return Describe(fmt.Sprintf("[Tier %d] %s", tier, name), body)
@@ -62,6 +76,32 @@ func DescribeTierN(tier uint) func(string, func()) bool {
 var DescribeTier1 = DescribeTierN(1)
 var DescribeTier2 = DescribeTierN(2)
 
+// InstallType defines several OSM test deployment scenarios
+type InstallType string
+
+const (
+	// SelfInstall uses current kube cluster, installs OSM using CLI
+	SelfInstall InstallType = "SelfInstall"
+	// KindCluster Creates Kind cluster on docker and uses it as cluster, OSM installs through CLI
+	KindCluster InstallType = "KindCluster"
+	// NoInstall uses current kube cluster, assumes an OSM is present in `osmNamespace`
+	NoInstall InstallType = "NoInstall"
+)
+
+// Verifies the instType string flag option is a valid enum type
+func verifyValidInstallType(t InstallType) error {
+	switch t {
+	case SelfInstall:
+		fallthrough
+	case KindCluster:
+		fallthrough
+	case NoInstall:
+		return nil
+	default:
+		return errors.Errorf("%s is not a valid OSM install type", string(t))
+	}
+}
+
 // OsmTestData stores common state, variables and flags for the test at hand
 type OsmTestData struct {
 	T GinkgoTInterface // for common test logging
@@ -70,6 +110,7 @@ type OsmTestData struct {
 	waitForCleanup bool // Forces test to wait for effective deletion of resources upon cleanup
 
 	// OSM install-time variables
+	instType     InstallType // Install type.
 	osmNamespace string
 	osmImageTag  string
 
@@ -79,7 +120,6 @@ type OsmTestData struct {
 	ctrRegistryServer   string // server name. Has to be network reachable
 
 	// Kind cluster related vars
-	kindCluster                    bool   // Create and use a kind cluster
 	clusterName                    string // Kind cluster name (used if kindCluster)
 	cleanupKindClusterBetweenTests bool   // Clean and re-create kind cluster between tests
 	cleanupKindCluster             bool   // Cleanup kind cluster upon test finish
@@ -98,7 +138,8 @@ func registerFlags(td *OsmTestData) {
 	flag.BoolVar(&td.cleanupTest, "cleanupTest", true, "Cleanup test resources when done")
 	flag.BoolVar(&td.waitForCleanup, "waitForCleanup", true, "Wait for effective deletion of resources")
 
-	flag.BoolVar(&td.kindCluster, "kindCluster", false, "Creates kind cluster")
+	flag.StringVar((*string)(&td.instType), "installType", string(SelfInstall), "Type of install/deployment for OSM")
+
 	flag.StringVar(&td.clusterName, "kindClusterName", "osm-e2e", "Name of the Kind cluster to be created")
 	flag.BoolVar(&td.cleanupKindCluster, "cleanupKindCluster", true, "Cleanup kind cluster upon exit")
 	flag.BoolVar(&td.cleanupKindClusterBetweenTests, "cleanupKindClusterBetweenTests", false, "Cleanup kind cluster between tests")
@@ -108,7 +149,7 @@ func registerFlags(td *OsmTestData) {
 	flag.StringVar(&td.ctrRegistryPassword, "ctrRegistrySecret", os.Getenv("CTR_REGISTRY_PASSWORD"), "Container registry secret")
 
 	flag.StringVar(&td.osmImageTag, "osmImageTag", utils.GetEnv("CTR_TAG", defaultImageTag), "OSM image tag")
-	flag.StringVar(&td.osmNamespace, "osmNamespace", utils.GetEnv("K8S_NAMESPACE", defaultMeshName), "OSM mesh name")
+	flag.StringVar(&td.osmNamespace, "osmNamespace", utils.GetEnv("K8S_NAMESPACE", defaultOsmNamespace), "OSM Namespace")
 }
 
 // GetTestNamespaceSelectorMap returns a string-based selector used to refer/select all namespace
@@ -131,11 +172,12 @@ func (td *OsmTestData) AreRegistryCredsPresent() bool {
 func (td *OsmTestData) InitTestData(t GinkgoTInterface) error {
 	td.T = t
 
-	if len(td.ctrRegistryServer) == 0 {
-		td.T.Errorf("Did not read any container registry (is CTR_REGISTRY set?)")
+	err := verifyValidInstallType(td.instType)
+	if err != nil {
+		return err
 	}
 
-	if td.kindCluster && td.clusterProvider == nil {
+	if (td.instType == KindCluster) && td.clusterProvider == nil {
 		td.clusterProvider = cluster.NewProvider()
 		td.T.Logf("Creating local kind cluster")
 		if err := td.clusterProvider.Create(td.clusterName); err != nil {
@@ -168,7 +210,7 @@ func (td *OsmTestData) InitTestData(t GinkgoTInterface) error {
 
 	// After client creations, do a wait for kind cluster just in case it's not done yet coming up
 	// Ballparking pod number. kind has a large number of containers to run by default
-	if td.kindCluster && td.clusterProvider != nil {
+	if (td.instType == KindCluster) && td.clusterProvider != nil {
 		if err := td.WaitForPodsRunningReady("kube-system", 120*time.Second, 5); err != nil {
 			return errors.Wrap(err, "failed to wait for kube-system pods")
 		}
@@ -206,14 +248,14 @@ type InstallOSMOpts struct {
 func (td *OsmTestData) GetOSMInstallOpts() InstallOSMOpts {
 	return InstallOSMOpts{
 		controlPlaneNS:          td.osmNamespace,
-		certManager:             "tresor",
+		certManager:             defaultCertManager,
 		containerRegistryLoc:    td.ctrRegistryServer,
 		containerRegistrySecret: td.ctrRegistryPassword,
 		osmImagetag:             td.osmImageTag,
-		deployGrafana:           false,
-		deployPrometheus:        false,
-		deployJaeger:            false,
-		deployFluentbit:         false,
+		deployGrafana:           defaultDeployGrafana,
+		deployPrometheus:        defaultDeployPrometheus,
+		deployJaeger:            defaultDeployJaeger,
+		deployFluentbit:         defaultDeployFluentbit,
 
 		vaultHost:     "vault." + td.osmNamespace + ".svc.cluster.local",
 		vaultProtocol: "http",
@@ -228,7 +270,7 @@ func (td *OsmTestData) GetOSMInstallOpts() InstallOSMOpts {
 
 // HelmInstallOSM installs an osm control plane using the osm chart which lives in charts/osm
 func (td *OsmTestData) HelmInstallOSM(release, namespace string) error {
-	if td.kindCluster {
+	if td.instType == KindCluster {
 		if err := td.loadOSMImagesIntoKind(); err != nil {
 			return err
 		}
@@ -254,21 +296,72 @@ func (td *OsmTestData) DeleteHelmRelease(name, namespace string) error {
 	return nil
 }
 
-// InstallOSM installs OSM. Right now relies on externally calling the binary and a subset of possible opts
-// TODO: refactor install to be able to call it directly here vs. exec-ing CLI.
+// InstallOSM installs OSM. The behavior of this function is dependant on
+// installType and instOpts
 func (td *OsmTestData) InstallOSM(instOpts InstallOSMOpts) error {
-	if td.kindCluster {
-		if err := td.loadOSMImagesIntoKind(); err != nil {
+	if td.instType == NoInstall {
+		if instOpts.certManager != defaultCertManager ||
+			instOpts.deployPrometheus != defaultDeployPrometheus ||
+			instOpts.deployGrafana != defaultDeployGrafana ||
+			instOpts.deployJaeger != defaultDeployJaeger ||
+			instOpts.deployFluentbit != defaultDeployFluentbit {
+			Skip("Skipping test: NoInstall marked on a test that requires modified install")
+		}
+
+		// TODO: Check there is a valid OSM instance running already in osmNamespace
+
+		// This resets supported dynamic configs expected by the caller
+		err := td.UpdateOSMConfig("egress",
+			fmt.Sprintf("%t", instOpts.egressEnabled))
+		if err != nil {
+			return err
+		}
+		err = td.UpdateOSMConfig("permissive_traffic_policy_mode",
+			fmt.Sprintf("%t", instOpts.enablePermissiveMode))
+		if err != nil {
 			return err
 		}
+
+		return nil
+	}
+
+	if td.instType == KindCluster {
+		td.T.Log("Getting image data")
+		imageNames := []string{
+			"osm-controller",
+			"init",
+		}
+		docker, err := client.NewClientWithOpts(client.WithAPIVersionNegotiation())
+		if err != nil {
+			return errors.Wrap(err, "failed to create docker client")
+		}
+		var imageIDs []string
+		for _, name := range imageNames {
+			imageName := fmt.Sprintf("%s/%s:%s", td.ctrRegistryServer, name, td.osmImageTag)
+			imageIDs = append(imageIDs, imageName)
+		}
+		imageData, err := docker.ImageSave(context.TODO(), imageIDs)
+		if err != nil {
+			return errors.Wrap(err, "failed to get image data")
+		}
+		defer imageData.Close()
+		nodes, err := td.clusterProvider.ListNodes(td.clusterName)
+		if err != nil {
+			return errors.Wrap(err, "failed to list kind nodes")
+		}
+		for _, n := range nodes {
+			td.T.Log("Loading images onto node", n)
+			if err := nodeutils.LoadImageArchive(n, imageData); err != nil {
+				return errors.Wrap(err, "failed to load images")
+			}
+		}
 	}
 
 	if err := td.CreateNs(instOpts.controlPlaneNS, nil); err != nil {
 		return errors.Wrap(err, "failed to create namespace "+instOpts.controlPlaneNS)
 	}
 
 	var args []string
-
 	args = append(args, "install",
 		"--container-registry="+instOpts.containerRegistryLoc,
 		"--osm-image-tag="+instOpts.osmImagetag,
@@ -301,7 +394,7 @@ func (td *OsmTestData) InstallOSM(instOpts InstallOSMOpts) error {
 		)
 	}
 
-	if !td.kindCluster {
+	if !(td.instType == KindCluster) {
 		// Making sure the image is always pulled in registry-based testing
 		args = append(args, "--osm-image-pull-policy=Always")
 	}
@@ -324,7 +417,7 @@ func (td *OsmTestData) InstallOSM(instOpts InstallOSMOpts) error {
 		return errors.Wrap(err, "failed to run osm install")
 	}
 
-	return nil
+	return td.WaitForPodsRunningReady(td.osmNamespace, 90*time.Second, 1)
 }
 
 // GetConfigMap is a wrapper to get a config map by name in a particular namespace
@@ -639,6 +732,7 @@ func (td *OsmTestData) AddNsToMesh(sidecardInject bool, ns ...string) error {
 	return nil
 }
 
+// UpdateOSMConfig updates OSM configmap
 func (td *OsmTestData) UpdateOSMConfig(key, value string) error {
 	patch := []byte(fmt.Sprintf(`{"data": {%q: %q}}`, key, value))
 	_, err := td.client.CoreV1().ConfigMaps(td.osmNamespace).Patch(context.TODO(), "osm-config", types.StrategicMergePatchType, patch, metav1.PatchOptions{})
@@ -802,7 +896,7 @@ func (td *OsmTestData) RunRemote(
 
 // WaitForPodsRunningReady waits for a <n> number of pods on an NS to be running and ready
 func (td *OsmTestData) WaitForPodsRunningReady(ns string, timeout time.Duration, nExpectedRunningPods int) error {
-	td.T.Logf("Wait for pods ready in ns [%s]...", ns)
+	td.T.Logf("Wait up to %v for %d pods ready in ns [%s]...", timeout, nExpectedRunningPods, ns)
 	for start := time.Now(); time.Since(start) < timeout; time.Sleep(2 * time.Second) {
 		pods, err := td.client.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{
 			FieldSelector: "status.phase=Running",
@@ -872,15 +966,21 @@ const (
 
 // Cleanup is Used to cleanup resorces once the test is done
 func (td *OsmTestData) Cleanup(ct CleanupType) {
+	if td.client == nil {
+		// Avoid any cleanup (crash) if no test is run;
+		// init doesn't happen and clientsets are nil
+		return
+	}
+
 	// The condition enters to cleanup K8s resources if
 	// - cleanup is enabled and it's not a kind cluster
 	// - cleanup is enabled and it is a kind cluster, but the kind cluster will NOT be
 	//   destroyed after this test.
 	//   The latter is a condition to speed up and not wait for k8s resources to vanish
 	//   if the current kind cluster has to be destroyed anyway.
 	if td.cleanupTest &&
-		(!td.kindCluster ||
-			(td.kindCluster &&
+		(!(td.instType == KindCluster) ||
+			(td.instType == KindCluster &&
 				(ct == Test && !td.cleanupKindClusterBetweenTests) ||
 				(ct == Suite && !td.cleanupKindCluster))) {
 		// Use selector to refer to all namespaces used in this test
@@ -919,7 +1019,7 @@ func (td *OsmTestData) Cleanup(ct CleanupType) {
 	}
 
 	// Kind cluster deletion, if needed
-	if td.kindCluster && td.clusterProvider != nil {
+	if (td.instType == KindCluster) && td.clusterProvider != nil {
 		if ct == Test && td.cleanupKindClusterBetweenTests || ct == Suite && td.cleanupKindCluster {
 			td.T.Logf("Deleting kind cluster: %s", td.clusterName)
 			if err := td.clusterProvider.Delete(td.clusterName, clientcmd.RecommendedHomeFile); err != nil {

tests/e2e/e2e_deployment_client_server_test.go

@@ -33,7 +33,6 @@ var _ = DescribeTier1("Test HTTP traffic from N deployment client -> 1 deploymen
 		It("Tests HTTP traffic from multiple client deployments to a server deployment", func() {
 			// Install OSM
 			Expect(td.InstallOSM(td.GetOSMInstallOpts())).To(Succeed())
-			Expect(td.WaitForPodsRunningReady(td.osmNamespace, 90*time.Second, 1)).To(Succeed())
 
 			// Server NS
 			Expect(td.CreateNs(destApp, nil)).To(Succeed())

tests/e2e/e2e_egress_test.go

@@ -17,7 +17,6 @@ var _ = DescribeTier1("HTTP and HTTPS Egress", func() {
 			installOpts := td.GetOSMInstallOpts()
 			installOpts.egressEnabled = true
 			Expect(td.InstallOSM(installOpts)).To(Succeed())
-			Expect(td.WaitForPodsRunningReady(td.osmNamespace, 60*time.Second, 1)).To(Succeed())
 
 			// Create Test NS
 			Expect(td.CreateNs(sourceNs, nil)).To(Succeed())

tests/e2e/e2e_fluentbit_test.go

@@ -18,7 +18,6 @@ var _ = DescribeTier2("Test deployment of Fluent Bit sidecar", func() {
 			installOpts := td.GetOSMInstallOpts()
 			installOpts.deployFluentbit = true
 			Expect(td.InstallOSM(installOpts)).To(Succeed())
-			Expect(td.WaitForPodsRunningReady(td.osmNamespace, 60*time.Second, 1)).To(Succeed())
 
 			pods, err := td.client.CoreV1().Pods(td.osmNamespace).List(context.TODO(), metav1.ListOptions{
 				LabelSelector: labels.SelectorFromSet(map[string]string{"app": "osm-controller"}).String(),

tests/e2e/e2e_helm_install_test.go

@@ -8,6 +8,10 @@ import (
 var _ = DescribeTier2("Test osm control plane installation with Helm", func() {
 	Context("Using default values", func() {
 		It("installs osm control plane successfully", func() {
+			if td.instType == NoInstall {
+				Skip("Test is not going through InstallOSM, hence cannot be automatically skipped with NoInstall (#1908)")
+			}
+
 			namespace := "helm-install-namespace"
 			release := "helm-install-osm"
 

tests/e2e/e2e_permissive_test.go

@@ -20,7 +20,6 @@ var _ = DescribeTier1("Permissive Traffic Policy Mode", func() {
 			installOpts := td.GetOSMInstallOpts()
 			installOpts.enablePermissiveMode = true
 			Expect(td.InstallOSM(installOpts)).To(Succeed())
-			Expect(td.WaitForPodsRunningReady(td.osmNamespace, 90*time.Second, 1)).To(Succeed())
 
 			// Create Test NS
 			for _, n := range ns {

tests/e2e/e2e_pod_client_server_test.go

@@ -20,7 +20,6 @@ var _ = DescribeTier1("Test HTTP traffic from 1 pod client -> 1 pod server", fun
 		It("Tests HTTP traffic for client pod -> server pod", func() {
 			// Install OSM
 			Expect(td.InstallOSM(td.GetOSMInstallOpts())).To(Succeed())
-			Expect(td.WaitForPodsRunningReady(td.osmNamespace, 90*time.Second, 1)).To(Succeed())
 
 			// Create Test NS
 			for _, n := range ns {

tests/e2e/e2e_trafficsplit_same_sa_test.go

@@ -53,7 +53,6 @@ var _ = DescribeTier1("Test TrafficSplit where each backend shares the same Serv
 		It("Tests HTTP traffic from Clients to the traffic split Cluster IP", func() {
 			// Install OSM
 			Expect(td.InstallOSM(td.GetOSMInstallOpts())).To(Succeed())
-			Expect(td.WaitForPodsRunningReady(td.osmNamespace, 90*time.Second, 1)).To(Succeed())
 
 			// Create NSs
 			Expect(td.CreateMultipleNs(allNamespaces...)).To(Succeed())

tests/e2e/e2e_trafficsplit_test.go

@@ -52,7 +52,6 @@ var _ = DescribeTier1("Test HTTP from N Clients deployments to 1 Server deployme
 		It("Tests HTTP traffic from Clients to the traffic split Cluster IP", func() {
 			// Install OSM
 			Expect(td.InstallOSM(td.GetOSMInstallOpts())).To(Succeed())
-			Expect(td.WaitForPodsRunningReady(td.osmNamespace, 90*time.Second, 1)).To(Succeed())
 
 			// Create NSs
 			Expect(td.CreateMultipleNs(allNamespaces...)).To(Succeed())