Add NFR tests logging

Author: tataruty

tests/framework/generate_manifests.go

@@ -151,7 +151,11 @@ func decodeObjects(reader io.Reader) ([]client.Object, error) {
 }
 
 // GenerateScaleListenerObjects generates objects for a given number of listeners for the scale test.
-func GenerateScaleListenerObjects(numListeners int, tls bool) (ScaleObjects, error) {
+func GenerateScaleListenerObjects(
+	numListeners int,
+	tls bool,
+	opts ...Option,
+) (ScaleObjects, error) {
 	var result ScaleObjects
 
 	listeners := make([]listener, 0)
@@ -184,7 +188,13 @@ func GenerateScaleListenerObjects(numListeners int, tls bool) (ScaleObjects, err
 
 		backends = append(backends, backendName)
 
-		GinkgoWriter.Printf("Generating manifests for listeners %v and route %v\n", listeners, r)
+		options := &Options{logEnabled: true}
+		for _, opt := range opts {
+			opt(options)
+		}
+		if !options.logEnabled {
+			GinkgoWriter.Printf("Generating manifests for listeners %v and route %v\n", listeners, r)
+		}
 		objects, err := generateManifests(listeners, []route{r})
 		if err != nil {
 			GinkgoWriter.Printf("Error generating manifests: %v\n", err)

tests/framework/info.go

@@ -99,12 +99,12 @@ func GetBuildInfo() (commitHash string, commitTime string, dirtyBuild string) {
 }
 
 // AddNginxLogsAndEventsToReport adds nginx logs and events from the namespace to the report if the spec failed.
-func AddNginxLogsAndEventsToReport(rm ResourceManager, namespace string) {
+func AddNginxLogsAndEventsToReport(rm ResourceManager, namespace string, opts ...Option) {
 	if CurrentSpecReport().Failed() {
 		GinkgoWriter.Printf("Current spec failed. Adding Nginx logs and events to report for namespace %q\n", namespace)
 		var returnLogs string
 
-		nginxPodNames, _ := GetReadyNginxPodNames(rm.K8sClient, namespace, rm.TimeoutConfig.GetStatusTimeout)
+		nginxPodNames, _ := GetReadyNginxPodNames(rm.K8sClient, namespace, rm.TimeoutConfig.GetStatusTimeout, opts...)
 
 		for _, nginxPodName := range nginxPodNames {
 			returnLogs += fmt.Sprintf("Logs for Nginx Pod %s:\n", nginxPodName)

tests/framework/prometheus.go

@@ -471,22 +471,31 @@ func CreateMetricExistChecker(
 	query string,
 	getTime func() time.Time,
 	modifyTime func(),
+	opts ...Option,
 ) func() error {
 	return func() error {
 		queryWithTimestamp := fmt.Sprintf("%s @ %d", query, getTime().Unix())
+		options := &Options{logEnabled: true}
+		for _, opt := range opts {
+			opt(options)
+		}
 
 		result, err := promInstance.Query(queryWithTimestamp)
 		if err != nil {
 			queryErr := fmt.Errorf("failed to query Prometheus: %w", err)
-			GinkgoWriter.Printf("ERROR during creating metric existence checker: %v\n", queryErr)
+			if options.logEnabled {
+				GinkgoWriter.Printf("ERROR during creating metric existence checker: %v\n", queryErr)
+			}
 
 			return queryErr
 		}
 
 		if result.String() == "" {
 			modifyTime()
 			emptyResultErr := errors.New("empty result")
-			GinkgoWriter.Printf("ERROR during creating metric existence checker: %v\n", emptyResultErr)
+			if options.logEnabled {
+				GinkgoWriter.Printf("ERROR during creating metric existence checker: %v\n", emptyResultErr)
+			}
 
 			return emptyResultErr
 		}

tests/framework/request.go

@@ -170,11 +170,28 @@ func K8sGet(
 	return err
 }
 
-func K8sList(ctx context.Context, client k8sClient.Client, list k8sClient.ObjectList) error {
+func K8sList(
+	ctx context.Context,
+	client k8sClient.Client,
+	list k8sClient.ObjectList,
+) error {
 	err := client.List(ctx, list)
 	if err != nil {
 		return fmt.Errorf("ERROR listing resource %q: %w", list.GetObjectKind(), err)
 	}
 
 	return err
 }
+
+func K8sCreate(
+	ctx context.Context,
+	client k8sClient.Client,
+	obj k8sClient.Object,
+) error {
+	err := client.Create(ctx, obj)
+	if err != nil {
+		return fmt.Errorf("ERROR creating resource %q: %w", obj.GetName(), err)
+	}
+
+	return err
+}

tests/framework/resourcemanager.go

@@ -553,6 +553,7 @@ func (rm *ResourceManager) WaitForPodsToBeReady(ctx context.Context, namespace s
 					}
 				}
 			}
+			GinkgoWriter.Printf("Pods ready: %d out of %d in namespace %q\n", podsReady, len(podList.Items), namespace)
 
 			return podsReady == len(podList.Items), nil
 		},
@@ -1145,17 +1146,24 @@ func GetReadyNginxPodNames(
 			return len(nginxPodNames) > 0, nil
 		},
 	)
+	options := &Options{logEnabled: true}
+	for _, opt := range opts {
+		opt(options)
+	}
+
 	if err != nil {
 		waitingPodsErr := fmt.Errorf("timed out waiting for NGINX Pods to be ready: %w", err)
-		GinkgoWriter.Printf("ERROR occurred during waiting for NGINX Pods to be ready in namespace %q, error: %s\n",
-			namespace,
-			waitingPodsErr,
-		)
+		if options.logEnabled {
+			GinkgoWriter.Printf("ERROR occurred during waiting for NGINX Pods to be ready in namespace %q, error: %s\n",
+				namespace,
+				waitingPodsErr,
+			)
+		}
 
 		return nil, waitingPodsErr
 	}
 	GinkgoWriter.Printf(
-		"Successfully found ready NGINX Pod names in namespace %q: %v\n",
+		"Successfully found ready NGINX Pod name(s) in namespace %q: %v\n",
 		namespace,
 		nginxPodNames,
 	)
@@ -1263,8 +1271,18 @@ func (rm *ResourceManager) WaitForGatewayObservedGeneration(
 
 // GetNginxConfig uses crossplane to get the nginx configuration and convert it to JSON.
 // If the crossplane image is loaded locally on the node, crossplaneImageRepo can be empty.
-func (rm *ResourceManager) GetNginxConfig(nginxPodName, namespace, crossplaneImageRepo string) (*Payload, error) {
+func (rm *ResourceManager) GetNginxConfig(
+	nginxPodName,
+	namespace,
+	crossplaneImageRepo string,
+	opts ...Option,
+) (*Payload, error) {
 	GinkgoWriter.Printf("Getting NGINX config from pod %q in namespace %q\n", nginxPodName, namespace)
+	options := &Options{logEnabled: true}
+	for _, opt := range opts {
+		opt(options)
+	}
+
 	if err := injectCrossplaneContainer(
 		rm.ClientGoClient,
 		rm.TimeoutConfig.UpdateTimeout,
@@ -1310,10 +1328,12 @@ func (rm *ResourceManager) GetNginxConfig(nginxPodName, namespace, crossplaneIma
 		},
 	); err != nil {
 		containerErr := fmt.Errorf("could not connect to ephemeral container: %w", err)
-		GinkgoWriter.Printf("ERROR occurred during waiting for NGINX Pods to be ready in namespace %q, error: %s\n",
-			namespace,
-			containerErr,
-		)
+		if options.logEnabled {
+			GinkgoWriter.Printf("ERROR occurred during waiting for NGINX Pods to be ready in namespace %q, error: %s\n",
+				namespace,
+				containerErr,
+			)
+		}
 
 		return nil, containerErr
 	}

tests/framework/results.go

@@ -30,6 +30,8 @@ func CreateResultsFile(filename string) (*os.File, error) {
 	GinkgoWriter.Printf("Creating results file %q\n", filename)
 	outFile, err := OpenFile(filename, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0o644)
 	if err != nil {
+		GinkgoWriter.Printf("ERROR occurred during creating results file %q, error: %s\n", filename, err)
+
 		return nil, err
 	}
 
@@ -61,10 +63,14 @@ func WriteSystemInfoToFile(file *os.File, ci ClusterInfo, plus bool) error {
 		plus, commit, date, dirty, clusterType, ci.NodeCount, ci.K8sVersion, ci.CPUCountPerNode, ci.MemoryPerNode, ci.MaxPodsPerNode,
 	)
 	if _, err := fmt.Fprint(file, text); err != nil {
+		GinkgoWriter.Printf("ERROR occurred during writing system info to results file, error: %s\n", err)
+
 		return err
 	}
 	if ci.IsGKE {
 		if _, err := fmt.Fprintf(file, "- Zone: %s\n- Instance Type: %s\n", ci.GkeZone, ci.GkeInstanceType); err != nil {
+			GinkgoWriter.Printf("ERROR occurred during writing GKE info to results file, error: %s\n", err)
+
 			return err
 		}
 	}
@@ -246,12 +252,16 @@ func NewCSVResultsWriter(resultsDir, fileName string, resultHeaders ...string) (
 
 	file, err := OpenFile(filepath.Join(resultsDir, fileName), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644)
 	if err != nil {
+		GinkgoWriter.Printf("ERROR occurred during creating CSV results file %q, error: %s\n", fileName, err)
+
 		return nil, nil, err
 	}
 
 	writer := csv.NewWriter(file)
 
 	if err = WriteCSVRecord(writer, resultHeaders); err != nil {
+		GinkgoWriter.Printf("ERROR occurred during writing CSV headers to file %q, error: %s\n", fileName, err)
+
 		return nil, nil, err
 	}
 

tests/suite/dataplane_perf_test.go

@@ -66,7 +66,12 @@ var _ = Describe("Dataplane performance", Ordered, Label("nfr", "performance"),
 		Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed())
 		Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed())
 
-		nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetTimeout)
+		nginxPodNames, err := framework.GetReadyNginxPodNames(
+			k8sClient,
+			namespace,
+			timeoutConfig.GetTimeout,
+			framework.WithLoggingDisabled(),
+		)
 		Expect(err).ToNot(HaveOccurred())
 		Expect(nginxPodNames).To(HaveLen(1))
 
@@ -88,7 +93,7 @@ var _ = Describe("Dataplane performance", Ordered, Label("nfr", "performance"),
 	})
 
 	AfterAll(func() {
-		framework.AddNginxLogsAndEventsToReport(resourceManager, namespace)
+		framework.AddNginxLogsAndEventsToReport(resourceManager, namespace, framework.WithLoggingDisabled())
 		cleanUpPortForward()
 
 		Expect(resourceManager.DeleteFromFiles(files, namespace)).To(Succeed())

tests/suite/longevity_test.go

@@ -82,7 +82,7 @@ var _ = Describe("Longevity", Label("longevity-setup", "longevity-teardown"), fu
 		Expect(writeTrafficResults(resultsFile, homeDir, "coffee.txt", "HTTP")).To(Succeed())
 		Expect(writeTrafficResults(resultsFile, homeDir, "tea.txt", "HTTPS")).To(Succeed())
 
-		framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name)
+		framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name, framework.WithLoggingDisabled())
 		Expect(resourceManager.DeleteFromFiles(files, ns.Name)).To(Succeed())
 		Expect(resourceManager.DeleteNamespace(ns.Name)).To(Succeed())
 	})

tests/suite/reconfig_test.go

@@ -24,7 +24,7 @@ import (
 )
 
 // Cluster node size must be greater than or equal to 4 for test to perform correctly.
-var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("tina", "nfr", "reconfiguration"), func() {
+var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "reconfiguration"), func() {
 	const (
 		// used for cleaning up resources
 		maxResourceCount = 150
@@ -136,7 +136,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("tina", "
 					Name: "namespace" + strconv.Itoa(i),
 				},
 			}
-			Expect(k8sClient.Create(ctx, &ns)).To(Succeed())
+			Expect(framework.K8sCreate(ctx, k8sClient, &ns)).To(Succeed())
 		}
 
 		Expect(resourceManager.Apply([]client.Object{&reconfigNamespace})).To(Succeed())
@@ -246,6 +246,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("tina", "
 		if stringTimeToReadyTotal == "0" {
 			stringTimeToReadyTotal = "< 1"
 		}
+		GinkgoWriter.Printf("Calculated time to ready total for %q: %s\n", nginxPodName, stringTimeToReadyTotal)
 
 		return stringTimeToReadyTotal
 	}
@@ -351,6 +352,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("tina", "
 							k8sClient,
 							reconfigNamespace.Name,
 							timeoutConfig.GetStatusTimeout,
+							framework.WithLoggingDisabled(),
 						)
 						return len(nginxPodNames) == 1 && err == nil
 					}).
@@ -416,6 +418,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("tina", "
 							k8sClient,
 							reconfigNamespace.Name,
 							timeoutConfig.GetStatusTimeout,
+							framework.WithLoggingDisabled(),
 						)
 						return len(nginxPodNames) == 1 && err == nil
 					}).
@@ -469,7 +472,11 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("tina", "
 	})
 
 	AfterEach(func() {
-		framework.AddNginxLogsAndEventsToReport(resourceManager, reconfigNamespace.Name)
+		framework.AddNginxLogsAndEventsToReport(
+			resourceManager,
+			reconfigNamespace.Name,
+			framework.WithLoggingDisabled(),
+		)
 
 		Expect(cleanupResources()).Should(Succeed())
 		teardown(releaseName)