diff --git a/test/e2e/framework/kubernetes/install-hubble-helm.go b/test/e2e/framework/kubernetes/install-hubble-helm.go new file mode 100644 index 0000000000..5e225d08e6 --- /dev/null +++ b/test/e2e/framework/kubernetes/install-hubble-helm.go @@ -0,0 +1,155 @@ +package kubernetes + +import ( + "context" + "fmt" + "log" + "os" + "strings" + "time" + + "github.com/microsoft/retina/test/e2e/common" + generic "github.com/microsoft/retina/test/e2e/framework/generic" + "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/cli" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +const ( + HubbleNamespace = "kube-system" + HubbleUIApp = "hubble-ui" + HubbleRelayApp = "hubble-relay" +) + +type ValidateHubbleStep struct { + Namespace string + ReleaseName string + KubeConfigFilePath string + ChartPath string + TagEnv string +} + +func (v *ValidateHubbleStep) Run() error { + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeoutSeconds*time.Second) + defer cancel() + + settings := cli.New() + settings.KubeConfig = v.KubeConfigFilePath + actionConfig := new(action.Configuration) + + err := actionConfig.Init(settings.RESTClientGetter(), v.Namespace, os.Getenv("HELM_DRIVER"), log.Printf) + if err != nil { + return fmt.Errorf("failed to initialize helm action config: %w", err) + } + + // Creating extra namespace to deploy test pods + err = CreateNamespaceFn(v.KubeConfigFilePath, common.TestPodNamespace) + if err != nil { + return fmt.Errorf("failed to create namespace %s: %w", v.Namespace, err) + } + + tag := os.Getenv(generic.DefaultTagEnv) + if tag == "" { + return fmt.Errorf("tag is not set: %w", errEmpty) + } + imageRegistry := os.Getenv(generic.DefaultImageRegistry) + if imageRegistry == "" { + return fmt.Errorf("image registry is not set: %w", errEmpty) + } + + imageNamespace := os.Getenv(generic.DefaultImageNamespace) + if imageNamespace == "" { + return fmt.Errorf("image namespace is not set: %w", errEmpty) + } + + // load chart from the path + chart, err := loader.Load(v.ChartPath) + if err != nil { + return fmt.Errorf("failed to load chart from path %s: %w", v.ChartPath, err) + } + + chart.Values["imagePullSecrets"] = []map[string]interface{}{ + { + "name": "acr-credentials", + }, + } + chart.Values["operator"].(map[string]interface{})["enabled"] = true + chart.Values["operator"].(map[string]interface{})["repository"] = imageRegistry + "/" + imageNamespace + "/retina-operator" + chart.Values["operator"].(map[string]interface{})["tag"] = tag + chart.Values["agent"].(map[string]interface{})["enabled"] = true + chart.Values["agent"].(map[string]interface{})["repository"] = imageRegistry + "/" + imageNamespace + "/retina-agent" + chart.Values["agent"].(map[string]interface{})["tag"] = tag + chart.Values["agent"].(map[string]interface{})["init"].(map[string]interface{})["enabled"] = true + chart.Values["agent"].(map[string]interface{})["init"].(map[string]interface{})["repository"] = imageRegistry + "/" + imageNamespace + "/retina-init" + chart.Values["agent"].(map[string]interface{})["init"].(map[string]interface{})["tag"] = tag + chart.Values["hubble"].(map[string]interface{})["tls"].(map[string]interface{})["enabled"] = false + chart.Values["hubble"].(map[string]interface{})["relay"].(map[string]interface{})["tls"].(map[string]interface{})["server"].(map[string]interface{})["enabled"] = false + chart.Values["hubble"].(map[string]interface{})["tls"].(map[string]interface{})["auto"].(map[string]interface{})["enabled"] = false + + getclient := action.NewGet(actionConfig) + release, err := getclient.Run(v.ReleaseName) + if err == nil && release != nil { + log.Printf("found existing release by same name, removing before installing %s", release.Name) + delclient := action.NewUninstall(actionConfig) + delclient.Wait = true + delclient.Timeout = deleteTimeout + _, err = delclient.Run(v.ReleaseName) + if err != nil { + return fmt.Errorf("failed to delete existing release %s: %w", v.ReleaseName, err) + } + } else if err != nil && !strings.Contains(err.Error(), "not found") { + return fmt.Errorf("failed to get release %s: %w", v.ReleaseName, err) + } + + client := action.NewInstall(actionConfig) + client.Namespace = v.Namespace + client.ReleaseName = v.ReleaseName + client.Timeout = createTimeout + client.Wait = true + client.WaitForJobs = true + + // install the chart here + rel, err := client.RunWithContext(ctx, chart, chart.Values) + if err != nil { + return fmt.Errorf("failed to install chart: %w", err) + } + + log.Printf("installed chart from path: %s in namespace: %s\n", rel.Name, rel.Namespace) + // this will confirm the values set during installation + log.Printf("chart values: %v\n", rel.Config) + + // ensure all pods are running, since helm doesn't care about windows + config, err := clientcmd.BuildConfigFromFlags("", v.KubeConfigFilePath) + if err != nil { + return fmt.Errorf("error building kubeconfig: %w", err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("error creating Kubernetes client: %w", err) + } + + // Validate Hubble Relay Pod + if err := WaitForPodReady(ctx, clientset, HubbleNamespace, "k8s-app="+HubbleRelayApp); err != nil { + return fmt.Errorf("error waiting for Hubble Relay pods to be ready: %w", err) + } + log.Printf("Hubble Relay Pod is ready") + + // Validate Hubble UI Pod + if err := WaitForPodReady(ctx, clientset, HubbleNamespace, "k8s-app="+HubbleUIApp); err != nil { + return fmt.Errorf("error waiting for Hubble UI pods to be ready: %w", err) + } + log.Printf("Hubble UI Pod is ready") + + return nil +} + +func (v *ValidateHubbleStep) Prevalidate() error { + return nil +} + +func (v *ValidateHubbleStep) Stop() error { + return nil +} diff --git a/test/e2e/framework/kubernetes/validate-service.go b/test/e2e/framework/kubernetes/validate-service.go new file mode 100644 index 0000000000..4ba7463bc2 --- /dev/null +++ b/test/e2e/framework/kubernetes/validate-service.go @@ -0,0 +1,83 @@ +package kubernetes + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +type ResourceTypes string + +const ( + ResourceTypePod = "pod" + ResourceTypeService = "service" +) + +type ValidateResource struct { + ResourceName string + ResourceNamespace string + ResourceType string + Labels string + KubeConfigFilePath string +} + +func (v *ValidateResource) Run() error { + config, err := clientcmd.BuildConfigFromFlags("", v.KubeConfigFilePath) + if err != nil { + return fmt.Errorf("error building kubeconfig: %w", err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("error creating Kubernetes client: %w", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeoutSeconds*time.Second) + defer cancel() + + switch v.ResourceType { + case ResourceTypePod: + err = WaitForPodReady(ctx, clientset, v.ResourceNamespace, v.Labels) + if err != nil { + return fmt.Errorf("pod not found: %w", err) + } + case ResourceTypeService: + exists, err := serviceExists(ctx, clientset, v.ResourceNamespace, v.ResourceName, v.Labels) + if err != nil || !exists { + return fmt.Errorf("service not found: %w", err) + } + + default: + return fmt.Errorf("resource type %s not supported", v.ResourceType) + } + + if err != nil { + return fmt.Errorf("error waiting for pod to be ready: %w", err) + } + return nil +} + +func serviceExists(ctx context.Context, clientset *kubernetes.Clientset, namespace, serviceName, labels string) (bool, error) { + var serviceList *corev1.ServiceList + serviceList, err := clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{LabelSelector: labels}) + if err != nil { + return false, fmt.Errorf("error listing Services: %w", err) + } + if len(serviceList.Items) == 0 { + return false, nil + } + return true, nil +} + +func (v *ValidateResource) Prevalidate() error { + return nil +} + +func (v *ValidateResource) Stop() error { + return nil +} diff --git a/test/e2e/framework/kubernetes/validateHttp.go b/test/e2e/framework/kubernetes/validateHttp.go new file mode 100644 index 0000000000..39e2e400b2 --- /dev/null +++ b/test/e2e/framework/kubernetes/validateHttp.go @@ -0,0 +1,50 @@ +package kubernetes + +import ( + "context" + "fmt" + "log" + "net/http" + "time" +) + +const ( + RequestTimeout = 30 * time.Second +) + +type ValidateHTTPResponse struct { + URL string + ExpectedStatus int +} + +func (v *ValidateHTTPResponse) Run() error { + ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, v.URL, nil) + if err != nil { + return fmt.Errorf("error creating HTTP request: %w", err) + } + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("error making HTTP request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != v.ExpectedStatus { + return fmt.Errorf("unexpected status code: got %d, want %d", resp.StatusCode, v.ExpectedStatus) + } + log.Printf("HTTP validation succeeded for URL: %s with status code %d\n", v.URL, resp.StatusCode) + + return nil +} + +func (v *ValidateHTTPResponse) Prevalidate() error { + return nil +} + +func (v *ValidateHTTPResponse) Stop() error { + return nil +} diff --git a/test/e2e/hubble/scenario.go b/test/e2e/hubble/scenario.go new file mode 100644 index 0000000000..92de650b32 --- /dev/null +++ b/test/e2e/hubble/scenario.go @@ -0,0 +1,56 @@ +package hubble + +import ( + "net/http" + + k8s "github.com/microsoft/retina/test/e2e/framework/kubernetes" + "github.com/microsoft/retina/test/e2e/framework/types" +) + +func ValidateHubbleRelayService() *types.Scenario { + name := "Validate Hubble Relay Service" + steps := []*types.StepWrapper{ + { + Step: &k8s.ValidateResource{ + ResourceName: "hubble-relay-service", + ResourceNamespace: k8s.HubbleNamespace, + ResourceType: k8s.ResourceTypeService, + Labels: "k8s-app=" + k8s.HubbleRelayApp, + }, + }, + } + + return types.NewScenario(name, steps...) +} + +func ValidateHubbleUIService(kubeConfigFilePath string) *types.Scenario { + name := "Validate Hubble UI Services" + steps := []*types.StepWrapper{ + { + Step: &k8s.ValidateResource{ + ResourceName: k8s.HubbleUIApp, + ResourceNamespace: k8s.HubbleNamespace, + ResourceType: k8s.ResourceTypeService, + Labels: "k8s-app=" + k8s.HubbleUIApp, + }, + }, + { + Step: &k8s.PortForward{ + LabelSelector: "k8s-app=hubble-ui", + LocalPort: "8080", + RemotePort: "8081", + OptionalLabelAffinity: "k8s-app=hubble-ui", + Endpoint: "?namespace=default", // set as default namespace query since endpoint can't be nil + KubeConfigFilePath: kubeConfigFilePath, + }, + }, + { + Step: &k8s.ValidateHTTPResponse{ + URL: "http://localhost:8080", + ExpectedStatus: http.StatusOK, + }, + }, + } + + return types.NewScenario(name, steps...) +} diff --git a/test/e2e/jobs/jobs.go b/test/e2e/jobs/jobs.go index bd921df2a3..925df6dc01 100644 --- a/test/e2e/jobs/jobs.go +++ b/test/e2e/jobs/jobs.go @@ -9,6 +9,7 @@ import ( "github.com/microsoft/retina/test/e2e/framework/generic" "github.com/microsoft/retina/test/e2e/framework/kubernetes" "github.com/microsoft/retina/test/e2e/framework/types" + "github.com/microsoft/retina/test/e2e/hubble" "github.com/microsoft/retina/test/e2e/scenarios/dns" "github.com/microsoft/retina/test/e2e/scenarios/drop" "github.com/microsoft/retina/test/e2e/scenarios/latency" @@ -232,6 +233,79 @@ func UpgradeAndTestRetinaAdvancedMetrics(kubeConfigFilePath, chartPath, valuesFi return job } +func ValidateHubble(kubeConfigFilePath, chartPath string, testPodNamespace string) *types.Job { + job := types.NewJob("Validate Hubble") + + job.AddStep(&kubernetes.ValidateHubbleStep{ + Namespace: common.KubeSystemNamespace, + ReleaseName: "retina", + KubeConfigFilePath: kubeConfigFilePath, + ChartPath: chartPath, + TagEnv: generic.DefaultTagEnv, + }, nil) + + job.AddScenario(hubble.ValidateHubbleRelayService()) + + job.AddScenario(hubble.ValidateHubbleUIService(kubeConfigFilePath)) + + job.AddScenario(drop.ValidateDropMetric(testPodNamespace)) + + job.AddScenario(tcp.ValidateTCPMetrics(testPodNamespace)) + + dnsScenarios := []struct { + name string + req *dns.RequestValidationParams + resp *dns.ResponseValidationParams + }{ + { + name: "Validate basic DNS request and response metrics for a valid domain", + req: &dns.RequestValidationParams{ + NumResponse: "0", + Query: "kubernetes.default.svc.cluster.local.", + QueryType: "A", + Command: "nslookup kubernetes.default", + ExpectError: false, + }, + resp: &dns.ResponseValidationParams{ + NumResponse: "1", + Query: "kubernetes.default.svc.cluster.local.", + QueryType: "A", + ReturnCode: "No Error", + Response: "10.0.0.1", + }, + }, + { + name: "Validate basic DNS request and response metrics for a non-existent domain", + req: &dns.RequestValidationParams{ + NumResponse: "0", + Query: "some.non.existent.domain.", + QueryType: "A", + Command: "nslookup some.non.existent.domain", + ExpectError: true, + }, + resp: &dns.ResponseValidationParams{ + NumResponse: "0", + Query: "some.non.existent.domain.", + QueryType: "A", + Response: dns.EmptyResponse, // hacky way to bypass the framework for now + ReturnCode: "Non-Existent Domain", + }, + }, + } + + for _, scenario := range dnsScenarios { + job.AddScenario(dns.ValidateBasicDNSMetrics(scenario.name, scenario.req, scenario.resp, testPodNamespace)) + } + + job.AddStep(&kubernetes.EnsureStableComponent{ + PodNamespace: common.KubeSystemNamespace, + LabelSelector: "k8s-app=retina", + IgnoreContainerRestart: false, + }, nil) + + return job +} + func RunPerfTest(kubeConfigFilePath string, chartPath string) *types.Job { job := types.NewJob("Run performance tests") @@ -279,3 +353,4 @@ func RunPerfTest(kubeConfigFilePath string, chartPath string) *types.Job { return job } + diff --git a/test/e2e/retina_e2e_test.go b/test/e2e/retina_e2e_test.go index 7b13d1782b..d1b1b16e95 100644 --- a/test/e2e/retina_e2e_test.go +++ b/test/e2e/retina_e2e_test.go @@ -57,6 +57,7 @@ func TestE2ERetina(t *testing.T) { rootDir := filepath.Dir(filepath.Dir(cwd)) chartPath := filepath.Join(rootDir, "deploy", "legacy", "manifests", "controller", "helm", "retina") + hubblechartPath := filepath.Join(rootDir, "deploy", "hubble", "manifests", "controller", "helm", "retina") profilePath := filepath.Join(rootDir, "test", "profiles", "advanced", "values.yaml") kubeConfigFilePath := filepath.Join(rootDir, "test", "e2e", "test.pem") @@ -77,4 +78,8 @@ func TestE2ERetina(t *testing.T) { // Upgrade and test Retina with advanced metrics advanceMetricsE2E := types.NewRunner(t, jobs.UpgradeAndTestRetinaAdvancedMetrics(kubeConfigFilePath, chartPath, profilePath, common.TestPodNamespace)) advanceMetricsE2E.Run(ctx) + + // Install and test Hubble basic metrics + validatehubble := types.NewRunner(t, jobs.ValidateHubble(kubeConfigFilePath, hubblechartPath, common.TestPodNamespace)) + validatehubble.Run(ctx) }