From a08ff7b8294c7406384694bbf9ceb0bb0dbd2d56 Mon Sep 17 00:00:00 2001 From: Vincent Shen Date: Thu, 16 May 2024 08:51:03 -0700 Subject: [PATCH] CMP-2526: Disable automatic remediation for ROSA HCP environments Since the Compliance Operator is only going to support running on HCP environments, where end users don't have access to the master nodes (or control plane), it doesn't make sense to give them a scan setting with auto remediation functionality applied. For this case, the Compliance Operator should detect if it is running on ROSA HCP and adjust, or just not create the `default-auto-apply` scan settings. --- cmd/manager/operator.go | 59 ++++++++++++++++------------ tests/e2e/framework/common.go | 73 +++++++++++++++++++++++++++++++++++ tests/e2e/rosa/main_test.go | 25 ++++++++++++ 3 files changed, 133 insertions(+), 24 deletions(-) diff --git a/cmd/manager/operator.go b/cmd/manager/operator.go index cbbdf9c86..a3e874450 100644 --- a/cmd/manager/operator.go +++ b/cmd/manager/operator.go @@ -141,6 +141,15 @@ var ( }, } + defaultAutoRemediationPerPlatform = map[PlatformType]bool{ + PlatformOpenShift: true, + PlatformOpenShiftOnPower: true, + PlatformOpenShiftOnZ: true, + PlatformEKS: false, + PlatformGeneric: false, + PlatformHyperShift: true, + PlatformROSA: false, + } serviceMonitorBearerTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" serviceMonitorTLSCAFile = "/etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt" alertName = "compliance" @@ -599,6 +608,7 @@ func ensureDefaultScanSettings( var lastErr error for _, ns := range namespaceList { roles := getDefaultRoles(platform) + autoRemediationEnabled := defaultAutoRemediationPerPlatform[platform] d := &compv1alpha1.ScanSetting{ ObjectMeta: metav1.ObjectMeta{ Name: defaultScanSettingsName, @@ -622,31 +632,32 @@ func ensureDefaultScanSettings( if !k8serrors.IsAlreadyExists(derr) { lastErr = derr } - - a := &compv1alpha1.ScanSetting{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultAutoApplyScanSettingsName, - Namespace: ns, - }, - ComplianceScanSettings: compv1alpha1.ComplianceScanSettings{ - RawResultStorage: compv1alpha1.RawResultStorageSettings{ - NodeSelector: si.Selector, - Tolerations: si.Tolerations, + if autoRemediationEnabled { + a := &compv1alpha1.ScanSetting{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultAutoApplyScanSettingsName, + Namespace: ns, }, - }, - ComplianceSuiteSettings: compv1alpha1.ComplianceSuiteSettings{ - AutoApplyRemediations: true, - AutoUpdateRemediations: true, - Schedule: defaultScanSettingsSchedule, - }, - Roles: roles, - } - setupLog.Info("Ensuring ScanSetting is available", - "ScanSetting.Name", d.GetName(), - "ScanSetting.Namespace", d.GetNamespace()) - aerr := crclient.Create(ctx, a) - if !k8serrors.IsAlreadyExists(aerr) { - lastErr = aerr + ComplianceScanSettings: compv1alpha1.ComplianceScanSettings{ + RawResultStorage: compv1alpha1.RawResultStorageSettings{ + NodeSelector: si.Selector, + Tolerations: si.Tolerations, + }, + }, + ComplianceSuiteSettings: compv1alpha1.ComplianceSuiteSettings{ + AutoApplyRemediations: true, + AutoUpdateRemediations: true, + Schedule: defaultScanSettingsSchedule, + }, + Roles: roles, + } + setupLog.Info("Ensuring ScanSetting is available", + "ScanSetting.Name", d.GetName(), + "ScanSetting.Namespace", d.GetNamespace()) + aerr := crclient.Create(ctx, a) + if !k8serrors.IsAlreadyExists(aerr) { + lastErr = aerr + } } } return lastErr diff --git a/tests/e2e/framework/common.go b/tests/e2e/framework/common.go index 05a6b8e9b..0d1c99011 100644 --- a/tests/e2e/framework/common.go +++ b/tests/e2e/framework/common.go @@ -11,6 +11,7 @@ import ( "regexp" "strconv" "strings" + "testing" "time" "github.com/cenkalti/backoff/v4" @@ -21,17 +22,21 @@ import ( mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" batchv1 "k8s.io/api/batch/v1" core "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/rbac/v1" schedulingv1 "k8s.io/api/scheduling/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + clusterv1alpha1 "open-cluster-management.io/api/cluster/v1alpha1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" psapi "k8s.io/pod-security-admission/api" + "sigs.k8s.io/controller-runtime/pkg/client" dynclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" @@ -117,6 +122,49 @@ func (f *Framework) cleanUpFromYAMLFile(p *string) error { return nil } +func (f *Framework) PrintROSADebugInfo(t *testing.T) { + // List cluster claims + clusterClaimList := clusterv1alpha1.ClusterClaimList{} + err := f.Client.List(context.TODO(), &clusterClaimList) + if err != nil { + t.Fatalf("Failed to list cluster claims: %v", err) + } + for _, clusterClaim := range clusterClaimList.Items { + t.Logf("ClusterClaim: %s", clusterClaim.Name) + t.Logf("ClusterClaim.Spec.Value: %v", clusterClaim.Spec.Value) + + } + + // print out logs for compliance-operator deployment + podList := corev1.PodList{} + err = f.Client.List(context.TODO(), &podList, client.InNamespace(f.OperatorNamespace)) + if err != nil { + t.Fatalf("Failed to list pods: %v", err) + } + for _, pod := range podList.Items { + // find pod named contains compliance-operator substring + if strings.Contains(pod.Name, "compliance-operator") { + log.Printf("Pod: %s", pod.Name) + log.Printf("Pod.Status: %v", pod.Status) + // print out logs for compliance-operator pod + req := f.KubeClient.CoreV1().Pods(f.OperatorNamespace).GetLogs(pod.Name, &corev1.PodLogOptions{}) + log.Printf("Request: %v", req) + reader, err := req.Stream(context.Background()) + if err != nil { + t.Fatalf("Failed to get logs: %v", err) + } + buf := make([]byte, 1024) + for { + n, err := reader.Read(buf) + if err != nil { + break + } + log.Printf("Logs: %s", string(buf[:n])) + } + } + } +} + func (f *Framework) cleanUpProfileBundle(p string) error { pb := &compv1alpha1.ProfileBundle{ ObjectMeta: metav1.ObjectMeta{ @@ -222,6 +270,19 @@ func (f *Framework) addFrameworks() error { } } + // ClusterClaim objects + if f.Platform == "rosa" { + ccObjs := [1]dynclient.ObjectList{ + &clusterv1alpha1.ClusterClaimList{}, + } + for _, obj := range ccObjs { + err := AddToFrameworkScheme(clusterv1alpha1.Install, obj) + if err != nil { + return fmt.Errorf("failed to add custom resource scheme to framework: %v", err) + } + } + } + // OpenShift objects ocpObjs := [2]dynclient.ObjectList{ &imagev1.ImageStreamList{}, @@ -420,6 +481,10 @@ func (f *Framework) GetReadyProfileBundle(name, namespace string) (*compv1alpha1 } func (f *Framework) updateScanSettingsForDebug() error { + if f.Platform == "rosa" { + fmt.Printf("bypassing ScanSettings test setup because it's not supported on %s\n", f.Platform) + return nil + } for _, ssName := range []string{"default", "default-auto-apply"} { ss := &compv1alpha1.ScanSetting{} sskey := types.NamespacedName{Name: ssName, Namespace: f.OperatorNamespace} @@ -438,6 +503,10 @@ func (f *Framework) updateScanSettingsForDebug() error { } func (f *Framework) ensureE2EScanSettings() error { + if f.Platform == "rosa" { + fmt.Printf("bypassing ScanSettings test setup because it's not supported on %s\n", f.Platform) + return nil + } for _, ssName := range []string{"default", "default-auto-apply"} { ss := &compv1alpha1.ScanSetting{} sskey := types.NamespacedName{Name: ssName, Namespace: f.OperatorNamespace} @@ -464,6 +533,10 @@ func (f *Framework) ensureE2EScanSettings() error { } func (f *Framework) deleteScanSettings(name string) error { + if f.Platform == "rosa" { + fmt.Printf("bypassing ScanSettings test setup because it's not supported on %s\n", f.Platform) + return nil + } ss := &compv1alpha1.ScanSetting{} sskey := types.NamespacedName{Name: name, Namespace: f.OperatorNamespace} if err := f.Client.Get(context.TODO(), sskey, ss); err != nil { diff --git a/tests/e2e/rosa/main_test.go b/tests/e2e/rosa/main_test.go index 9ee5b0639..80e9e07ae 100644 --- a/tests/e2e/rosa/main_test.go +++ b/tests/e2e/rosa/main_test.go @@ -50,3 +50,28 @@ func TestInstallOnlyParsesNodeProfiles(t *testing.T) { } } + +func TestScanSetting(t *testing.T) { + f := framework.Global + // prinout all scan settings + scanSettingList := compv1alpha1.ScanSettingList{} + err := f.Client.List(context.TODO(), &scanSettingList) + if err != nil { + t.Fatalf("Failed to list scan settings: %v", err) + } + for _, scanSetting := range scanSettingList.Items { + if scanSetting.Name == "default-auto-apply" { + f.PrintROSADebugInfo(t) + t.Fatalf("ScanSetting: %s is not expected", scanSetting.Name) + } + t.Logf("ScanSetting: %s", scanSetting.Name) + for _, role := range scanSetting.Roles { + if role == "master" { + f.PrintROSADebugInfo(t) + t.Fatalf("Role: %s is not expected", role) + } + t.Logf("Role: %s", role) + + } + } +}