From adfb9ba1194e918c453486151b716e701108063c Mon Sep 17 00:00:00 2001 From: Justin Kulikauskas Date: Tue, 2 Apr 2024 11:49:34 -0400 Subject: [PATCH] Implement mustnothave mode for OperatorPolicy Enforced mustnohave policies will delete things based on the new RemovalBehavior field. In inform mode, resources that would be removed by an enforced policy will be causes for NonCompliance. This gives control to users in order to prevent unintended side effects. Refs: - https://issues.redhat.com/browse/ACM-9287 Signed-off-by: Justin Kulikauskas --- api/v1beta1/operatorpolicy_types.go | 87 +- api/v1beta1/zz_generated.deepcopy.go | 1 + controllers/operatorpolicy_controller.go | 394 ++++++++- controllers/operatorpolicy_status.go | 121 ++- .../allowed-compliance-types.json | 2 +- ...luster-management.io_operatorpolicies.yaml | 52 ++ ...luster-management.io_operatorpolicies.yaml | 53 ++ test/e2e/case38_install_operator_test.go | 834 +++++++++++++++++- .../operator-policy-mustnothave.yaml | 32 + 9 files changed, 1525 insertions(+), 51 deletions(-) create mode 100644 test/resources/case38_operator_install/operator-policy-mustnothave.yaml diff --git a/api/v1beta1/operatorpolicy_types.go b/api/v1beta1/operatorpolicy_types.go index a77ef632..f9d7c127 100644 --- a/api/v1beta1/operatorpolicy_types.go +++ b/api/v1beta1/operatorpolicy_types.go @@ -4,6 +4,8 @@ package v1beta1 import ( + "strings" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -15,7 +17,6 @@ import ( type StatusConfigAction string // RemovalAction : Keep, Delete, or DeleteIfUnused -// +kubebuilder:validation:Enum=Keep;Delete;DeleteIfUnused type RemovalAction string const ( @@ -36,20 +37,78 @@ const ( DeleteIfUnused RemovalAction = "DeleteIfUnused" ) -// RemovalBehavior defines resource behavior when policy is removed +func (ra RemovalAction) IsKeep() bool { + return strings.EqualFold(string(ra), string(Keep)) +} + +func (ra RemovalAction) IsDelete() bool { + return strings.EqualFold(string(ra), string(Delete)) +} + +func (ra RemovalAction) IsDeleteIfUnused() bool { + return strings.EqualFold(string(ra), string(DeleteIfUnused)) +} + type RemovalBehavior struct { - // Kind OperatorGroup + //+kubebuilder:default=DeleteIfUnused + //+kubebuilder:validation:Enum=Keep;Delete;DeleteIfUnused + // Specifies whether to delete the OperatorGroup; defaults to 'DeleteIfUnused' which + // will only delete the OperatorGroup if there is not another Subscription using it. OperatorGroups RemovalAction `json:"operatorGroups,omitempty"` - // Kind Subscription + + //+kubebuilder:default=Delete + //+kubebuilder:validation:Enum=Keep;Delete + // Specifies whether to delete the Subscription; defaults to 'Delete' Subscriptions RemovalAction `json:"subscriptions,omitempty"` - // Kind ClusterServiceVersion + + //+kubebuilder:default=Delete + //+kubebuilder:validation:Enum=Keep;Delete + // Specifies whether to delete the ClusterServiceVersion; defaults to 'Delete' CSVs RemovalAction `json:"clusterServiceVersions,omitempty"` - // Kind InstallPlan + + //+kubebuilder:default=Keep + //+kubebuilder:validation:Enum=Keep;Delete + // Specifies whether to delete any InstallPlans associated with the operator; defaults + // to 'Keep' because those objects are only for history InstallPlan RemovalAction `json:"installPlans,omitempty"` - // Kind CustomResourceDefinitions + + //+kubebuilder:default=Keep + //+kubebuilder:validation:Enum=Keep;Delete + // Specifies whether to delete any CustomResourceDefinitions associated with the operator; + // defaults to 'Keep' because deleting them should be done deliberately CRDs RemovalAction `json:"customResourceDefinitions,omitempty"` - // Kind APIServiceDefinitions - APIServiceDefinitions RemovalAction `json:"apiServiceDefinitions,omitempty"` + + // Future? + // APIServiceDefinitions RemovalAction `json:"apiServiceDefinitions,omitempty"` +} + +// ApplyDefaults ensures that unset fields in a RemovalBehavior behave as if they were +// set to the default values. In a cluster, kubernetes API validation should ensure that +// there are no unset values, and should apply the default values itself. +func (rb RemovalBehavior) ApplyDefaults() RemovalBehavior { + withDefaults := *rb.DeepCopy() + + if withDefaults.OperatorGroups == "" { + withDefaults.OperatorGroups = DeleteIfUnused + } + + if withDefaults.Subscriptions == "" { + withDefaults.Subscriptions = Delete + } + + if withDefaults.CSVs == "" { + withDefaults.CSVs = Delete + } + + if withDefaults.InstallPlan == "" { + withDefaults.InstallPlan = Keep + } + + if withDefaults.CRDs == "" { + withDefaults.CRDs = Keep + } + + return withDefaults } // StatusConfig defines how resource statuses affect the OperatorPolicy status and compliance @@ -64,7 +123,7 @@ type StatusConfig struct { type OperatorPolicySpec struct { Severity policyv1.Severity `json:"severity,omitempty"` // low, medium, high RemediationAction policyv1.RemediationAction `json:"remediationAction,omitempty"` // inform, enforce - ComplianceType policyv1.ComplianceType `json:"complianceType"` // musthave + ComplianceType policyv1.ComplianceType `json:"complianceType"` // musthave, mustnothave // Include the name, namespace, and any `spec` fields for the OperatorGroup. // For more info, see `kubectl explain operatorgroup.spec` or @@ -84,10 +143,14 @@ type OperatorPolicySpec struct { // in 'inform' mode, and which installPlans are approved when in 'enforce' mode Versions []policyv1.NonEmptyString `json:"versions,omitempty"` + //+kubebuilder:default={} + // RemovalBehavior defines what resources will be removed by enforced mustnothave policies. + // When in inform mode, any resources that would be deleted if the policy was enforced will + // be causes for NonCompliance, but resources that would be kept will be considered Compliant. + RemovalBehavior RemovalBehavior `json:"removalBehavior,omitempty"` + // FUTURE //nolint:dupword - // RemovalBehavior RemovalBehavior `json:"removalBehavior,omitempty"` - //nolint:dupword // StatusConfig StatusConfig `json:"statusConfig,omitempty"` } diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 0f8b5366..7a901a9a 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -86,6 +86,7 @@ func (in *OperatorPolicySpec) DeepCopyInto(out *OperatorPolicySpec) { *out = make([]v1.NonEmptyString, len(*in)) copy(*out, *in) } + out.RemovalBehavior = in.RemovalBehavior } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OperatorPolicySpec. diff --git a/controllers/operatorpolicy_controller.go b/controllers/operatorpolicy_controller.go index ca5657da..19a2cd69 100644 --- a/controllers/operatorpolicy_controller.go +++ b/controllers/operatorpolicy_controller.go @@ -238,7 +238,8 @@ func (r *OperatorPolicyReconciler) handleResources(ctx context.Context, policy * return earlyComplianceEvents, condChanged, err } - changed, err = r.handleInstallPlan(ctx, policy, subscription) + earlyConds, changed, err = r.handleInstallPlan(ctx, policy, subscription) + earlyComplianceEvents = append(earlyComplianceEvents, earlyConds...) condChanged = condChanged || changed if err != nil { @@ -247,11 +248,12 @@ func (r *OperatorPolicyReconciler) handleResources(ctx context.Context, policy * return earlyComplianceEvents, condChanged, err } - csv, changed, err := r.handleCSV(policy, subscription) + csv, earlyConds, changed, err := r.handleCSV(ctx, policy, subscription) + earlyComplianceEvents = append(earlyComplianceEvents, earlyConds...) condChanged = condChanged || changed if err != nil { - OpLog.Error(err, "Error handling CSVs") + OpLog.Error(err, "Error handling ClusterServiceVersions") return earlyComplianceEvents, condChanged, err } @@ -287,12 +289,14 @@ func (r *OperatorPolicyReconciler) handleResources(ctx context.Context, policy * return earlyComplianceEvents, condChanged, nil } -// buildResources builds desired states for the Subscription and OperatorGroup, and +// buildResources builds 'musthave' desired states for the Subscription and OperatorGroup, and // checks if the policy's spec is valid. It returns: // - the built Subscription // - the built OperatorGroup // - whether the status has changed because of the validity condition // - an error if an API call failed +// +// The built objects can be used to find relevant objects for a 'mustnothave' policy. func (r *OperatorPolicyReconciler) buildResources(policy *policyv1beta1.OperatorPolicy) ( *operatorv1alpha1.Subscription, *operatorv1.OperatorGroup, bool, error, ) { @@ -474,6 +478,19 @@ func (r *OperatorPolicyReconciler) handleOpGroup( return nil, false, fmt.Errorf("error listing OperatorGroups: %w", err) } + if policy.Spec.ComplianceType.IsMustHave() { + return r.musthaveOpGroup(ctx, policy, desiredOpGroup, foundOpGroups) + } + + return r.mustnothaveOpGroup(ctx, policy, desiredOpGroup, foundOpGroups) +} + +func (r *OperatorPolicyReconciler) musthaveOpGroup( + ctx context.Context, + policy *policyv1beta1.OperatorPolicy, + desiredOpGroup *operatorv1.OperatorGroup, + foundOpGroups []unstructured.Unstructured, +) ([]metav1.Condition, bool, error) { switch len(foundOpGroups) { case 0: // Missing OperatorGroup: report NonCompliance @@ -489,7 +506,7 @@ func (r *OperatorPolicyReconciler) handleOpGroup( earlyConds = append(earlyConds, calculateComplianceCondition(policy)) } - err = r.Create(ctx, desiredOpGroup) + err := r.Create(ctx, desiredOpGroup) if err != nil { return nil, changed, fmt.Errorf("error creating the OperatorGroup: %w", err) } @@ -591,6 +608,87 @@ func (r *OperatorPolicyReconciler) handleOpGroup( } } +func (r *OperatorPolicyReconciler) mustnothaveOpGroup( + ctx context.Context, + policy *policyv1beta1.OperatorPolicy, + desiredOpGroup *operatorv1.OperatorGroup, + foundOpGroups []unstructured.Unstructured, +) ([]metav1.Condition, bool, error) { + if len(foundOpGroups) == 0 { + // Missing OperatorGroup: report Compliance + changed := updateStatus(policy, missingNotWantedCond("OperatorGroup"), missingNotWantedObj(desiredOpGroup)) + + return nil, changed, nil + } + + foundOpGroupName := "" + + for _, opGroup := range foundOpGroups { + emptyNameMatch := desiredOpGroup.Name == "" && opGroup.GetGenerateName() == desiredOpGroup.GenerateName + + if opGroup.GetName() == desiredOpGroup.Name || emptyNameMatch { + foundOpGroupName = opGroup.GetName() + } + } + + if foundOpGroupName == "" { + // no found OperatorGroup matches what the policy is looking for, report Compliance. + changed := updateStatus(policy, missingNotWantedCond("OperatorGroup"), missingNotWantedObj(desiredOpGroup)) + + return nil, changed, nil + } + + desiredOpGroup.SetName(foundOpGroupName) + + removalBehavior := policy.Spec.RemovalBehavior.ApplyDefaults() + + if removalBehavior.OperatorGroups.IsKeep() { + changed := updateStatus(policy, keptCond("OperatorGroup"), leftoverObj(desiredOpGroup)) + + return nil, changed, nil + } + + // The found OperatorGroup matches what is *not* wanted by the policy. Report NonCompliance. + changed := updateStatus(policy, foundNotWantedCond("OperatorGroup"), foundNotWantedObj(desiredOpGroup)) + + if policy.Spec.RemediationAction.IsInform() { + return nil, changed, nil + } + + if removalBehavior.OperatorGroups.IsDeleteIfUnused() { + // Check the namespace for any subscriptions, including the sub for this mustnothave policy, + // since deleting the OperatorGroup before that could cause problems + watcher := opPolIdentifier(policy.Namespace, policy.Name) + + foundSubscriptions, err := r.DynamicWatcher.List( + watcher, subscriptionGVK, desiredOpGroup.Namespace, labels.Everything()) + if err != nil { + return nil, false, fmt.Errorf("error listing Subscriptions: %w", err) + } + + if len(foundSubscriptions) != 0 { + return nil, changed, nil + } + } + + earlyConds := []metav1.Condition{} + + if changed { + earlyConds = append(earlyConds, calculateComplianceCondition(policy)) + } + + err := r.Delete(ctx, desiredOpGroup) + if err != nil { + return earlyConds, changed, fmt.Errorf("error deleting the OperatorGroup: %w", err) + } + + desiredOpGroup.SetGroupVersionKind(operatorGroupGVK) // Delete stripped this information + + updateStatus(policy, deletedCond("OperatorGroup"), deletedObj(desiredOpGroup)) + + return earlyConds, true, nil +} + func (r *OperatorPolicyReconciler) handleSubscription( ctx context.Context, policy *policyv1beta1.OperatorPolicy, desiredSub *operatorv1alpha1.Subscription, ) (*operatorv1alpha1.Subscription, []metav1.Condition, bool, error) { @@ -606,6 +704,19 @@ func (r *OperatorPolicyReconciler) handleSubscription( return nil, nil, false, fmt.Errorf("error getting the Subscription: %w", err) } + if policy.Spec.ComplianceType.IsMustHave() { + return r.musthaveSubscription(ctx, policy, desiredSub, foundSub) + } + + return r.mustnothaveSubscription(ctx, policy, desiredSub, foundSub) +} + +func (r *OperatorPolicyReconciler) musthaveSubscription( + ctx context.Context, + policy *policyv1beta1.OperatorPolicy, + desiredSub *operatorv1alpha1.Subscription, + foundSub *unstructured.Unstructured, +) (*operatorv1alpha1.Subscription, []metav1.Condition, bool, error) { if foundSub == nil { // Missing Subscription: report NonCompliance changed := updateStatus(policy, missingWantedCond("Subscription"), missingWantedObj(desiredSub)) @@ -717,6 +828,53 @@ func (r *OperatorPolicyReconciler) handleSubscription( return mergedSub, earlyConds, true, nil } +func (r *OperatorPolicyReconciler) mustnothaveSubscription( + ctx context.Context, + policy *policyv1beta1.OperatorPolicy, + desiredSub *operatorv1alpha1.Subscription, + foundUnstructSub *unstructured.Unstructured, +) (*operatorv1alpha1.Subscription, []metav1.Condition, bool, error) { + if foundUnstructSub == nil { + // Missing Subscription: report Compliance + changed := updateStatus(policy, missingNotWantedCond("Subscription"), missingNotWantedObj(desiredSub)) + + return desiredSub, nil, changed, nil + } + + foundSub := new(operatorv1alpha1.Subscription) + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(foundUnstructSub.Object, foundSub); err != nil { + return nil, nil, false, fmt.Errorf("error converting the retrieved Subscription to the go type: %w", err) + } + + if policy.Spec.RemovalBehavior.ApplyDefaults().Subscriptions.IsKeep() { + changed := updateStatus(policy, keptCond("Subscription"), leftoverObj(foundSub)) + + return foundSub, nil, changed, nil + } + + // Subscription found, not wanted: report NonCompliance. + changed := updateStatus(policy, foundNotWantedCond("Subscription"), foundNotWantedObj(foundSub)) + + if policy.Spec.RemediationAction.IsInform() { + return foundSub, nil, changed, nil + } + + earlyConds := []metav1.Condition{} + + if changed { + earlyConds = append(earlyConds, calculateComplianceCondition(policy)) + } + + err := r.Delete(ctx, foundUnstructSub) + if err != nil { + return foundSub, earlyConds, changed, fmt.Errorf("error deleting the Subscription: %w", err) + } + + updateStatus(policy, deletedCond("Subscription"), deletedObj(desiredSub)) + + return foundSub, earlyConds, true, nil +} + // messageIncludesSubscription checks if the ConstraintsNotSatisfiable message includes the input // subscription or package. Some examples that it catches: // https://github.com/operator-framework/operator-lifecycle-manager/blob/dc0c564f62d526bae0467d53f439e1c91a17ed8a/pkg/controller/registry/resolver/resolver.go#L257-L267 @@ -775,42 +933,42 @@ func constraintMessageMatch(policy *policyv1beta1.OperatorPolicy, cond *operator func (r *OperatorPolicyReconciler) handleInstallPlan( ctx context.Context, policy *policyv1beta1.OperatorPolicy, sub *operatorv1alpha1.Subscription, -) (bool, error) { +) ([]metav1.Condition, bool, error) { if sub == nil { // Note: existing related objects will not be removed by this status update - return updateStatus(policy, invalidCausingUnknownCond("InstallPlan")), nil + return nil, updateStatus(policy, invalidCausingUnknownCond("InstallPlan")), nil } watcher := opPolIdentifier(policy.Namespace, policy.Name) + selector := subLabelSelector(sub) - foundInstallPlans, err := r.DynamicWatcher.List( - watcher, installPlanGVK, sub.Namespace, labels.Everything()) + ipList, err := r.DynamicWatcher.List(watcher, installPlanGVK, sub.Namespace, selector) if err != nil { - return false, fmt.Errorf("error listing InstallPlans: %w", err) + return nil, false, fmt.Errorf("error listing InstallPlans: %w", err) } - ownedInstallPlans := make([]unstructured.Unstructured, 0, len(foundInstallPlans)) + // InstallPlans are generally kept in order to provide a history of actions on the cluster, but + // they can be deleted without impacting the installed operator. So, not finding any should not + // be considered a reason for NonCompliance, regardless of musthave or mustnothave. + if len(ipList) == 0 { + return nil, updateStatus(policy, noInstallPlansCond, noInstallPlansObj(sub.Namespace)), nil + } - for _, installPlan := range foundInstallPlans { - for _, owner := range installPlan.GetOwnerReferences() { - match := owner.Name == sub.Name && - owner.Kind == subscriptionGVK.Kind && - owner.APIVersion == subscriptionGVK.GroupVersion().String() - if match { - ownedInstallPlans = append(ownedInstallPlans, installPlan) + if policy.Spec.ComplianceType.IsMustHave() { + changed, err := r.musthaveInstallPlan(ctx, policy, sub, ipList) - break - } - } + return nil, changed, err } - // InstallPlans are generally kept in order to provide a history of actions on the cluster, but - // they can be deleted without impacting the installed operator. So, not finding any should not - // be considered a reason for NonCompliance. - if len(ownedInstallPlans) == 0 { - return updateStatus(policy, noInstallPlansCond, noInstallPlansObj(sub.Namespace)), nil - } + return r.mustnothaveInstallPlan(ctx, policy, ipList) +} +func (r *OperatorPolicyReconciler) musthaveInstallPlan( + ctx context.Context, + policy *policyv1beta1.OperatorPolicy, + sub *operatorv1alpha1.Subscription, + ownedInstallPlans []unstructured.Unstructured, +) (bool, error) { OpLog := ctrl.LoggerFrom(ctx) relatedInstallPlans := make([]policyv1.RelatedObject, len(ownedInstallPlans)) ipsRequiringApproval := make([]unstructured.Unstructured, 0) @@ -944,14 +1102,63 @@ func (r *OperatorPolicyReconciler) handleInstallPlan( return updateStatus(policy, installPlanApprovedCond(approvedVersion), relatedInstallPlans...), nil } +func (r *OperatorPolicyReconciler) mustnothaveInstallPlan( + ctx context.Context, + policy *policyv1beta1.OperatorPolicy, + ownedInstallPlans []unstructured.Unstructured, +) ([]metav1.Condition, bool, error) { + relatedInstallPlans := make([]policyv1.RelatedObject, len(ownedInstallPlans)) + + if policy.Spec.RemovalBehavior.ApplyDefaults().InstallPlan.IsKeep() { + for i := range ownedInstallPlans { + relatedInstallPlans[i] = leftoverObj(&ownedInstallPlans[i]) + } + + return nil, updateStatus(policy, keptCond("InstallPlan"), relatedInstallPlans...), nil + } + + for i := range ownedInstallPlans { + relatedInstallPlans[i] = foundNotWantedObj(&ownedInstallPlans[i]) + } + + changed := updateStatus(policy, foundNotWantedCond("InstallPlan"), relatedInstallPlans...) + + if policy.Spec.RemediationAction.IsInform() { + return nil, changed, nil + } + + earlyConds := []metav1.Condition{} + + if changed { + earlyConds = append(earlyConds, calculateComplianceCondition(policy)) + } + + for i := range ownedInstallPlans { + err := r.Delete(ctx, &ownedInstallPlans[i]) + if err != nil { + changed := updateStatus(policy, foundNotWantedCond("InstallPlan"), relatedInstallPlans...) + + return earlyConds, changed, fmt.Errorf("error deleting the InstallPlan: %w", err) + } + + ownedInstallPlans[i].SetGroupVersionKind(installPlanGVK) // Delete stripped this information + relatedInstallPlans[i] = deletedObj(&ownedInstallPlans[i]) + } + + updateStatus(policy, deletedCond("InstallPlan"), relatedInstallPlans...) + + return earlyConds, true, nil +} + func (r *OperatorPolicyReconciler) handleCSV( + ctx context.Context, policy *policyv1beta1.OperatorPolicy, sub *operatorv1alpha1.Subscription, -) (*operatorv1alpha1.ClusterServiceVersion, bool, error) { +) (*operatorv1alpha1.ClusterServiceVersion, []metav1.Condition, bool, error) { // case where subscription is nil if sub == nil { // need to report lack of existing CSV - return nil, updateStatus(policy, noCSVCond, noExistingCSVObj), nil + return nil, nil, updateStatus(policy, noCSVCond, noExistingCSVObj), nil } watcher := opPolIdentifier(policy.Namespace, policy.Name) @@ -959,7 +1166,7 @@ func (r *OperatorPolicyReconciler) handleCSV( csvList, err := r.DynamicWatcher.List(watcher, clusterServiceVersionGVK, sub.Namespace, selector) if err != nil { - return nil, false, fmt.Errorf("error listing CSVs: %w", err) + return nil, nil, false, fmt.Errorf("error listing CSVs: %w", err) } var foundCSV *operatorv1alpha1.ClusterServiceVersion @@ -970,23 +1177,84 @@ func (r *OperatorPolicyReconciler) handleCSV( err = runtime.DefaultUnstructuredConverter.FromUnstructured(csv.UnstructuredContent(), &matchedCSV) if err != nil { - return nil, false, err + return nil, nil, false, err } foundCSV = &matchedCSV } } + if policy.Spec.ComplianceType.IsMustNotHave() { + earlyConds, changed, err := r.mustnothaveCSV(ctx, policy, csvList, sub.Namespace) + + return foundCSV, earlyConds, changed, err + } // CSV has not yet been created by OLM if foundCSV == nil { changed := updateStatus(policy, missingWantedCond("ClusterServiceVersion"), missingCSVObj(sub.Name, sub.Namespace)) - return foundCSV, changed, nil + return foundCSV, nil, changed, nil + } + + return foundCSV, nil, updateStatus(policy, buildCSVCond(foundCSV), existingCSVObj(foundCSV)), nil +} + +func (r *OperatorPolicyReconciler) mustnothaveCSV( + ctx context.Context, + policy *policyv1beta1.OperatorPolicy, + csvList []unstructured.Unstructured, + namespace string, +) ([]metav1.Condition, bool, error) { + if len(csvList) == 0 { + changed := updateStatus(policy, missingNotWantedCond("ClusterServiceVersion"), + missingNotWantedCSVObj(namespace)) + + return nil, changed, nil + } + + relatedCSVs := make([]policyv1.RelatedObject, len(csvList)) + + if policy.Spec.RemovalBehavior.ApplyDefaults().CSVs.IsKeep() { + for i := range csvList { + relatedCSVs[i] = leftoverObj(&csvList[i]) + } + + return nil, updateStatus(policy, keptCond("ClusterServiceVersion"), relatedCSVs...), nil } - return foundCSV, updateStatus(policy, buildCSVCond(foundCSV), existingCSVObj(foundCSV)), nil + for i := range csvList { + relatedCSVs[i] = foundNotWantedObj(&csvList[i]) + } + + changed := updateStatus(policy, foundNotWantedCond("ClusterServiceVersion"), relatedCSVs...) + + if policy.Spec.RemediationAction.IsInform() { + return nil, changed, nil + } + + earlyConds := []metav1.Condition{} + + if changed { + earlyConds = append(earlyConds, calculateComplianceCondition(policy)) + } + + for i := range csvList { + err := r.Delete(ctx, &csvList[i]) + if err != nil { + changed := updateStatus(policy, foundNotWantedCond("ClusterServiceVersion"), relatedCSVs...) + + return earlyConds, changed, fmt.Errorf("error deleting ClusterServiceVersion: %w", err) + } + + csvList[i].SetGroupVersionKind(clusterServiceVersionGVK) + relatedCSVs[i] = deletedObj(&csvList[i]) + } + + updateStatus(policy, deletedCond("ClusterServiceVersion"), relatedCSVs...) + + return earlyConds, true, nil } func (r *OperatorPolicyReconciler) handleDeployment( @@ -994,6 +1262,10 @@ func (r *OperatorPolicyReconciler) handleDeployment( policy *policyv1beta1.OperatorPolicy, csv *operatorv1alpha1.ClusterServiceVersion, ) (bool, error) { + if policy.Spec.ComplianceType.IsMustNotHave() { + return updateStatus(policy, irrelevantCond("Deployment")), nil + } + // case where csv is nil if csv == nil { // need to report lack of existing Deployments @@ -1046,7 +1318,7 @@ func (r *OperatorPolicyReconciler) handleDeployment( } func (r *OperatorPolicyReconciler) handleCRDs( - _ context.Context, + ctx context.Context, policy *policyv1beta1.OperatorPolicy, sub *operatorv1alpha1.Subscription, ) ([]metav1.Condition, bool, error) { @@ -1062,23 +1334,73 @@ func (r *OperatorPolicyReconciler) handleCRDs( return nil, false, fmt.Errorf("error listing CRDs: %w", err) } + // Same condition for musthave and mustnothave if len(crdList) == 0 { return nil, updateStatus(policy, noCRDCond, noExistingCRDObj), nil } relatedCRDs := make([]policyv1.RelatedObject, len(crdList)) + if policy.Spec.ComplianceType.IsMustHave() { + for i := range crdList { + relatedCRDs[i] = matchedObj(&crdList[i]) + } + + return nil, updateStatus(policy, crdFoundCond, relatedCRDs...), nil + } + + if policy.Spec.RemovalBehavior.ApplyDefaults().CRDs.IsKeep() { + for i := range crdList { + relatedCRDs[i] = leftoverObj(&crdList[i]) + } + + return nil, updateStatus(policy, keptCond("CustomResourceDefinition"), relatedCRDs...), nil + } + for i := range crdList { - relatedCRDs[i] = matchedObj(&crdList[i]) + relatedCRDs[i] = foundNotWantedObj(&crdList[i]) } - return nil, updateStatus(policy, crdFoundCond, relatedCRDs...), nil + changed := updateStatus(policy, foundNotWantedCond("CustomResourceDefinition"), relatedCRDs...) + + if policy.Spec.RemediationAction.IsInform() { + return nil, changed, nil // goog + } + + earlyConds := []metav1.Condition{} + + if changed { + earlyConds = append(earlyConds, calculateComplianceCondition(policy)) + } + + for i := range crdList { + err := r.Delete(ctx, &crdList[i]) + if err != nil { + changed := updateStatus(policy, foundNotWantedCond("CustomResourceDefinition"), relatedCRDs...) + + return earlyConds, changed, fmt.Errorf("error deleting the CRD: %w", err) + } + + crdList[i].SetGroupVersionKind(customResourceDefinitionGVK) + relatedCRDs[i] = deletedObj(&crdList[i]) + } + + updateStatus(policy, deletedCond("CustomResourceDefinition"), relatedCRDs...) + + return earlyConds, true, nil } func (r *OperatorPolicyReconciler) handleCatalogSource( policy *policyv1beta1.OperatorPolicy, subscription *operatorv1alpha1.Subscription, ) (bool, error) { + if policy.Spec.ComplianceType.IsMustNotHave() { + cond := irrelevantCond("CatalogSource") + cond.Status = metav1.ConditionFalse // CatalogSource condition has the opposite polarity + + return updateStatus(policy, cond), nil + } + watcher := opPolIdentifier(policy.Namespace, policy.Name) if subscription == nil { diff --git a/controllers/operatorpolicy_status.go b/controllers/operatorpolicy_status.go index 67974868..0650e767 100644 --- a/controllers/operatorpolicy_status.go +++ b/controllers/operatorpolicy_status.go @@ -415,7 +415,29 @@ func missingWantedCond(kind string) metav1.Condition { } } -// createdCond returns a Compliant condition, with a Reason like'____Created', +// missingNotWantedCond returns a Compliant condition with a Reason like '____Excluded' +// and a Message like 'the ____ checked by the mustnothave policy was not found' +func missingNotWantedCond(kind string) metav1.Condition { + return metav1.Condition{ + Type: condType(kind), + Status: metav1.ConditionTrue, + Reason: kind + "Excluded", + Message: "the " + kind + " checked by the mustnothave policy was not found", + } +} + +// foundNotWantedCond returns a NonCompliant condition with a Reason like '____Included' +// and a Message like 'the ____ checked by the mustnothave policy was found' +func foundNotWantedCond(kind string) metav1.Condition { + return metav1.Condition{ + Type: condType(kind), + Status: metav1.ConditionFalse, + Reason: kind + "Included", + Message: "the " + kind + " checked by the mustnothave policy was found", + } +} + +// createdCond returns a Compliant condition, with a Reason like '____Created', // and a Message like 'the ____ required by the policy was created' func createdCond(kind string) metav1.Condition { return metav1.Condition{ @@ -426,6 +448,30 @@ func createdCond(kind string) metav1.Condition { } } +// deletedCond returns a Compliant condition, with a Reason like '____Deleted', +// and a Message like 'the ____ checked by the mustnothave policy was deleted' +func deletedCond(kind string) metav1.Condition { + return metav1.Condition{ + Type: condType(kind), + Status: metav1.ConditionTrue, + Reason: kind + "Deleted", + Message: "the " + kind + " checked by the mustnothave policy was deleted", + } +} + +// keptCond returns a Compliant condition, with a Reason like '____Kept', +// and a Message like 'the ____ checked by the mustnothave policy specifies +// not to remove this kind' +func keptCond(kind string) metav1.Condition { + return metav1.Condition{ + Type: condType(kind), + Status: metav1.ConditionTrue, + Reason: kind + "Kept", + Message: "the " + kind + " checked by the mustnothave policy specifies " + + "not to remove this kind", + } +} + // matchesCond returns a Compliant condition, with a Reason like'____Matches', // and a Message like 'the ____ matches what is required by the policy' func matchesCond(kind string) metav1.Condition { @@ -514,6 +560,17 @@ func subResFailedCond(subFailedCond operatorv1alpha1.SubscriptionCondition) meta return cond } +// irrelevantCond returns a Compliant condition, with a Reason like '____Irrelevant', +// and a Message like 'MustNotHave policies ignore kind ____' +func irrelevantCond(kind string) metav1.Condition { + return metav1.Condition{ + Type: condType(kind), + Status: metav1.ConditionTrue, + Reason: kind + "Irrelevant", + Message: "MustNotHave policies ignore kind " + kind, + } +} + // opGroupPreexistingCond is a Compliant condition with Reason 'PreexistingOperatorGroupFound', // and Message 'the policy does not specify an OperatorGroup but one already exists in the // namespace - assuming that OperatorGroup is correct' @@ -739,6 +796,27 @@ func missingWantedObj(obj client.Object) policyv1.RelatedObject { } } +// missingNotWantedObj returns a Compliant RelatedObject with reason = 'Resource not found as expected' +func missingNotWantedObj(obj client.Object) policyv1.RelatedObject { + return policyv1.RelatedObject{ + Object: policyv1.ObjectResourceFromObj(obj), + Compliant: string(policyv1.Compliant), + Reason: reasonWantNotFoundDNE, + } +} + +// foundNotWantedObj returns a NonCompliant RelatedObject with reason = 'Resource found but should not exist' +func foundNotWantedObj(obj client.Object) policyv1.RelatedObject { + return policyv1.RelatedObject{ + Object: policyv1.ObjectResourceFromObj(obj), + Compliant: string(policyv1.NonCompliant), + Reason: reasonWantNotFoundExists, + Properties: &policyv1.ObjectProperties{ + UID: string(obj.GetUID()), + }, + } +} + // createdObj returns a Compliant RelatedObject with reason = 'K8s creation success' func createdObj(obj client.Object) policyv1.RelatedObject { created := true @@ -754,6 +832,15 @@ func createdObj(obj client.Object) policyv1.RelatedObject { } } +// deletedObj returns a Compliant RelatedObject with reason = 'K8s deletion success' +func deletedObj(obj client.Object) policyv1.RelatedObject { + return policyv1.RelatedObject{ + Object: policyv1.ObjectResourceFromObj(obj), + Compliant: string(policyv1.Compliant), + Reason: reasonDeleteSuccess, + } +} + // matchedObj returns a Compliant RelatedObject with reason = 'Resource found as expected' func matchedObj(obj client.Object) policyv1.RelatedObject { return policyv1.RelatedObject{ @@ -803,6 +890,21 @@ func nonCompObj(obj client.Object, reason string) policyv1.RelatedObject { } } +// leftoverObj returns a RelatedObject for an object related to a +// mustnothave policy which specifies to keep this kind of object. +// The object does not have a compliance associated with it. +func leftoverObj(obj client.Object) policyv1.RelatedObject { + kind := obj.GetObjectKind().GroupVersionKind().Kind + + return policyv1.RelatedObject{ + Object: policyv1.ObjectResourceFromObj(obj), + Properties: &policyv1.ObjectProperties{ + UID: string(obj.GetUID()), + }, + Reason: "The " + kind + " is attached to a mustnothave policy, but does not need to be removed", + } +} + // opGroupTooManyObjs returns a list of NonCompliant RelatedObjects, each with // reason = 'There is more than one OperatorGroup in this namespace' func opGroupTooManyObjs(opGroups []unstructured.Unstructured) []policyv1.RelatedObject { @@ -888,6 +990,23 @@ func missingCSVObj(name string, namespace string) policyv1.RelatedObject { } } +// missingNotWantedCSVObj returns a Compliant RelatedObject for the ClusterServiceVersion, +// with Reason 'Resource not found as expected' +func missingNotWantedCSVObj(namespace string) policyv1.RelatedObject { + return policyv1.RelatedObject{ + Object: policyv1.ObjectResource{ + Kind: clusterServiceVersionGVK.Kind, + APIVersion: clusterServiceVersionGVK.GroupVersion().String(), + Metadata: policyv1.ObjectMetadata{ + Name: "-", + Namespace: namespace, + }, + }, + Compliant: string(policyv1.Compliant), + Reason: reasonWantNotFoundDNE, + } +} + // existingCSVObj returns a RelatedObject for the ClusterServiceVersion, with a // Reason that reflects the CSV's status, and will only be Compliant if the CSV // is in the Succeeded phase. diff --git a/deploy/crds/kustomize_operatorpolicy/allowed-compliance-types.json b/deploy/crds/kustomize_operatorpolicy/allowed-compliance-types.json index 55ffc812..547d3ef9 100644 --- a/deploy/crds/kustomize_operatorpolicy/allowed-compliance-types.json +++ b/deploy/crds/kustomize_operatorpolicy/allowed-compliance-types.json @@ -2,6 +2,6 @@ { "op":"replace", "path":"/spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/complianceType/enum", - "value": ["musthave"] + "value": ["musthave", "mustnothave"] } ] diff --git a/deploy/crds/kustomize_operatorpolicy/policy.open-cluster-management.io_operatorpolicies.yaml b/deploy/crds/kustomize_operatorpolicy/policy.open-cluster-management.io_operatorpolicies.yaml index be7e48bc..b293df9f 100644 --- a/deploy/crds/kustomize_operatorpolicy/policy.open-cluster-management.io_operatorpolicies.yaml +++ b/deploy/crds/kustomize_operatorpolicy/policy.open-cluster-management.io_operatorpolicies.yaml @@ -68,6 +68,58 @@ spec: - Enforce - enforce type: string + removalBehavior: + default: {} + description: |- + RemovalBehavior defines what resources will be removed by enforced mustnothave policies. + When in inform mode, any resources that would be deleted if the policy was enforced will + be causes for NonCompliance, but resources that would be kept will be considered Compliant. + properties: + clusterServiceVersions: + default: Delete + description: Specifies whether to delete the ClusterServiceVersion; + defaults to 'Delete' + enum: + - Keep + - Delete + type: string + customResourceDefinitions: + default: Keep + description: |- + Specifies whether to delete any CustomResourceDefinitions associated with the operator; + defaults to 'Keep' because deleting them should be done deliberately + enum: + - Keep + - Delete + type: string + installPlans: + default: Keep + description: |- + Specifies whether to delete any InstallPlans associated with the operator; defaults + to 'Keep' because those objects are only for history + enum: + - Keep + - Delete + type: string + operatorGroups: + default: DeleteIfUnused + description: |- + Specifies whether to delete the OperatorGroup; defaults to 'DeleteIfUnused' which + will only delete the OperatorGroup if there is not another Subscription using it. + enum: + - Keep + - Delete + - DeleteIfUnused + type: string + subscriptions: + default: Delete + description: Specifies whether to delete the Subscription; defaults + to 'Delete' + enum: + - Keep + - Delete + type: string + type: object severity: description: 'Severity : low, medium, high, or critical' enum: diff --git a/deploy/crds/policy.open-cluster-management.io_operatorpolicies.yaml b/deploy/crds/policy.open-cluster-management.io_operatorpolicies.yaml index 05f74803..30a22aba 100644 --- a/deploy/crds/policy.open-cluster-management.io_operatorpolicies.yaml +++ b/deploy/crds/policy.open-cluster-management.io_operatorpolicies.yaml @@ -46,6 +46,7 @@ spec: have a given resource enum: - musthave + - mustnothave type: string operatorGroup: description: |- @@ -62,6 +63,58 @@ spec: - Enforce - enforce type: string + removalBehavior: + default: {} + description: |- + RemovalBehavior defines what resources will be removed by enforced mustnothave policies. + When in inform mode, any resources that would be deleted if the policy was enforced will + be causes for NonCompliance, but resources that would be kept will be considered Compliant. + properties: + clusterServiceVersions: + default: Delete + description: Specifies whether to delete the ClusterServiceVersion; + defaults to 'Delete' + enum: + - Keep + - Delete + type: string + customResourceDefinitions: + default: Keep + description: |- + Specifies whether to delete any CustomResourceDefinitions associated with the operator; + defaults to 'Keep' because deleting them should be done deliberately + enum: + - Keep + - Delete + type: string + installPlans: + default: Keep + description: |- + Specifies whether to delete any InstallPlans associated with the operator; defaults + to 'Keep' because those objects are only for history + enum: + - Keep + - Delete + type: string + operatorGroups: + default: DeleteIfUnused + description: |- + Specifies whether to delete the OperatorGroup; defaults to 'DeleteIfUnused' which + will only delete the OperatorGroup if there is not another Subscription using it. + enum: + - Keep + - Delete + - DeleteIfUnused + type: string + subscriptions: + default: Delete + description: Specifies whether to delete the Subscription; defaults + to 'Delete' + enum: + - Keep + - Delete + type: string + type: object severity: description: 'Severity : low, medium, high, or critical' enum: diff --git a/test/e2e/case38_install_operator_test.go b/test/e2e/case38_install_operator_test.go index 396c6005..415a8278 100644 --- a/test/e2e/case38_install_operator_test.go +++ b/test/e2e/case38_install_operator_test.go @@ -19,7 +19,7 @@ import ( "open-cluster-management.io/config-policy-controller/test/utils" ) -var _ = Describe("Test installing an operator from OperatorPolicy", Ordered, func() { +var _ = Describe("Testing OperatorPolicy", Ordered, func() { const ( opPolTestNS = "operator-policy-testns" parentPolicyYAML = "../resources/case38_operator_install/parent-policy.yaml" @@ -1433,4 +1433,836 @@ var _ = Describe("Test installing an operator from OperatorPolicy", Ordered, fun ) }) }) + Describe("Testing general OperatorPolicy mustnothave behavior", Ordered, func() { + const ( + opPolYAML = "../resources/case38_operator_install/operator-policy-mustnothave.yaml" + opPolName = "oppol-mustnothave" + subName = "project-quay" + ) + + BeforeAll(func() { + utils.Kubectl("create", "ns", opPolTestNS) + utils.Kubectl("delete", "crd", "--selector=olm.managed=true") + DeferCleanup(func() { + utils.Kubectl("delete", "ns", opPolTestNS) + }) + + createObjWithParent(parentPolicyYAML, parentPolicyName, + opPolYAML, opPolTestNS, gvrPolicy, gvrOperatorPolicy) + }) + + It("Should be Compliant and report all the things are correctly missing", func() { + check( + opPolName, + false, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "OperatorGroup", + APIVersion: "operators.coreos.com/v1", + Metadata: policyv1.ObjectMetadata{ + Namespace: opPolTestNS, + Name: "-", + }, + }, + Compliant: "Compliant", + Reason: "Resource not found as expected", + }}, + metav1.Condition{ + Type: "OperatorGroupCompliant", + Status: metav1.ConditionTrue, + Reason: "OperatorGroupExcluded", + Message: "the OperatorGroup checked by the mustnothave policy was not found", + }, + `the OperatorGroup checked by the mustnothave policy was not found`, + ) + check( + opPolName, + false, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "Subscription", + APIVersion: "operators.coreos.com/v1alpha1", + Metadata: policyv1.ObjectMetadata{ + Namespace: opPolTestNS, + Name: "project-quay", + }, + }, + Compliant: "Compliant", + Reason: "Resource not found as expected", + }}, + metav1.Condition{ + Type: "SubscriptionCompliant", + Status: metav1.ConditionTrue, + Reason: "SubscriptionExcluded", + Message: "the Subscription checked by the mustnothave policy was not found", + }, + `the Subscription checked by the mustnothave policy was not found`, + ) + check( + opPolName, + false, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "InstallPlan", + APIVersion: "operators.coreos.com/v1alpha1", + Metadata: policyv1.ObjectMetadata{ + Namespace: opPolTestNS, + Name: "-", + }, + }, + Compliant: "Compliant", + Reason: "There are no relevant InstallPlans in this namespace", + }}, + metav1.Condition{ + Type: "InstallPlanCompliant", + Status: metav1.ConditionTrue, + Reason: "NoInstallPlansFound", + Message: "there are no relevant InstallPlans in the namespace", + }, + `there are no relevant InstallPlans in the namespace`, + ) + check( + opPolName, + false, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "ClusterServiceVersion", + APIVersion: "operators.coreos.com/v1alpha1", + Metadata: policyv1.ObjectMetadata{ + Namespace: opPolTestNS, + Name: "-", + }, + }, + Compliant: "Compliant", + Reason: "Resource not found as expected", + }}, + metav1.Condition{ + Type: "ClusterServiceVersionCompliant", + Status: metav1.ConditionTrue, + Reason: "ClusterServiceVersionExcluded", + Message: "the ClusterServiceVersion checked by the mustnothave policy was not found", + }, + `the ClusterServiceVersion checked by the mustnothave policy was not found`, + ) + check( + opPolName, + false, + []policyv1.RelatedObject{}, + metav1.Condition{ + Type: "DeploymentCompliant", + Status: metav1.ConditionTrue, + Reason: "DeploymentIrrelevant", + Message: "MustNotHave policies ignore kind Deployment", + }, + `MustNotHave policies ignore kind Deployment`, + ) + check( + opPolName, + false, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "CustomResourceDefinition", + APIVersion: "apiextensions.k8s.io/v1", + Metadata: policyv1.ObjectMetadata{ + Name: "-", + }, + }, + Compliant: "Compliant", + Reason: "No relevant CustomResourceDefinitions found", + }}, + metav1.Condition{ + Type: "CustomResourceDefinitionCompliant", + Status: metav1.ConditionTrue, + Reason: "RelevantCRDNotFound", + Message: "No CRDs were found for the operator", + }, + `No CRDs were found for the operator`, + ) + check( + opPolName, + false, + []policyv1.RelatedObject{}, + metav1.Condition{ + Type: "CatalogSourcesUnhealthy", + Status: metav1.ConditionFalse, + Reason: "CatalogSourceIrrelevant", + Message: "MustNotHave policies ignore kind CatalogSource", + }, + `MustNotHave policies ignore kind CatalogSource`, + ) + + // The `check` function doesn't check that it is compliant, only that each piece is compliant + pol := utils.GetWithTimeout(clientManagedDynamic, gvrOperatorPolicy, opPolName, + opPolTestNS, true, eventuallyTimeout) + compliance, found, err := unstructured.NestedString(pol.Object, "status", "compliant") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(compliance).To(Equal("Compliant")) + }) + It("Should be NonCompliant and report resources when the operator is installed", func(ctx SpecContext) { + // Make it musthave and enforced, to install the operator + utils.Kubectl("patch", "operatorpolicy", opPolName, "-n", opPolTestNS, "--type=json", "-p", + `[{"op": "replace", "path": "/spec/complianceType", "value": "musthave"},`+ + `{"op": "replace", "path": "/spec/remediationAction", "value": "enforce"}]`) + + By("Waiting for a CRD to appear, which should indicate the operator is installing") + Eventually(func(ctx SpecContext) *unstructured.Unstructured { + crd, _ := clientManagedDynamic.Resource(gvrCRD).Get(ctx, + "quayregistries.quay.redhat.com", metav1.GetOptions{}) + + return crd + }, olmWaitTimeout, 5, ctx).ShouldNot(BeNil()) + + By("Waiting for the policy to become compliant, indicating the operator is installed") + Eventually(func(g Gomega) string { + pol := utils.GetWithTimeout(clientManagedDynamic, gvrOperatorPolicy, opPolName, + opPolTestNS, true, eventuallyTimeout) + compliance, found, err := unstructured.NestedString(pol.Object, "status", "compliant") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + + return compliance + }, olmWaitTimeout, 5, ctx).Should(Equal("Compliant")) + + // Revert to the original mustnothave policy + utils.Kubectl("patch", "operatorpolicy", opPolName, "-n", opPolTestNS, "--type=json", "-p", + `[{"op": "replace", "path": "/spec/complianceType", "value": "mustnothave"},`+ + `{"op": "replace", "path": "/spec/remediationAction", "value": "inform"}]`) + + By("Checking the OperatorPolicy status") + check( + opPolName, + true, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "OperatorGroup", + APIVersion: "operators.coreos.com/v1", + Metadata: policyv1.ObjectMetadata{ + Namespace: opPolTestNS, + }, + }, + Compliant: "NonCompliant", + Reason: "Resource found but should not exist", + }}, + metav1.Condition{ + Type: "OperatorGroupCompliant", + Status: metav1.ConditionFalse, + Reason: "OperatorGroupIncluded", + Message: "the OperatorGroup checked by the mustnothave policy was found", + }, + `the OperatorGroup checked by the mustnothave policy was found`, + ) + check( + opPolName, + true, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "Subscription", + APIVersion: "operators.coreos.com/v1alpha1", + Metadata: policyv1.ObjectMetadata{ + Namespace: opPolTestNS, + Name: "project-quay", + }, + }, + Compliant: "NonCompliant", + Reason: "Resource found but should not exist", + }}, + metav1.Condition{ + Type: "SubscriptionCompliant", + Status: metav1.ConditionFalse, + Reason: "SubscriptionIncluded", + Message: "the Subscription checked by the mustnothave policy was found", + }, + `the Subscription checked by the mustnothave policy was found`, + ) + check( + opPolName, + true, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "InstallPlan", + APIVersion: "operators.coreos.com/v1alpha1", + Metadata: policyv1.ObjectMetadata{ + Namespace: opPolTestNS, + }, + }, + Compliant: "NonCompliant", + Reason: "Resource found but should not exist", + }}, + metav1.Condition{ + Type: "InstallPlanCompliant", + Status: metav1.ConditionFalse, + Reason: "InstallPlanIncluded", + Message: "the InstallPlan checked by the mustnothave policy was found", + }, + `the InstallPlan checked by the mustnothave policy was found`, + ) + check( + opPolName, + true, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "ClusterServiceVersion", + APIVersion: "operators.coreos.com/v1alpha1", + Metadata: policyv1.ObjectMetadata{ + Namespace: opPolTestNS, + }, + }, + Compliant: "NonCompliant", + Reason: "Resource found but should not exist", + }}, + metav1.Condition{ + Type: "ClusterServiceVersionCompliant", + Status: metav1.ConditionFalse, + Reason: "ClusterServiceVersionIncluded", + Message: "the ClusterServiceVersion checked by the mustnothave policy was found", + }, + `the ClusterServiceVersion checked by the mustnothave policy was found`, + ) + check( + opPolName, + true, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "CustomResourceDefinition", + APIVersion: "apiextensions.k8s.io/v1", + Metadata: policyv1.ObjectMetadata{ + Name: "quayecosystems.redhatcop.redhat.io", + }, + }, + Compliant: "NonCompliant", + Reason: "Resource found but should not exist", + }, { + Object: policyv1.ObjectResource{ + Kind: "CustomResourceDefinition", + APIVersion: "apiextensions.k8s.io/v1", + Metadata: policyv1.ObjectMetadata{ + Name: "quayregistries.quay.redhat.com", + }, + }, + Compliant: "NonCompliant", + Reason: "Resource found but should not exist", + }}, + metav1.Condition{ + Type: "CustomResourceDefinitionCompliant", + Status: metav1.ConditionFalse, + Reason: "CustomResourceDefinitionIncluded", + Message: "the CustomResourceDefinition checked by the mustnothave policy was found", + }, + `the CustomResourceDefinition checked by the mustnothave policy was found`, + ) + }) + + // These are the same for inform and enforce, so just write them once + keptChecks := func() { + check( + opPolName, + false, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "OperatorGroup", + APIVersion: "operators.coreos.com/v1", + Metadata: policyv1.ObjectMetadata{ + Namespace: opPolTestNS, + }, + }, + Reason: "The OperatorGroup is attached to a mustnothave policy, but does not need to be removed", + }}, + metav1.Condition{ + Type: "OperatorGroupCompliant", + Status: metav1.ConditionTrue, + Reason: "OperatorGroupKept", + Message: "the OperatorGroup checked by the mustnothave policy specifies not to remove this kind", + }, + `the OperatorGroup checked by the mustnothave policy specifies not to remove this kind`, + ) + check( + opPolName, + false, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "Subscription", + APIVersion: "operators.coreos.com/v1alpha1", + Metadata: policyv1.ObjectMetadata{ + Namespace: opPolTestNS, + Name: "project-quay", + }, + }, + Reason: "The Subscription is attached to a mustnothave policy, but does not need to be removed", + }}, + metav1.Condition{ + Type: "SubscriptionCompliant", + Status: metav1.ConditionTrue, + Reason: "SubscriptionKept", + Message: "the Subscription checked by the mustnothave policy specifies not to remove this kind", + }, + `the Subscription checked by the mustnothave policy specifies not to remove this kind`, + ) + check( + opPolName, + false, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "InstallPlan", + APIVersion: "operators.coreos.com/v1alpha1", + Metadata: policyv1.ObjectMetadata{ + Namespace: opPolTestNS, + }, + }, + Reason: "The InstallPlan is attached to a mustnothave policy, but does not need to be removed", + }}, + metav1.Condition{ + Type: "InstallPlanCompliant", + Status: metav1.ConditionTrue, + Reason: "InstallPlanKept", + Message: "the InstallPlan checked by the mustnothave policy specifies not to remove this kind", + }, + `the InstallPlan checked by the mustnothave policy specifies not to remove this kind`, + ) + check( + opPolName, + false, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "ClusterServiceVersion", + APIVersion: "operators.coreos.com/v1alpha1", + Metadata: policyv1.ObjectMetadata{ + Namespace: opPolTestNS, + }, + }, + Reason: "The ClusterServiceVersion is attached to a mustnothave policy, " + + "but does not need to be removed", + }}, + metav1.Condition{ + Type: "ClusterServiceVersionCompliant", + Status: metav1.ConditionTrue, + Reason: "ClusterServiceVersionKept", + Message: "the ClusterServiceVersion checked by the mustnothave policy specifies " + + "not to remove this kind", + }, + `the ClusterServiceVersion checked by the mustnothave policy specifies not to remove this kind`, + ) + check( + opPolName, + false, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "CustomResourceDefinition", + APIVersion: "apiextensions.k8s.io/v1", + Metadata: policyv1.ObjectMetadata{ + Name: "quayecosystems.redhatcop.redhat.io", + }, + }, + Reason: "The CustomResourceDefinition is attached to a mustnothave policy, but " + + "does not need to be removed", + }, { + Object: policyv1.ObjectResource{ + Kind: "CustomResourceDefinition", + APIVersion: "apiextensions.k8s.io/v1", + Metadata: policyv1.ObjectMetadata{ + Name: "quayregistries.quay.redhat.com", + }, + }, + Reason: "The CustomResourceDefinition is attached to a mustnothave policy, but " + + "does not need to be removed", + }}, + metav1.Condition{ + Type: "CustomResourceDefinitionCompliant", + Status: metav1.ConditionTrue, + Reason: "CustomResourceDefinitionKept", + Message: "the CustomResourceDefinition checked by the mustnothave policy specifies " + + "not to remove this kind", + }, + `the CustomResourceDefinition checked by the mustnothave policy specifies not to remove this kind`, + ) + } + It("Should report resources differently when told to keep them", func(ctx SpecContext) { + // Change the removal behaviors from Delete to Keep + utils.Kubectl("patch", "operatorpolicy", opPolName, "-n", opPolTestNS, "--type=json", "-p", + `[{"op": "replace", "path": "/spec/removalBehavior/operatorGroups", "value": "Keep"},`+ + `{"op": "replace", "path": "/spec/removalBehavior/subscriptions", "value": "Keep"},`+ + `{"op": "replace", "path": "/spec/removalBehavior/clusterServiceVersions", "value": "Keep"},`+ + `{"op": "replace", "path": "/spec/removalBehavior/installPlans", "value": "Keep"},`+ + `{"op": "replace", "path": "/spec/removalBehavior/customResourceDefinitions", "value": "Keep"}]`) + By("Checking the OperatorPolicy status") + keptChecks() + }) + It("Should not remove anything when enforced while set to Keep everything", func(ctx SpecContext) { + // Enforce the policy + utils.Kubectl("patch", "operatorpolicy", opPolName, "-n", opPolTestNS, "--type=json", "-p", + `[{"op": "replace", "path": "/spec/remediationAction", "value": "enforce"}]`) + By("Checking the OperatorPolicy status") + keptChecks() + + By("Checking that certain (named) resources are still there") + utils.GetWithTimeout(clientManagedDynamic, gvrClusterServiceVersion, "quay-operator.v3.8.13", + opPolTestNS, true, eventuallyTimeout) + utils.GetWithTimeout(clientManagedDynamic, gvrSubscription, subName, + opPolTestNS, true, eventuallyTimeout) + utils.GetWithTimeout(clientManagedDynamic, gvrCRD, "quayecosystems.redhatcop.redhat.io", + "", true, eventuallyTimeout) + utils.GetWithTimeout(clientManagedDynamic, gvrCRD, "quayregistries.quay.redhat.com", + "", true, eventuallyTimeout) + }) + It("Should not remove anything when enforced while set to Keep everything", func(ctx SpecContext) { + // Change the removal behaviors from Delete to Keep + utils.Kubectl("patch", "operatorpolicy", opPolName, "-n", opPolTestNS, "--type=json", "-p", + `[{"op": "replace", "path": "/spec/removalBehavior/operatorGroups", "value": "Delete"},`+ + `{"op": "replace", "path": "/spec/removalBehavior/subscriptions", "value": "Delete"},`+ + `{"op": "replace", "path": "/spec/removalBehavior/clusterServiceVersions", "value": "Delete"},`+ + `{"op": "replace", "path": "/spec/removalBehavior/installPlans", "value": "Delete"},`+ + `{"op": "replace", "path": "/spec/removalBehavior/customResourceDefinitions", "value": "Delete"}]`) + + By("Checking that certain (named) resources are not there, indicating the removal was completed") + utils.GetWithTimeout(clientManagedDynamic, gvrClusterServiceVersion, "quay-operator.v3.8.13", + opPolTestNS, false, eventuallyTimeout) + utils.GetWithTimeout(clientManagedDynamic, gvrSubscription, subName, + opPolTestNS, false, eventuallyTimeout) + utils.GetWithTimeout(clientManagedDynamic, gvrCRD, "quayecosystems.redhatcop.redhat.io", + "", false, eventuallyTimeout) + utils.GetWithTimeout(clientManagedDynamic, gvrCRD, "quayregistries.quay.redhat.com", + "", false, eventuallyTimeout) + + By("Checking the OperatorPolicy status") + check( + opPolName, + false, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "OperatorGroup", + APIVersion: "operators.coreos.com/v1", + Metadata: policyv1.ObjectMetadata{ + Namespace: opPolTestNS, + Name: "-", + }, + }, + Compliant: "Compliant", + Reason: "Resource not found as expected", + }}, + metav1.Condition{ + Type: "OperatorGroupCompliant", + Status: metav1.ConditionTrue, + Reason: "OperatorGroupExcluded", + Message: "the OperatorGroup checked by the mustnothave policy was not found", + }, + `the OperatorGroup checked by the mustnothave policy was deleted`, + ) + check( + opPolName, + false, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "Subscription", + APIVersion: "operators.coreos.com/v1alpha1", + Metadata: policyv1.ObjectMetadata{ + Namespace: opPolTestNS, + Name: "project-quay", + }, + }, + Compliant: "Compliant", + Reason: "Resource not found as expected", + }}, + metav1.Condition{ + Type: "SubscriptionCompliant", + Status: metav1.ConditionTrue, + Reason: "SubscriptionExcluded", + Message: "the Subscription checked by the mustnothave policy was not found", + }, + `the Subscription checked by the mustnothave policy was deleted`, + ) + check( + opPolName, + false, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "InstallPlan", + APIVersion: "operators.coreos.com/v1alpha1", + Metadata: policyv1.ObjectMetadata{ + Namespace: opPolTestNS, + Name: "-", + }, + }, + Compliant: "Compliant", + Reason: "There are no relevant InstallPlans in this namespace", + }}, + metav1.Condition{ + Type: "InstallPlanCompliant", + Status: metav1.ConditionTrue, + Reason: "NoInstallPlansFound", + Message: "there are no relevant InstallPlans in the namespace", + }, + `the InstallPlan checked by the mustnothave policy was deleted`, + ) + check( + opPolName, + false, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "ClusterServiceVersion", + APIVersion: "operators.coreos.com/v1alpha1", + Metadata: policyv1.ObjectMetadata{ + Namespace: opPolTestNS, + Name: "-", + }, + }, + Compliant: "Compliant", + Reason: "Resource not found as expected", + }}, + metav1.Condition{ + Type: "ClusterServiceVersionCompliant", + Status: metav1.ConditionTrue, + Reason: "ClusterServiceVersionExcluded", + Message: "the ClusterServiceVersion checked by the mustnothave policy was not found", + }, + `the ClusterServiceVersion checked by the mustnothave policy was deleted`, + ) + check( + opPolName, + false, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "CustomResourceDefinition", + APIVersion: "apiextensions.k8s.io/v1", + Metadata: policyv1.ObjectMetadata{ + Name: "-", + }, + }, + Compliant: "Compliant", + Reason: "No relevant CustomResourceDefinitions found", + }}, + metav1.Condition{ + Type: "CustomResourceDefinitionCompliant", + Status: metav1.ConditionTrue, + Reason: "RelevantCRDNotFound", + Message: "No CRDs were found for the operator", + }, + `the CustomResourceDefinition checked by the mustnothave policy was deleted`, + ) + + // the checks don't verify that the policy is compliant, do that now: + pol := utils.GetWithTimeout(clientManagedDynamic, gvrOperatorPolicy, opPolName, + opPolTestNS, true, eventuallyTimeout) + compliance, found, err := unstructured.NestedString(pol.Object, "status", "compliant") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(compliance).To(Equal("Compliant")) + }) + }) + Describe("Testing mustnothave behavior for an operator group that is different than the specified one", func() { + const ( + opPolYAML = "../resources/case38_operator_install/operator-policy-with-group.yaml" + opPolName = "oppol-with-group" + subName = "project-quay" + ) + + BeforeEach(func() { + utils.Kubectl("create", "ns", opPolTestNS) + DeferCleanup(func() { + utils.Kubectl("delete", "ns", opPolTestNS) + }) + + createObjWithParent(parentPolicyYAML, parentPolicyName, + opPolYAML, opPolTestNS, gvrPolicy, gvrOperatorPolicy) + }) + + It("should not report an operator group that does not match the spec", func() { + // create the extra operator group + utils.Kubectl("apply", "-f", "../resources/case38_operator_install/incorrect-operator-group.yaml", + "-n", opPolTestNS) + // change the operator policy to mustnothave + utils.Kubectl("patch", "operatorpolicy", opPolName, "-n", opPolTestNS, "--type=json", "-p", + `[{"op": "replace", "path": "/spec/complianceType", "value": "mustnothave"}]`) + + check( + opPolName, + false, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "OperatorGroup", + APIVersion: "operators.coreos.com/v1", + Metadata: policyv1.ObjectMetadata{ + Namespace: opPolTestNS, + Name: "scoped-operator-group", + }, + }, + Compliant: "Compliant", + Reason: "Resource not found as expected", + }}, + metav1.Condition{ + Type: "OperatorGroupCompliant", + Status: metav1.ConditionTrue, + Reason: "OperatorGroupExcluded", + Message: "the OperatorGroup checked by the mustnothave policy was not found", + }, + "the OperatorGroup checked by the mustnothave policy was not found", + ) + }) + }) + Describe("Testing mustnothave behavior of operator groups in DeleteIfUnused mode", Ordered, func() { + const ( + opPolYAML = "../resources/case38_operator_install/operator-policy-mustnothave.yaml" + otherYAML = "../resources/case38_operator_install/operator-policy-authorino.yaml" + opPolName = "oppol-mustnothave" + subName = "project-quay" + ) + + BeforeEach(func() { + utils.Kubectl("create", "ns", opPolTestNS) + utils.Kubectl("delete", "crd", "--selector=olm.managed=true") + DeferCleanup(func() { + utils.Kubectl("delete", "ns", opPolTestNS) + }) + + createObjWithParent(parentPolicyYAML, parentPolicyName, + opPolYAML, opPolTestNS, gvrPolicy, gvrOperatorPolicy) + }) + + It("should delete the operator group when there is only one subscription", func(ctx SpecContext) { + // enforce it as a musthave in order to install the operator + utils.Kubectl("patch", "operatorpolicy", opPolName, "-n", opPolTestNS, "--type=json", "-p", + `[{"op": "replace", "path": "/spec/complianceType", "value": "musthave"},`+ + `{"op": "replace", "path": "/spec/remediationAction", "value": "enforce"},`+ + `{"op": "replace", "path": "/spec/removalBehavior/operatorGroups", "value": "DeleteIfUnused"}]`) + + By("Waiting for a CRD to appear, which should indicate the operator is installing.") + Eventually(func(ctx SpecContext) *unstructured.Unstructured { + crd, _ := clientManagedDynamic.Resource(gvrCRD).Get(ctx, + "quayregistries.quay.redhat.com", metav1.GetOptions{}) + + return crd + }, olmWaitTimeout, 5, ctx).ShouldNot(BeNil()) + + By("Waiting for the policy to become compliant, indicating the operator is installed") + Eventually(func(g Gomega) string { + pol := utils.GetWithTimeout(clientManagedDynamic, gvrOperatorPolicy, opPolName, + opPolTestNS, true, eventuallyTimeout) + compliance, found, err := unstructured.NestedString(pol.Object, "status", "compliant") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + + return compliance + }, olmWaitTimeout, 5, ctx).Should(Equal("Compliant")) + + By("Verifying that an operator group exists") + Eventually(func(g Gomega) []unstructured.Unstructured { + list, err := clientManagedDynamic.Resource(gvrOperatorGroup).Namespace(opPolTestNS). + List(ctx, metav1.ListOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + + return list.Items + }, eventuallyTimeout, 3, ctx).ShouldNot(BeEmpty()) + + // revert it to mustnothave + utils.Kubectl("patch", "operatorpolicy", opPolName, "-n", opPolTestNS, "--type=json", "-p", + `[{"op": "replace", "path": "/spec/complianceType", "value": "mustnothave"}]`) + + By("Verifying that the operator group was removed") + Eventually(func(g Gomega) []unstructured.Unstructured { + list, err := clientManagedDynamic.Resource(gvrOperatorGroup).Namespace(opPolTestNS). + List(ctx, metav1.ListOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + + return list.Items + }, eventuallyTimeout, 3, ctx).Should(BeEmpty()) + }) + + It("should not delete the operator group when there is another subscription", func(ctx SpecContext) { + // enforce it as a musthave in order to install the operator + utils.Kubectl("patch", "operatorpolicy", opPolName, "-n", opPolTestNS, "--type=json", "-p", + `[{"op": "replace", "path": "/spec/complianceType", "value": "musthave"},`+ + `{"op": "replace", "path": "/spec/remediationAction", "value": "enforce"},`+ + `{"op": "replace", "path": "/spec/removalBehavior/operatorGroups", "value": "DeleteIfUnused"}]`) + + By("Waiting for a CRD to appear, which should indicate the operator is installing.") + Eventually(func(ctx SpecContext) *unstructured.Unstructured { + crd, _ := clientManagedDynamic.Resource(gvrCRD).Get(ctx, + "quayregistries.quay.redhat.com", metav1.GetOptions{}) + + return crd + }, olmWaitTimeout, 5, ctx).ShouldNot(BeNil()) + + By("Waiting for the policy to become compliant, indicating the operator is installed") + Eventually(func(g Gomega) string { + pol := utils.GetWithTimeout(clientManagedDynamic, gvrOperatorPolicy, opPolName, + opPolTestNS, true, eventuallyTimeout) + compliance, found, err := unstructured.NestedString(pol.Object, "status", "compliant") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + + return compliance + }, olmWaitTimeout, 5, ctx).Should(Equal("Compliant")) + + By("Verifying that an operator group exists") + Eventually(func(g Gomega) []unstructured.Unstructured { + list, err := clientManagedDynamic.Resource(gvrOperatorGroup).Namespace(opPolTestNS). + List(ctx, metav1.ListOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + + return list.Items + }, eventuallyTimeout, 3, ctx).ShouldNot(BeEmpty()) + + By("Creating another operator policy in the namespace") + createObjWithParent(parentPolicyYAML, parentPolicyName, + otherYAML, opPolTestNS, gvrPolicy, gvrOperatorPolicy) + + // enforce the other policy + utils.Kubectl("patch", "operatorpolicy", "oppol-authorino", "-n", opPolTestNS, "--type=json", "-p", + `[{"op": "replace", "path": "/spec/remediationAction", "value": "enforce"}]`) + + By("Waiting for a CRD to appear, which should indicate the other operator was successfully installed.") + Eventually(func(ctx SpecContext) *unstructured.Unstructured { + crd, _ := clientManagedDynamic.Resource(gvrCRD).Get(ctx, + "authconfigs.authorino.kuadrant.io", metav1.GetOptions{}) + + return crd + }, olmWaitTimeout, 5, ctx).ShouldNot(BeNil()) + + // revert main policy to mustnothave + utils.Kubectl("patch", "operatorpolicy", opPolName, "-n", opPolTestNS, "--type=json", "-p", + `[{"op": "replace", "path": "/spec/complianceType", "value": "mustnothave"}]`) + + By("Verifying the operator group was not removed") + Consistently(func(g Gomega) []unstructured.Unstructured { + list, err := clientManagedDynamic.Resource(gvrOperatorGroup).Namespace(opPolTestNS). + List(ctx, metav1.ListOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + + return list.Items + }, consistentlyDuration, 3, ctx).ShouldNot(BeEmpty()) + }) + }) + Describe("Testing defaulted values in an OperatorPolicy", func() { + const ( + opPolYAML = "../resources/case38_operator_install/operator-policy-authorino.yaml" + opPolName = "oppol-authorino" + subName = "authorino-operator" + ) + + BeforeEach(func() { + utils.Kubectl("create", "ns", opPolTestNS) + DeferCleanup(func() { + utils.Kubectl("delete", "ns", opPolTestNS) + }) + + createObjWithParent(parentPolicyYAML, parentPolicyName, + opPolYAML, opPolTestNS, gvrPolicy, gvrOperatorPolicy) + }) + + It("Should have applied defaults to the removalBehavior field", func(ctx SpecContext) { + policy, err := clientManagedDynamic.Resource(gvrOperatorPolicy).Namespace(opPolTestNS). + Get(ctx, opPolName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(policy).NotTo(BeNil()) + + remBehavior, found, err := unstructured.NestedStringMap(policy.Object, "spec", "removalBehavior") + Expect(found).To(BeTrue()) + Expect(err).NotTo(HaveOccurred()) + + Expect(remBehavior).To(HaveKeyWithValue("operatorGroups", "DeleteIfUnused")) + Expect(remBehavior).To(HaveKeyWithValue("subscriptions", "Delete")) + Expect(remBehavior).To(HaveKeyWithValue("clusterServiceVersions", "Delete")) + Expect(remBehavior).To(HaveKeyWithValue("installPlans", "Keep")) + Expect(remBehavior).To(HaveKeyWithValue("customResourceDefinitions", "Keep")) + }) + }) }) diff --git a/test/resources/case38_operator_install/operator-policy-mustnothave.yaml b/test/resources/case38_operator_install/operator-policy-mustnothave.yaml new file mode 100644 index 00000000..4b994677 --- /dev/null +++ b/test/resources/case38_operator_install/operator-policy-mustnothave.yaml @@ -0,0 +1,32 @@ +apiVersion: policy.open-cluster-management.io/v1beta1 +kind: OperatorPolicy +metadata: + name: oppol-mustnothave + annotations: + policy.open-cluster-management.io/parent-policy-compliance-db-id: "124" + policy.open-cluster-management.io/policy-compliance-db-id: "64" + ownerReferences: + - apiVersion: policy.open-cluster-management.io/v1 + kind: Policy + name: parent-policy + uid: 12345678-90ab-cdef-1234-567890abcdef # must be replaced before creation +spec: + remediationAction: inform + severity: medium + complianceType: mustnothave + subscription: + channel: stable-3.8 + name: project-quay + namespace: operator-policy-testns + installPlanApproval: Manual + source: operatorhubio-catalog + sourceNamespace: olm + startingCSV: quay-operator.v3.8.13 + versions: + - quay-operator.v3.8.13 + removalBehavior: + operatorGroups: Delete + subscriptions: Delete + clusterServiceVersions: Delete + installPlans: Delete + customResourceDefinitions: Delete