diff --git a/controllers/operatorpolicy_controller.go b/controllers/operatorpolicy_controller.go index b6d83355..9a66f190 100644 --- a/controllers/operatorpolicy_controller.go +++ b/controllers/operatorpolicy_controller.go @@ -13,6 +13,7 @@ import ( operatorv1 "github.com/operator-framework/api/pkg/operators/v1" operatorv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" depclient "github.com/stolostron/kubernetes-dependency-watches/client" + appsv1 "k8s.io/api/apps/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -28,6 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" + policyv1 "open-cluster-management.io/config-policy-controller/api/v1" policyv1beta1 "open-cluster-management.io/config-policy-controller/api/v1beta1" ) @@ -38,15 +40,23 @@ const ( var ( subscriptionGVK = schema.GroupVersionKind{ Group: "operators.coreos.com", - Version: "v1alpha1", Kind: "Subscription", + Version: "v1alpha1", + Kind: "Subscription", } operatorGroupGVK = schema.GroupVersionKind{ Group: "operators.coreos.com", - Version: "v1", Kind: "OperatorGroup", + Version: "v1", + Kind: "OperatorGroup", } clusterServiceVersionGVK = schema.GroupVersionKind{ Group: "operators.coreos.com", - Version: "v1alpha1", Kind: "ClusterServiceVersion", + Version: "v1alpha1", + Kind: "ClusterServiceVersion", + } + deploymentGVK = schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", } ) @@ -135,20 +145,26 @@ func (r *OperatorPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reque return reconcile.Result{}, err } - _, err = r.handleSubscription(ctx, policy) + subscription, err := r.handleSubscription(ctx, policy) if err != nil { OpLog.Error(err, "Error handling Subscription") return reconcile.Result{}, err } - _, err = r.handleCSV(ctx, policy, nil) + csv, err := r.handleCSV(ctx, policy, subscription) if err != nil { OpLog.Error(err, "Error handling CSVs") return reconcile.Result{}, err } + if err := r.handleDeployment(ctx, policy, csv); err != nil { + OpLog.Error(err, "Error handling Deployments") + + return reconcile.Result{}, err + } + return reconcile.Result{}, nil } @@ -512,33 +528,42 @@ func buildSubscription( func (r *OperatorPolicyReconciler) handleCSV(ctx context.Context, policy *policyv1beta1.OperatorPolicy, - subscription *unstructured.Unstructured, + sub *operatorv1alpha1.Subscription, ) (*operatorv1alpha1.ClusterServiceVersion, error) { // case where subscription is nil - if subscription == nil { - return nil, nil + if sub == nil { + // need to report lack of existing CSV + err := r.updateStatus(ctx, policy, noCSVCond, noExistingCSVObj) + if err != nil { + return nil, fmt.Errorf("error updating the status for Deployments: %w", err) + } + + return nil, err } watcher := opPolIdentifier(policy.Namespace, policy.Name) - unstructured := subscription.UnstructuredContent() - var sub operatorv1alpha1.Subscription + // case where subscription status has not been populated yet + if sub.Status.InstalledCSV == "" { + err := r.updateStatus(ctx, policy, noCSVCond, noExistingCSVObj) + if err != nil { + return nil, fmt.Errorf("error updating the status for Deployments: %w", err) + } - err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructured, &sub) - if err != nil { return nil, err } // Get the CSV related to the object foundCSV, err := r.DynamicWatcher.Get(watcher, clusterServiceVersionGVK, sub.Namespace, - sub.Status.CurrentCSV) + sub.Status.InstalledCSV) if err != nil { return nil, err } // CSV has not yet been created by OLM if foundCSV == nil { - err := r.updateStatus(ctx, policy, missingWantedCond("ClusterServiceVersion"), missingCSVObj(&sub)) + err := r.updateStatus(ctx, policy, + missingWantedCond("ClusterServiceVersion"), missingCSVObj(sub.Name, sub.Namespace)) if err != nil { return nil, fmt.Errorf("error updating the status for a missing ClusterServiceVersion: %w", err) } @@ -547,10 +572,10 @@ func (r *OperatorPolicyReconciler) handleCSV(ctx context.Context, } // Check CSV most recent condition - unstructured = foundCSV.UnstructuredContent() + unstructured := foundCSV.UnstructuredContent() var csv operatorv1alpha1.ClusterServiceVersion - err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructured, &csv) + err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructured, &csv) if err != nil { return nil, err } @@ -563,6 +588,73 @@ func (r *OperatorPolicyReconciler) handleCSV(ctx context.Context, return &csv, nil } +func (r *OperatorPolicyReconciler) handleDeployment( + ctx context.Context, + policy *policyv1beta1.OperatorPolicy, + csv *operatorv1alpha1.ClusterServiceVersion, +) error { + // case where csv is nil + if csv == nil { + // need to report lack of existing Deployments + err := r.updateStatus(ctx, policy, noDeploymentsCond, noExistingDeploymentObj) + if err != nil { + return fmt.Errorf("error updating the status for Deployments: %w", err) + } + + return err + } + + OpLog := ctrl.LoggerFrom(ctx) + + watcher := opPolIdentifier(policy.Namespace, policy.Name) + + var relatedObjects []policyv1.RelatedObject + var unavailableDeployments []appsv1.Deployment + + depNum := 0 + + for _, dep := range csv.Spec.InstallStrategy.StrategySpec.DeploymentSpecs { + foundDep, err := r.DynamicWatcher.Get(watcher, deploymentGVK, csv.Namespace, dep.Name) + if err != nil { + return fmt.Errorf("error getting the Deployment: %w", err) + } + + // report missing deployment in relatedObjects list + if foundDep == nil { + relatedObjects = append(relatedObjects, missingDeploymentObj(dep.Name, csv.Namespace)) + + continue + } + + unstructured := foundDep.UnstructuredContent() + var dep appsv1.Deployment + + err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructured, &dep) + if err != nil { + OpLog.Error(err, "Unable to convert unstructured Deployment to typed", "Deployment.Name", dep.Name) + + continue + } + + // check for unavailable deployments and build relatedObjects list + if dep.Status.UnavailableReplicas > 0 { + unavailableDeployments = append(unavailableDeployments, dep) + } + + depNum++ + + relatedObjects = append(relatedObjects, existingDeploymentObj(&dep)) + } + + err := r.updateStatus(ctx, policy, + buildDeploymentCond(depNum > 0, unavailableDeployments), relatedObjects...) + if err != nil { + return fmt.Errorf("error updating the status for Deployments: %w", err) + } + + return nil +} + func opPolIdentifier(namespace, name string) depclient.ObjectIdentifier { return depclient.ObjectIdentifier{ Group: policyv1beta1.GroupVersion.Group, diff --git a/controllers/operatorpolicy_status.go b/controllers/operatorpolicy_status.go index 309ca8ea..4afc25b8 100644 --- a/controllers/operatorpolicy_status.go +++ b/controllers/operatorpolicy_status.go @@ -8,6 +8,7 @@ import ( "time" operatorv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -212,6 +213,18 @@ func calculateComplianceCondition(policy *policyv1beta1.OperatorPolicy) metav1.C } } + idx, cond = policy.Status.GetCondition(deploymentConditionType) + if idx == -1 { + messages = append(messages, "the status of the Deployments are unknown") + foundNonCompliant = true + } else { + messages = append(messages, cond.Message) + + if cond.Status != metav1.ConditionTrue { + foundNonCompliant = true + } + } + // FUTURE: check additional conditions if foundNonCompliant { @@ -287,10 +300,11 @@ func (r *OperatorPolicyReconciler) emitComplianceEvent( } const ( - compliantConditionType = "Compliant" - opGroupConditionType = "OperatorGroupCompliant" - subConditionType = "SubscriptionCompliant" - csvConditionType = "CSVCompliant" + compliantConditionType = "Compliant" + opGroupConditionType = "OperatorGroupCompliant" + subConditionType = "SubscriptionCompliant" + csvConditionType = "ClusterServiceVersionCompliant" + deploymentConditionType = "DeploymentCompliant" ) func condType(kind string) string { @@ -301,6 +315,8 @@ func condType(kind string) string { return subConditionType case "ClusterServiceVersion": return csvConditionType + case "Deployment": + return deploymentConditionType default: panic("Unknown condition type for kind " + kind) } @@ -372,6 +388,23 @@ func updatedCond(kind string) metav1.Condition { } } +var opGroupPreexistingCond = metav1.Condition{ + Type: opGroupConditionType, + Status: metav1.ConditionTrue, + Reason: "PreexistingOperatorGroupFound", + Message: "the policy does not specify an OperatorGroup but one already exists in the namespace - " + + "assuming that OperatorGroup is correct", +} + +// opGroupTooManyCond is a NonCompliant condition with a Reason like 'TooManyOperatorGroups', +// and a Message like 'there is more than one OperatorGroup in the namespace' +var opGroupTooManyCond = metav1.Condition{ + Type: opGroupConditionType, + Status: metav1.ConditionFalse, + Reason: "TooManyOperatorGroups", + Message: "there is more than one OperatorGroup in the namespace", +} + // buildCSVCond takes a csv and returns a shortened version of its most recent Condition func buildCSVCond(csv *operatorv1alpha1.ClusterServiceVersion) metav1.Condition { status := metav1.ConditionFalse @@ -382,26 +415,57 @@ func buildCSVCond(csv *operatorv1alpha1.ClusterServiceVersion) metav1.Condition return metav1.Condition{ Type: condType(csv.Kind), Status: status, - Reason: string(csv.Status.Reason) + "Phase" + string(csv.Status.Phase), + Reason: string(csv.Status.Reason), Message: "ClusterServiceVersion - " + csv.Status.Message, } } -var opGroupPreexistingCond = metav1.Condition{ - Type: opGroupConditionType, - Status: metav1.ConditionTrue, - Reason: "PreexistingOperatorGroupFound", - Message: "the policy does not specify an OperatorGroup but one already exists in the namespace - " + - "assuming that OperatorGroup is correct", +var noCSVCond = metav1.Condition{ + Type: csvConditionType, + Status: metav1.ConditionTrue, + Reason: "NoRelevantClusterServiceVersion", + Message: "A relevant installed ClusterServiceVersion could not be found", } -// opGroupTooManyCond is a NonCompliant condition with a Reason like 'TooManyOperatorGroups', -// and a Message like 'there is more than one OperatorGroup in the namespace' -var opGroupTooManyCond = metav1.Condition{ - Type: opGroupConditionType, - Status: metav1.ConditionFalse, - Reason: "TooManyOperatorGroups", - Message: "there is more than one OperatorGroup in the namespace", +func buildDeploymentCond( + depsExist bool, + unavailableDeps []appsv1.Deployment, +) metav1.Condition { + status := metav1.ConditionTrue + reason := "DeploymentsAvailable" + message := "All operator Deployments have their minimum availability" + + if !depsExist { + reason = "NoExistingDeployments" + message = "No existing operator Deployments" + } + + if len(unavailableDeps) != 0 { + status = metav1.ConditionFalse + reason = "DeploymentsUnavailable" + + var depNames []string + for _, dep := range unavailableDeps { + depNames = append(depNames, dep.Name) + } + + names := strings.Join(depNames, ", ") + message = fmt.Sprintf("Deployments %s do not have their minimum availability", names) + } + + return metav1.Condition{ + Type: condType(deploymentGVK.Kind), + Status: status, + Reason: reason, + Message: message, + } +} + +var noDeploymentsCond = metav1.Condition{ + Type: deploymentConditionType, + Status: metav1.ConditionTrue, + Reason: "NoRelevantDeployments", + Message: "The ClusterServiceVersion is missing, thus meaning there are no relevant deployments", } // missingWantedObj returns a NonCompliant RelatedObject with reason = 'Resource not found but should exist' @@ -483,14 +547,14 @@ func opGroupTooManyObjs(opGroups []unstructured.Unstructured) []policyv1.Related return objs } -func missingCSVObj(sub *operatorv1alpha1.Subscription) policyv1.RelatedObject { +func missingCSVObj(name string, namespace string) policyv1.RelatedObject { return policyv1.RelatedObject{ Object: policyv1.ObjectResource{ Kind: clusterServiceVersionGVK.Kind, APIVersion: clusterServiceVersionGVK.GroupVersion().String(), Metadata: policyv1.ObjectMetadata{ - Name: sub.Status.CurrentCSV, - Namespace: sub.GetNamespace(), + Name: name, + Namespace: namespace, }, }, Compliant: string(policyv1.NonCompliant), @@ -504,19 +568,72 @@ func existingCSVObj(csv *operatorv1alpha1.ClusterServiceVersion) policyv1.Relate compliance = policyv1.Compliant } + return policyv1.RelatedObject{ + Object: policyv1.ObjectResourceFromObj(csv), + Compliant: string(compliance), + Reason: string(csv.Status.Reason), + Properties: &policyv1.ObjectProperties{ + UID: string(csv.GetUID()), + }, + } +} + +// represents a lack of relevant CSV +var noExistingCSVObj = policyv1.RelatedObject{ + Object: policyv1.ObjectResource{ + Kind: clusterServiceVersionGVK.Kind, + APIVersion: clusterServiceVersionGVK.GroupVersion().String(), + Metadata: policyv1.ObjectMetadata{ + Name: "*", + }, + }, + Compliant: string(policyv1.UnknownCompliancy), + Reason: "No relevant ClusterServiceVersion found", +} + +func missingDeploymentObj(name string, namespace string) policyv1.RelatedObject { return policyv1.RelatedObject{ Object: policyv1.ObjectResource{ - Kind: clusterServiceVersionGVK.Kind, - APIVersion: clusterServiceVersionGVK.GroupVersion().String(), + Kind: deploymentGVK.Kind, + APIVersion: deploymentGVK.GroupVersion().String(), Metadata: policyv1.ObjectMetadata{ - Name: csv.Name, - Namespace: csv.GetNamespace(), + Name: name, + Namespace: namespace, }, }, + Compliant: string(policyv1.NonCompliant), + Reason: reasonWantFoundDNE, + } +} + +func existingDeploymentObj(dep *appsv1.Deployment) policyv1.RelatedObject { + compliance := policyv1.NonCompliant + reason := "Deployment Unavailable" + + if dep.Status.UnavailableReplicas == 0 { + compliance = policyv1.Compliant + reason = "Deployment Available" + } + + return policyv1.RelatedObject{ + Object: policyv1.ObjectResourceFromObj(dep), Compliant: string(compliance), - Reason: string(csv.Status.Reason) + "Phase" + string(csv.Status.Phase), + Reason: reason, Properties: &policyv1.ObjectProperties{ - UID: string(csv.GetUID()), + UID: string(dep.GetUID()), }, } } + +// represents a lack of relevant deployments +var noExistingDeploymentObj = policyv1.RelatedObject{ + Object: policyv1.ObjectResource{ + Kind: deploymentGVK.Kind, + APIVersion: deploymentGVK.GroupVersion().String(), + Metadata: policyv1.ObjectMetadata{ + Name: "*", + }, + }, + Compliant: string(policyv1.UnknownCompliancy), + Reason: "No relevant deployments found", +} diff --git a/test/e2e/case38_install_operator_test.go b/test/e2e/case38_install_operator_test.go index 0ef4fed9..0406679d 100644 --- a/test/e2e/case38_install_operator_test.go +++ b/test/e2e/case38_install_operator_test.go @@ -17,7 +17,7 @@ var _ = Describe("Test installing an operator from OperatorPolicy", Ordered, fun opPolTestNS = "operator-policy-testns" parentPolicyYAML = "../resources/case38_operator_install/parent-policy.yaml" parentPolicyName = "parent-policy" - opPolTimeout = 60 + opPolTimeout = 30 ) check := func( @@ -551,7 +551,7 @@ var _ = Describe("Test installing an operator from OperatorPolicy", Ordered, fun }) }) - Describe("Test health checks on CSV after installing an operator with OperatorPolicy", Ordered, func() { + Describe("Test health checks on OLM resources after OperatorPolicy operator installation", Ordered, func() { const ( opPolYAML = "../resources/case38_operator_install/operator-policy-no-group-enforce.yaml" opPolName = "oppol-no-group-enforce" @@ -576,16 +576,100 @@ var _ = Describe("Test installing an operator from OperatorPolicy", Ordered, fun APIVersion: "operators.coreos.com/v1alpha1", }, Compliant: "Compliant", - Reason: "InstallSucceededPhaseSucceeded", + Reason: "InstallSucceeded", }}, metav1.Condition{ - Type: "CSVCompliant", + Type: "ClusterServiceVersionCompliant", Status: metav1.ConditionTrue, - Reason: "InstallSucceededPhaseSucceeded", + Reason: "InstallSucceeded", Message: "ClusterServiceVersion - install strategy completed with no errors", }, "ClusterServiceVersion - install strategy completed with no errors", ) }) + + It("Should generate conditions and relatedobjects of Deployments", func() { + check( + opPolName, + false, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + Compliant: "Compliant", + Reason: "Deployment Available", + }}, + metav1.Condition{ + Type: "DeploymentCompliant", + Status: metav1.ConditionTrue, + Reason: "DeploymentsAvailable", + Message: "All operator Deployments have their minimum availability", + }, + "All operator Deployments have their minimum availability", + ) + }) + }) + + Describe("Test health checks on OLM resources on OperatorPolicy with failed CSV", Ordered, func() { + const ( + opPolYAML = "../resources/case38_operator_install/operator-policy-no-group-csv-fail.yaml" + opPolName = "oppol-no-allnamespaces" + ) + BeforeAll(func() { + utils.Kubectl("create", "ns", opPolTestNS) + DeferCleanup(func() { + utils.Kubectl("delete", "ns", opPolTestNS) + }) + + createObjWithParent(parentPolicyYAML, parentPolicyName, + opPolYAML, opPolTestNS, gvrPolicy, gvrOperatorPolicy) + }) + + It("Should generate conditions and relatedobjects of CSV", func() { + check( + opPolName, + false, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "ClusterServiceVersion", + APIVersion: "operators.coreos.com/v1alpha1", + }, + Compliant: "NonCompliant", + Reason: "UnsupportedOperatorGroup", + }}, + metav1.Condition{ + Type: "ClusterServiceVersionCompliant", + Status: metav1.ConditionFalse, + Reason: "UnsupportedOperatorGroup", + Message: "ClusterServiceVersion - AllNamespaces InstallModeType not supported," + + " cannot configure to watch all namespaces", + }, + "ClusterServiceVersion - AllNamespaces InstallModeType not supported,"+ + " cannot configure to watch all namespaces", + ) + }) + + It("Should generate conditions and relatedobjects of Deployments", func() { + check( + opPolName, + false, + []policyv1.RelatedObject{{ + Object: policyv1.ObjectResource{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + Compliant: "NonCompliant", + Reason: "Resource not found but should exist", + }}, + metav1.Condition{ + Type: "DeploymentCompliant", + Status: metav1.ConditionTrue, + Reason: "NoExistingDeployments", + Message: "No existing operator Deployments", + }, + "No existing operator Deployments", + ) + }) }) }) diff --git a/test/resources/case38_operator_install/operator-policy-no-group-csv-fail.yaml b/test/resources/case38_operator_install/operator-policy-no-group-csv-fail.yaml new file mode 100644 index 00000000..71a39488 --- /dev/null +++ b/test/resources/case38_operator_install/operator-policy-no-group-csv-fail.yaml @@ -0,0 +1,21 @@ +apiVersion: policy.open-cluster-management.io/v1beta1 +kind: OperatorPolicy +metadata: + name: oppol-no-allnamespaces + ownerReferences: + - apiVersion: policy.open-cluster-management.io/v1 + kind: Policy + name: parent-policy + uid: 12345678-90ab-cdef-1234-567890abcdef # must be replaced before creation +spec: + remediationAction: enforce + severity: medium + complianceType: musthave + subscription: + channel: singlenamespace-alpha + name: etcd + namespace: operator-policy-testns + installPlanApproval: Automatic + source: operatorhubio-catalog + sourceNamespace: olm + startingCSV: etcdoperator.v0.9.2 diff --git a/test/resources/case38_operator_install/operator-policy-no-group-enforce.yaml b/test/resources/case38_operator_install/operator-policy-no-group-enforce.yaml index 724cae4c..27f547e0 100644 --- a/test/resources/case38_operator_install/operator-policy-no-group-enforce.yaml +++ b/test/resources/case38_operator_install/operator-policy-no-group-enforce.yaml @@ -18,4 +18,4 @@ spec: installPlanApproval: Automatic source: operatorhubio-catalog sourceNamespace: olm - startingCSV: quay-operator.v3.8.1 + startingCSV: quay-operator.v3.8.13