Add operand cleanup feature

This commit introduces the operand cleanup feature which enables a
CSV to be configured so the OLM operator can clean up CRs on operator
uninstall or CSV deletion.

Adds the logic for opt-in/opt-out for operand cleanup and adds a finalizer
that runs through the cleanup steps for removing CRs and updating the status.

Author: hasbro17

go.mod

@@ -64,6 +64,10 @@ replace (
 	github.com/openshift/api => github.com/openshift/api v0.0.0-20200331152225-585af27e34fd // release-4.5
 	github.com/openshift/client-go => github.com/openshift/client-go v0.0.0-20200326155132-2a6cd50aedd0 // release-4.5
 
+	// TODO: Remove when operator-framework/api has the cleanup API in a tagged release
+	// pinned for using APIs from feature branch "add-csv-cleanup-api"
+	github.com/operator-framework/api => github.com/hasbro17/api v0.3.13-0.20210304212710-93ba7a75566c
+
 	// pinned because latest etcd does not yet work with the latest grpc version (1.30.0)
 	go.etcd.io/etcd => go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f
 	google.golang.org/grpc => google.golang.org/grpc v1.27.0

go.sum

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"reflect"
 	"strings"
 	"time"
 
@@ -13,6 +14,7 @@ import (
 	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	extinf "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -50,6 +52,16 @@ import (
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/metrics"
 )
 
+const (
+	CleanupFinalizer = "operatorframework.io/cleanup-apis"
+	// MaxCRListSize is used to limit the number of CRs displayed
+	// on the status.cleanup.pendingDeletion block
+	// This prevents the CSV size from exceeding limits in the event of
+	// a large number of CRs being cleaned up.
+	// TODO: Should this be configurable?
+	MaxCRListSize = 10
+)
+
 var (
 	ErrRequirementsNotMet      = errors.New("requirements were not met")
 	ErrCRDOwnerConflict        = errors.New("conflicting CRD owner in namespace")
@@ -1062,6 +1074,189 @@ func (a *Operator) deleteChild(csv *v1alpha1.ClusterServiceVersion, logger *logr
 	return a.client.OperatorsV1alpha1().ClusterServiceVersions(csv.GetNamespace()).Delete(context.TODO(), csv.GetName(), *metav1.NewDeleteOptions(0))
 }
 
+// updateCleanupFinalizer will set or clear the CSV's cleanup finalizer based on the cleanup spec
+// It also unsets the finalizer when the CSV is in phase=Replacing to prevent CSV deletion from
+// trigerring cleanup during an upgrade.
+// Returns the updated CSV or nil if no update needed
+func (a *Operator) updateCleanupFinalizer(inCSV *v1alpha1.ClusterServiceVersion) *v1alpha1.ClusterServiceVersion {
+	outCSV := inCSV.DeepCopy()
+	hasFinalizer := inCSV.HasFinalizer(CleanupFinalizer)
+
+	// If being replaced, remove finalizer and opt-out to prevent cleanup of CRs
+	// when this replacing CSV gets deleted
+	if hasFinalizer && inCSV.Status.Phase == v1alpha1.CSVPhaseReplacing {
+		outCSV.RemoveFinalizer(CleanupFinalizer)
+		// We forcefully opt-out of cleanup by updating spec.cleanup to prevent the cleanup finalizer
+		// from being attached again on the next reconcile.
+		// TODO: Having the controller update the spec is an anti-pattern. So we could avoid updating the spec
+		// and always treat a CSV with phase=Replacing as spec.cleanup.enabled=false.
+		outCSV.Spec.Cleanup.Enabled = false
+		return outCSV
+	}
+
+	// No update if cleanup enabled with finalizer already present, or disabled with no finalizer
+	if inCSV.Spec.Cleanup.Enabled && hasFinalizer || !inCSV.Spec.Cleanup.Enabled && !hasFinalizer {
+		return nil
+	}
+
+	if inCSV.Spec.Cleanup.Enabled && !hasFinalizer {
+		// Add finalizer if missing
+		outCSV.ObjectMeta.Finalizers = append(inCSV.ObjectMeta.Finalizers, CleanupFinalizer)
+	} else if !inCSV.Spec.Cleanup.Enabled && hasFinalizer {
+		// Remove finalizer if not needed
+		outCSV.RemoveFinalizer(CleanupFinalizer)
+	}
+	return outCSV
+}
+
+// parseResourceGroup parses a "resource.group" string into "resource" and "group"
+// Returns an error if a malformed string results in an empty resource or group
+// TODO: Move to some util pkg
+func parseResourceGroup(name string) (resourcePlural string, group string, err error) {
+	rg := strings.SplitN(name, ".", 2)
+	if len(rg) != 2 {
+		err = fmt.Errorf("error parsing CSV name %s: should be of the format 'resource.group'", name)
+		return
+	}
+	if len(rg[0]) == 0 || len(rg[1]) == 0 {
+		err = fmt.Errorf("error parsing CSV name %s: resource(%s) and group(%s) cannot be empty", name, rg[0], rg[1])
+		return
+	}
+	resourcePlural = rg[0]
+	group = rg[1]
+	return
+}
+
+// getCRNamespaces returns the list of target namespaces to look at when cleaning up CRs
+// A cluster-scoped CRD returns a single item list that represents no namespace: [""]
+// A namespace-scoped CRD results in a single/multi/all namespaces list e.g [ns1, ns2, ...]
+func (a *Operator) getCRNamespaces(crdName string, csv *v1alpha1.ClusterServiceVersion) ([]string, error) {
+	var crNamespaces []string
+
+	crd, err := a.opClient.ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), crdName, metav1.GetOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("error getting CRD %s: %v", crdName, err)
+	}
+
+	// CRD is cluster-scoped
+	if crd.Spec.Scope == apiextensionsv1.ClusterScoped {
+		// We only return the namespace "" for which the CR client uses cluster-scoped requests
+		crNamespaces = append(crNamespaces, "")
+		return crNamespaces, nil
+	}
+
+	// Parse the namespaces list from the CSV's target namespaces annotation
+	targetNamespacesSet := resolver.NewNamespaceSetFromString(csv.Annotations[v1.OperatorGroupTargetsAnnotationKey])
+	if !targetNamespacesSet.IsAllNamespaces() {
+		for ns := range targetNamespacesSet {
+			crNamespaces = append(crNamespaces, ns)
+		}
+		return crNamespaces, nil
+	}
+
+	// All namespaces means olm.targetNamespaces=""
+	// Translate that into the acutal list of all namespaces
+	allNamespaces, err := a.lister.CoreV1().NamespaceLister().List(labels.Everything())
+	if err != nil {
+		return nil, fmt.Errorf("error listing all namespaces: %v", err)
+	}
+
+	for _, ns := range allNamespaces {
+		crNamespaces = append(crNamespaces, ns.Name)
+	}
+
+	return crNamespaces, nil
+}
+
+// runCleanupFinalizer runs the process of cleaning up CRs for the operator
+// and is called when a CSV is pending deletion on the cleanup finalizer.
+// The finalizer is cleared once cleanup finishes or there is an opt-out of cleanup.
+// Returns the CSV if there is an update to the status or finalizer, and nil if unchanged.
+func (a *Operator) runCleanupFinalizer(inCSV *v1alpha1.ClusterServiceVersion) (*v1alpha1.ClusterServiceVersion, error) {
+	outCSV := inCSV.DeepCopy()
+	removeCleanupFinalizer := true
+
+	// For each owned CRD, list and delete all CRs managed by the operator in the operator's target namespaces
+	pendingDeletion := []v1alpha1.ResourceList{}
+	for _, ownedCRD := range inCSV.Spec.CustomResourceDefinitions.Owned {
+		resourcePlural, group, err := parseResourceGroup(ownedCRD.Name)
+		if err != nil {
+			return nil, err
+		}
+
+		rl := v1alpha1.ResourceList{
+			Group:     group,
+			Version:   ownedCRD.Version,
+			Kind:      ownedCRD.Kind,
+			Instances: []v1alpha1.NamespacedName{},
+		}
+
+		// TODO: Add a GVK string method e.g ResourceList.GVK()
+		gvk := rl.Group + "/" + rl.Version + " " + rl.Kind
+
+		// Get the list of target namespaces to look at for cleaning up CRs
+		// This translates olm.targetNamespaces="" to the concrete list of all namespaces
+		// For cluster-scoped CRDs we get [""] which is handled by the CR client as cluster-scoped
+		crNamespaces, err := a.getCRNamespaces(ownedCRD.Name, inCSV)
+		if err != nil {
+			return nil, fmt.Errorf("failed to execute finalizer: error getting CR namespaces: %v", err)
+		}
+
+		for _, ns := range crNamespaces {
+			crList, err := a.opClient.ListCustomResource(group, rl.Version, ns, resourcePlural)
+			if err != nil {
+				return nil, fmt.Errorf("failed to execute finalizer: error listing CRs for type %s: %v", gvk, err)
+			}
+
+			for _, cr := range crList.Items {
+				// Delete the CR if it isn't already pending deletion
+				if cr.GetDeletionTimestamp().IsZero() {
+					err := a.opClient.DeleteCustomResource(group, rl.Version, ns, resourcePlural, cr.GetName())
+					if err != nil && !k8serrors.IsNotFound(err) {
+						return nil, fmt.Errorf("failed to execute finalizer: error deleting CR %s/%s of type %s: %v", cr.GetNamespace(), cr.GetName(), gvk, err)
+					}
+				}
+
+				// We only append a list of N CRs per GVK to display in the cleanup status block: status.cleanup.pendingDeletion
+				// This to prevent the CSV object size from exceeding the etcd enforced limit when there are a significantly large
+				// number of CRs present.
+				// DEBUG: Won't the previous ListCustomResource() operation fail for a sufficiently large number of CRs anyway?
+				if len(rl.Instances) < MaxCRListSize {
+					rl.Instances = append(rl.Instances, v1alpha1.NamespacedName{Name: cr.GetName(), Namespace: cr.GetNamespace()})
+				}
+			}
+
+			// Keep the cleanup finalizer if there is any CR still pending deletion
+			if len(crList.Items) != 0 {
+				removeCleanupFinalizer = false
+			}
+
+		}
+
+		pendingDeletion = append(pendingDeletion, rl)
+	}
+
+	// Clear the cleanup finalizer and status cleanup block if all CRs are deleted
+	if removeCleanupFinalizer {
+		outCSV.RemoveFinalizer(CleanupFinalizer)
+		outCSV.Status.Cleanup.PendingDeletion = []v1alpha1.ResourceList{}
+		return outCSV, nil
+	}
+
+	// TODO: Check if we need to add the cleanup status condition if we haven't already done so?
+
+	// Check if we need to update the cleanup status
+	// DEBUG: Are the instance arrays always ordered the same? If not, this could cause perpetual updates.
+	// Is the CR list order from List() and the existing status block always the same?
+	outCSV.Status.Cleanup.PendingDeletion = pendingDeletion
+	if reflect.DeepEqual(inCSV.Status.Cleanup.PendingDeletion, outCSV.Status.Cleanup.PendingDeletion) {
+		return nil, nil
+	}
+
+	// Return for status update
+	return outCSV, nil
+}
+
 // syncClusterServiceVersion is the method that gets called when we see a CSV event in the cluster
 func (a *Operator) syncClusterServiceVersion(obj interface{}) (syncError error) {
 	clusterServiceVersion, ok := obj.(*v1alpha1.ClusterServiceVersion)
@@ -1090,6 +1285,56 @@ func (a *Operator) syncClusterServiceVersion(obj interface{}) (syncError error)
 		return
 	}
 
+	// Add or remove the cleanup finalizer based on the cleanup spec
+	// This can also abort an in progress cleanup and unblock CSV deletion
+	// If the CSV is being replaced, it will also be opted out of cleanup
+	if outCSV := a.updateCleanupFinalizer(clusterServiceVersion); outCSV != nil {
+		_, err := a.client.OperatorsV1alpha1().ClusterServiceVersions(outCSV.GetNamespace()).Update(context.TODO(), outCSV, metav1.UpdateOptions{})
+		if err != nil {
+			syncError = fmt.Errorf("failed to update cleanup finalizer: %v", err)
+		}
+		return
+	}
+
+	// Check if the CSV is pending deletion
+	if !clusterServiceVersion.ObjectMeta.DeletionTimestamp.IsZero() {
+		if !clusterServiceVersion.HasFinalizer(CleanupFinalizer) {
+			// Stop reconciliation as the deleted CSV is not pending on the cleanup finalizer
+			return
+		}
+
+		// CSV deletion is blocked on the cleanup finalizer
+		outCSV, err := a.runCleanupFinalizer(clusterServiceVersion)
+		if err != nil {
+			syncError = fmt.Errorf("failed to run cleanup finalizer: %v", err)
+			return
+		}
+		if outCSV == nil {
+			// Still awaiting cleanup and nothing to update on the CSV status
+			// TODO: We'll want to requeue again after some time so we can check on the progress of cleanup
+			// When the CRs we're waiting for finish deletion we won't see a CSV reconcile event
+			// so the CSV finalizer is kept for longer than it needs to be.
+			return
+		}
+
+		// Update the CSV if the cleanup finalizer has been removed
+		if !outCSV.HasFinalizer(CleanupFinalizer) {
+			_, err := a.client.OperatorsV1alpha1().ClusterServiceVersions(outCSV.GetNamespace()).Update(context.TODO(), outCSV, metav1.UpdateOptions{})
+			if err != nil {
+				syncError = fmt.Errorf("error updating ClusterServiceVersion: %v ", err)
+				return
+			}
+			return
+		}
+
+		// Otherwise update the cleanup status
+		_, err = a.client.OperatorsV1alpha1().ClusterServiceVersions(outCSV.GetNamespace()).UpdateStatus(context.TODO(), outCSV, metav1.UpdateOptions{})
+		if err != nil {
+			syncError = fmt.Errorf("error updating ClusterServiceVersion status: %v ", err)
+		}
+		return
+	}
+
 	outCSV, syncError := a.transitionCSVState(*clusterServiceVersion)
 
 	if outCSV == nil {

pkg/controller/operators/olm/operator.go

@@ -36,17 +36,17 @@ type ClientInterface interface {
 
 // CustomResourceClient contains methods for the Custom Resource.
 type CustomResourceClient interface {
-	GetCustomResource(apiGroup, version, namespace, resourceKind, resourceName string) (*unstructured.Unstructured, error)
-	GetCustomResourceRaw(apiGroup, version, namespace, resourceKind, resourceName string) ([]byte, error)
+	GetCustomResource(apiGroup, version, namespace, resourcePlural, resourceName string) (*unstructured.Unstructured, error)
+	GetCustomResourceRaw(apiGroup, version, namespace, resourcePlural, resourceName string) ([]byte, error)
 	CreateCustomResource(item *unstructured.Unstructured) error
 	CreateCustomResourceRaw(apiGroup, version, namespace, kind string, data []byte) error
 	CreateCustomResourceRawIfNotFound(apiGroup, version, namespace, kind, name string, data []byte) (bool, error)
 	UpdateCustomResource(item *unstructured.Unstructured) error
-	UpdateCustomResourceRaw(apiGroup, version, namespace, resourceKind, resourceName string, data []byte) error
-	CreateOrUpdateCustomeResourceRaw(apiGroup, version, namespace, resourceKind, resourceName string, data []byte) error
-	DeleteCustomResource(apiGroup, version, namespace, resourceKind, resourceName string) error
-	AtomicModifyCustomResource(apiGroup, version, namespace, resourceKind, resourceName string, f CustomResourceModifier, data interface{}) error
-	ListCustomResource(apiGroup, version, namespace, resourceKind string) (*CustomResourceList, error)
+	UpdateCustomResourceRaw(apiGroup, version, namespace, resourcePlural, resourceName string, data []byte) error
+	CreateOrUpdateCustomeResourceRaw(apiGroup, version, namespace, resourcePlural, resourceName string, data []byte) error
+	DeleteCustomResource(apiGroup, version, namespace, resourcePlural, resourceName string) error
+	AtomicModifyCustomResource(apiGroup, version, namespace, resourcePlural, resourceName string, f CustomResourceModifier, data interface{}) error
+	ListCustomResource(apiGroup, version, namespace, resourcePlural string) (*CustomResourceList, error)
 }
 
 // APIServiceClient contains methods for manipulating APIServiceBindings.