Skip to content

Commit

Permalink
split cleanup process to postpone crd deletion after all resources ge…
Browse files Browse the repository at this point in the history
…t deleted
  • Loading branch information
ruanxin committed Feb 21, 2024
1 parent 9631d2e commit 631dfc5
Show file tree
Hide file tree
Showing 3 changed files with 139 additions and 90 deletions.
65 changes: 0 additions & 65 deletions internal/declarative/v2/cleanup.go

This file was deleted.

42 changes: 17 additions & 25 deletions internal/declarative/v2/reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ import (
"sigs.k8s.io/controller-runtime/pkg/manager"

"github.com/kyma-project/lifecycle-manager/api/shared"
"github.com/kyma-project/lifecycle-manager/api/v1beta2"
"github.com/kyma-project/lifecycle-manager/internal/pkg/metrics"
"github.com/kyma-project/lifecycle-manager/internal/pkg/resources"
"github.com/kyma-project/lifecycle-manager/pkg/common"
"github.com/kyma-project/lifecycle-manager/pkg/queue"
"github.com/kyma-project/lifecycle-manager/pkg/util"
Expand Down Expand Up @@ -155,8 +157,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
return r.ssaStatus(ctx, obj, metrics.ManifestRenderResources)
}

diff := ResourceList(current).Difference(target)
if err := r.pruneDiff(ctx, clnt, obj, diff, spec); errors.Is(err, ErrDeletionNotFinished) {
if err := r.pruneDiff(ctx, clnt, obj, current, target, spec); errors.Is(err, resources.ErrDeletionNotFinished) {
r.Metrics.RecordRequeueReason(metrics.ManifestPruneDiffNotFinished, queue.IntendedRequeue)
return ctrl.Result{Requeue: true}, nil
} else if err != nil {
Expand Down Expand Up @@ -311,7 +312,9 @@ func (r *Reconciler) renderResources(
return target, current, nil
}

func (r *Reconciler) syncResources(ctx context.Context, clnt Client, obj Object, target []*resource.Info) error {
func (r *Reconciler) syncResources(ctx context.Context, clnt Client, obj Object,
target []*resource.Info,
) error {
status := obj.GetStatus()

if err := ConcurrentSSA(clnt, r.FieldOwner).Run(ctx, target); err != nil {
Expand Down Expand Up @@ -406,23 +409,6 @@ func generateOperationMessage(installationCondition apimetav1.Condition, stateIn
return installationCondition.Message
}

func (r *Reconciler) deleteDiffResources(
ctx context.Context, clnt Client, obj Object, diff []*resource.Info,
) error {
status := obj.GetStatus()

if err := NewConcurrentCleanup(clnt).Run(ctx, diff); errors.Is(err, ErrDeletionNotFinished) {
r.Event(obj, "Normal", "Deletion", err.Error())
return err
} else if err != nil {
r.Event(obj, "Warning", "Deletion", err.Error())
obj.SetStatus(status.WithState(shared.StateError).WithErr(err))
return err
}

return nil
}

func (r *Reconciler) doPreDelete(ctx context.Context, clnt Client, obj Object) error {
if !obj.GetDeletionTimestamp().IsZero() {
for _, preDelete := range r.PreDeletes {
Expand Down Expand Up @@ -488,11 +474,10 @@ func (r *Reconciler) pruneDiff(
ctx context.Context,
clnt Client,
obj Object,
diff []*resource.Info,
current, target []*resource.Info,
spec *Spec,
) error {
var err error
diff, err = pruneResource(diff, "Namespace", namespaceNotBeRemoved)
diff, err := pruneResource(ResourceList(current).Difference(target), "Namespace", namespaceNotBeRemoved)
if err != nil {
return err
}
Expand All @@ -507,14 +492,21 @@ func (r *Reconciler) pruneDiff(
return ErrResourceSyncDiffInSameOCILayer
}

if err := r.deleteDiffResources(ctx, clnt, obj, diff); err != nil {
// Remove this type casting while in progress this issue: https://github.com/kyma-project/lifecycle-manager/issues/1006
manifest, ok := obj.(*v1beta2.Manifest)
if !ok {
return v1beta2.ErrTypeAssertManifest
}
if err := resources.NewConcurrentCleanup(clnt, manifest).DeleteDiffResources(ctx, diff); err != nil {
return err
}

return nil
}

func manifestNotInDeletingAndOciRefNotChangedButDiffDetected(diff []*resource.Info, obj Object, spec *Spec) bool {
func manifestNotInDeletingAndOciRefNotChangedButDiffDetected(diff []*resource.Info, obj Object,
spec *Spec,
) bool {
return len(diff) > 0 && ociRefNotChanged(obj, spec.OCIRef) && obj.GetDeletionTimestamp().IsZero()
}

Expand Down
122 changes: 122 additions & 0 deletions internal/pkg/resources/cleanup.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
package resources

import (
"context"
"errors"

apimetav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/cli-runtime/pkg/resource"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/kyma-project/lifecycle-manager/api/shared"
"github.com/kyma-project/lifecycle-manager/api/v1beta2"
"github.com/kyma-project/lifecycle-manager/pkg/common"
"github.com/kyma-project/lifecycle-manager/pkg/util"
)

var ErrDeletionNotFinished = errors.New("deletion is not yet finished")

type ConcurrentCleanup struct {
clnt client.Client
policy client.PropagationPolicy
manifest *v1beta2.Manifest
}

func NewConcurrentCleanup(clnt client.Client, manifest *v1beta2.Manifest) *ConcurrentCleanup {
return &ConcurrentCleanup{
clnt: clnt,
policy: client.PropagationPolicy(apimetav1.DeletePropagationBackground),
manifest: manifest,
}
}

func (c *ConcurrentCleanup) DeleteDiffResources(ctx context.Context, resources []*resource.Info,
) error {
status := c.manifest.GetStatus()
excludeCRDList, pureCRDList, err := splitResources(resources)
if err != nil {
return err
}

if err := c.CleanupResources(ctx, excludeCRDList, status); err != nil {
return err
}

if err := c.CleanupResources(ctx, pureCRDList, status); err != nil {
return err
}

return nil
}

func (c *ConcurrentCleanup) CleanupResources(
ctx context.Context,
resources []*resource.Info,
status shared.Status,
) error {
if err := c.Run(ctx, resources); errors.Is(err, ErrDeletionNotFinished) {
return err
} else if err != nil {
c.manifest.SetStatus(status.WithState(shared.StateError).WithErr(err))
return err
}
return nil
}

func splitResources(resources []*resource.Info) ([]*resource.Info, []*resource.Info, error) {
excludeCRDList := make([]*resource.Info, 0)
pureCRDList := make([]*resource.Info, 0)

for _, resource := range resources {
obj, ok := resource.Object.(client.Object)
if !ok {
return nil, nil, common.ErrTypeAssert
}
if obj.GetObjectKind().GroupVersionKind().Kind == "CustomResourceDefinition" {
pureCRDList = append(pureCRDList, resource)
continue
}
excludeCRDList = append(excludeCRDList, resource)
}

return excludeCRDList, pureCRDList, nil
}

func (c *ConcurrentCleanup) Run(ctx context.Context, infos []*resource.Info) error {
// The Runtime Complexity of this Branch is N as only ServerSideApplier Patch is required
results := make(chan error, len(infos))
for i := range infos {
i := i
go c.cleanupResource(ctx, infos[i], results)
}

var errs []error
present := len(infos)
for i := 0; i < len(infos); i++ {
err := <-results
if util.IsNotFound(err) {
present--
continue
}
if err != nil {
errs = append(errs, err)
}
}

if len(errs) > 0 {
return errors.Join(errs...)
}

if present > 0 {
return ErrDeletionNotFinished
}
return nil
}

func (c *ConcurrentCleanup) cleanupResource(ctx context.Context, info *resource.Info, results chan error) {
obj, ok := info.Object.(client.Object)
if !ok {
return
}
results <- c.clnt.Delete(ctx, obj, c.policy)
}

0 comments on commit 631dfc5

Please sign in to comment.