Do apply in stages and fix replicas conflicts

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>

.github/workflows/e2e.yaml

@@ -67,13 +67,13 @@ jobs:
           kubectl get crd tests.testing.kustomizer.dev 2>&1 | grep NotFound
       - name: Load test apply (110 objects)
         run: |
-          ./bin/kustomizer apply -k ./testdata/loadtest/ --inventory-name load-test
+          ./bin/kustomizer apply -i load-test -k ./testdata/loadtest/
       - name: Load test delete (110 objects)
         run: |
-          ./bin/kustomizer delete --inventory-name load-test
+          ./bin/kustomizer delete -i load-test
       - name: Test staged apply
         run: |
-          ./bin/kustomizer apply -k ./testdata/certs/ --prune --wait --inventory-name cert-test --mode=ApplyAllStaged
+          ./bin/kustomizer apply -i cert-test -k ./testdata/certs/ --prune --wait
           kubectl -n kustomizer-cert-test wait issuers/my-ca-issuer --for=condition=ready --timeout=1m
       - name: Debug failure
         if: failure()

Makefile

@@ -15,7 +15,7 @@ build:
 	CGO_ENABLED=0 go build -o ./bin/kustomizer ./cmd/kustomizer
 
 install:
-	go install cmd/kustomizer
+	go install ./cmd/kustomizer
 
 install-dev:
 	CGO_ENABLED=0 go build -o /usr/local/bin ./cmd/kustomizer

cmd/kustomizer/apply.go

@@ -19,11 +19,14 @@ package main
 import (
 	"context"
 	"fmt"
+	"sort"
 	"time"
 
 	"github.com/spf13/cobra"
 	"github.com/stefanprodan/kustomizer/pkg/inventory"
 	"github.com/stefanprodan/kustomizer/pkg/manager"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"sigs.k8s.io/cli-utils/pkg/ordering"
 )
 
 var applyCmd = &cobra.Command{
@@ -40,7 +43,6 @@ type applyFlags struct {
 	wait               bool
 	force              bool
 	prune              bool
-	mode               string
 }
 
 var applyArgs applyFlags
@@ -56,8 +58,7 @@ func init() {
 	applyCmd.Flags().StringVarP(&applyArgs.inventoryName, "inventory-name", "i", "", "The name of the inventory configmap.")
 	applyCmd.Flags().StringVar(&applyArgs.inventoryNamespace, "inventory-namespace", "default",
 		"The namespace of the inventory configmap. The namespace must exist on the target cluster.")
-	applyCmd.Flags().StringVar(&applyArgs.mode, "mode", "Apply",
-		"The ResourceManager apply method, can be `Apply`, `ApplyAll`, `ApplyAllStaged`.")
+
 	rootCmd.AddCommand(applyCmd)
 }
 
@@ -84,6 +85,10 @@ func runApplyCmd(cmd *cobra.Command, args []string) error {
 	}
 	logger.Println(fmt.Sprintf("applying %v manifest(s)...", len(objects)))
 
+	for _, object := range objects {
+		fixReplicasConflict(object, objects)
+	}
+
 	kubeClient, err := newKubeClient(rootArgs.kubeconfig, rootArgs.kubecontext)
 	if err != nil {
 		return fmt.Errorf("client init failed: %w", err)
@@ -101,33 +106,41 @@ func runApplyCmd(cmd *cobra.Command, args []string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), rootArgs.timeout)
 	defer cancel()
 
-	switch applyArgs.mode {
-	case "Apply":
-		for _, object := range objects {
-			change, err := resMgr.Apply(ctx, object, applyArgs.force)
-			if err != nil {
-				return err
-			}
-			logger.Println(change.String())
+	// contains only CRDs and Namespaces
+	var stageOne []*unstructured.Unstructured
+
+	// contains all objects except for CRDs and Namespaces
+	var stageTwo []*unstructured.Unstructured
+
+	for _, u := range objects {
+		if resMgr.IsClusterDefinition(u.GetKind()) {
+			stageOne = append(stageOne, u)
+		} else {
+			stageTwo = append(stageTwo, u)
 		}
-	case "ApplyAll":
-		changeSet, err := resMgr.ApplyAll(ctx, objects, applyArgs.force)
+	}
+
+	if len(stageOne) > 0 {
+		changeSet, err := resMgr.ApplyAll(ctx, stageOne, applyArgs.force)
 		if err != nil {
 			return err
 		}
 		for _, change := range changeSet.Entries {
 			logger.Println(change.String())
 		}
-	case "ApplyAllStaged":
-		changeSet, err := resMgr.ApplyAllStaged(ctx, objects, applyArgs.force, 30*time.Second)
-		if err != nil {
+
+		if err := resMgr.Wait(stageOne, 2*time.Second, 30*time.Second); err != nil {
 			return err
 		}
-		for _, change := range changeSet.Entries {
-			logger.Println(change.String())
+	}
+
+	sort.Sort(ordering.SortableUnstructureds(stageTwo))
+	for _, object := range stageTwo {
+		change, err := resMgr.Apply(ctx, object, applyArgs.force)
+		if err != nil {
+			return err
 		}
-	default:
-		return fmt.Errorf("mode not supported")
+		logger.Println(change.String())
 	}
 
 	staleObjects, err := resMgr.GetInventoryStaleObjects(ctx, newInventory)
@@ -170,3 +183,18 @@ func runApplyCmd(cmd *cobra.Command, args []string) error {
 
 	return nil
 }
+
+// fixReplicasConflict removes the replicas field from the given workload if it's managed by an HPA
+func fixReplicasConflict(object *unstructured.Unstructured, objects []*unstructured.Unstructured) {
+	for _, hpa := range objects {
+		if hpa.GetKind() == "HorizontalPodAutoscaler" && object.GetNamespace() == hpa.GetNamespace() {
+			targetKind, found, err := unstructured.NestedFieldCopy(hpa.Object, "spec", "scaleTargetRef", "kind")
+			if err == nil && found && fmt.Sprintf("%v", targetKind) == object.GetKind() {
+				targetName, found, err := unstructured.NestedFieldCopy(hpa.Object, "spec", "scaleTargetRef", "name")
+				if err == nil && found && fmt.Sprintf("%v", targetName) == object.GetName() {
+					unstructured.RemoveNestedField(object.Object, "spec", "replicas")
+				}
+			}
+		}
+	}
+}

pkg/manager/manager_apply.go

@@ -123,11 +123,11 @@ func (m *ResourceManager) ApplyAllStaged(ctx context.Context, objects []*unstruc
 	// contains only CRDs and Namespaces
 	var stageOne []*unstructured.Unstructured
 
-	// contains all objects except for  CRDs and Namespaces
+	// contains all objects except for CRDs and Namespaces
 	var stageTwo []*unstructured.Unstructured
 
 	for _, u := range objects {
-		if m.isClusterDefinition(u.GetKind()) {
+		if m.IsClusterDefinition(u.GetKind()) {
 			stageOne = append(stageOne, u)
 		} else {
 			stageTwo = append(stageTwo, u)
@@ -172,7 +172,7 @@ func (m *ResourceManager) apply(ctx context.Context, object *unstructured.Unstru
 	return m.client.Patch(ctx, object, client.Apply, opts...)
 }
 
-func (m *ResourceManager) isClusterDefinition(kind string) bool {
+func (m *ResourceManager) IsClusterDefinition(kind string) bool {
 	switch strings.ToLower(kind) {
 	case "customresourcedefinition":
 	case "namespace":

pkg/manager/manager_inventory.go

@@ -90,7 +90,7 @@ func (m *ResourceManager) DeleteInventory(ctx context.Context, i *inventory.Inve
 	return nil
 }
 
-// GetInventoryStaleObjects returns the list of objects subject to pruning.
+// GetInventoryStaleObjects returns the list of objects metadata subject to pruning.
 func (m *ResourceManager) GetInventoryStaleObjects(ctx context.Context, i *inventory.Inventory) ([]*unstructured.Unstructured, error) {
 	objects := make([]*unstructured.Unstructured, 0)
 	existingInventory := inventory.NewInventory(i.Name, i.Namespace)

pkg/objectutil/io.go

@@ -68,7 +68,7 @@ func ReadObjects(r io.Reader) ([]*unstructured.Unstructured, error) {
 			continue
 		}
 
-		if IsKubernetesObject(obj) {
+		if IsKubernetesObject(obj) && !IsKustomization(obj) {
 			objects = append(objects, obj)
 		}
 	}
@@ -83,6 +83,26 @@ func IsKubernetesObject(object *unstructured.Unstructured) bool {
 	return true
 }
 
+func IsKustomization(object *unstructured.Unstructured) bool {
+	if object.GetKind() == "Kustomization" && object.GroupVersionKind().GroupKind().Group == "kustomize.config.k8s.io" {
+		return true
+	}
+	return false
+}
+
+// ObjectToYAML encodes the given Kubernetes API object to YAML.
+func ObjectToYAML(object *unstructured.Unstructured) string {
+	var builder strings.Builder
+	data, err := yaml.Marshal(object)
+	if err != nil {
+		return ""
+	}
+	builder.Write(data)
+	builder.WriteString("---\n")
+
+	return builder.String()
+}
+
 // ObjectsToYAML encodes the given Kubernetes API objects to a YAML multi-doc.
 func ObjectsToYAML(objects []*unstructured.Unstructured) (string, error) {
 	var builder strings.Builder

testdata/plain/backend/deployment.yaml

@@ -4,6 +4,7 @@ metadata:
   name: backend
   namespace: kustomizer-demo
 spec:
+  replicas: 1
   minReadySeconds: 3
   revisionHistoryLimit: 5
   progressDeadlineSeconds: 60

testdata/plain/backend/hpa.yaml

@@ -8,8 +8,8 @@ spec:
     apiVersion: apps/v1
     kind: Deployment
     name: backend
-  minReplicas: 1
-  maxReplicas: 2
+  minReplicas: 2
+  maxReplicas: 4
   metrics:
   - type: Resource
     resource: