Curated packages PBC full cluster lifecycle integration

This change introduces the creation and deletion of a namespaced package
bundle controller for workload clusters created via the full cluster
lifecycle.

This change is dependent on a change in the eks-anywhere-packages repo to
grant permissions to the eksa-controller-manager service account.

aws/eks-anywhere-packages#833

Without the change above, the reconciliation will log errors removing some
resources, which can be left behind.

Future commits will add packages installation/deletion via full cluster
lifecycle.

Makefile

@@ -568,7 +568,7 @@ mocks: ## Generate mocks
 	${MOCKGEN} -destination=pkg/curatedpackages/mocks/bundlemanager.go -package=mocks -source "pkg/curatedpackages/bundlemanager.go" Manager
 	${MOCKGEN} -destination=pkg/clients/kubernetes/mocks/kubectl.go -package=mocks -source "pkg/clients/kubernetes/unauth.go"
 	${MOCKGEN} -destination=pkg/clients/kubernetes/mocks/kubeconfig.go -package=mocks -source "pkg/clients/kubernetes/kubeconfig.go"
-	${MOCKGEN} -destination=pkg/curatedpackages/mocks/installer.go -package=mocks -source "pkg/curatedpackages/packagecontrollerclient.go" ChartInstaller
+	${MOCKGEN} -destination=pkg/curatedpackages/mocks/installer.go -package=mocks -source "pkg/curatedpackages/packagecontrollerclient.go" ChartInstaller ChartInstallationDeleter
 	${MOCKGEN} -destination=pkg/cluster/mocks/client_builder.go -package=mocks -source "pkg/cluster/client_builder.go"
 	${MOCKGEN} -destination=controllers/mocks/factory.go -package=mocks "github.com/aws/eks-anywhere/controllers" Manager
 	${MOCKGEN} -destination=pkg/networking/cilium/reconciler/mocks/templater.go -package=mocks -source "pkg/networking/cilium/reconciler/reconciler.go"
@@ -578,7 +578,7 @@ mocks: ## Generate mocks
 	${MOCKGEN} -destination=pkg/providers/docker/reconciler/mocks/reconciler.go -package=mocks -source "pkg/providers/docker/reconciler/reconciler.go"
 	${MOCKGEN} -destination=pkg/providers/tinkerbell/reconciler/mocks/reconciler.go -package=mocks -source "pkg/providers/tinkerbell/reconciler/reconciler.go"
 	${MOCKGEN} -destination=pkg/awsiamauth/reconciler/mocks/reconciler.go -package=mocks -source "pkg/awsiamauth/reconciler/reconciler.go"
-	${MOCKGEN} -destination=controllers/mocks/cluster_controller.go -package=mocks -source "controllers/cluster_controller.go" AWSIamConfigReconciler ClusterValidator
+	${MOCKGEN} -destination=controllers/mocks/cluster_controller.go -package=mocks -source "controllers/cluster_controller.go" AWSIamConfigReconciler ClusterValidator PackageControllerClient
 	${MOCKGEN} -destination=pkg/workflow/task_mock_test.go -package=workflow_test -source "pkg/workflow/task.go"
 	${MOCKGEN} -destination=pkg/validations/createcluster/mocks/createcluster.go -package=mocks -source "pkg/validations/createcluster/createcluster.go"
 	${MOCKGEN} -destination=pkg/awsiamauth/mock_test.go -package=awsiamauth_test -source "pkg/awsiamauth/installer.go"

config/manifest/eksa-components.yaml

@@ -5901,6 +5901,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - packages.eks.amazonaws.com
+  resources:
+  - packagebundlecontrollers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - tinkerbell.org
   resources:

config/rbac/role.yaml

@@ -210,6 +210,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - packages.eks.amazonaws.com
+  resources:
+  - packagebundlecontrollers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - tinkerbell.org
   resources:

controllers/cluster_controller.go

@@ -3,10 +3,13 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"os"
 	"time"
 
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -27,7 +30,10 @@ import (
 	"github.com/aws/eks-anywhere/pkg/controller/clientutil"
 	"github.com/aws/eks-anywhere/pkg/controller/clusters"
 	"github.com/aws/eks-anywhere/pkg/controller/handlers"
+	"github.com/aws/eks-anywhere/pkg/curatedpackages"
+	"github.com/aws/eks-anywhere/pkg/registrymirror"
 	"github.com/aws/eks-anywhere/pkg/utils/ptr"
+	"github.com/aws/eks-anywhere/release/api/v1alpha1"
 )
 
 const (
@@ -42,6 +48,7 @@ type ClusterReconciler struct {
 	providerReconcilerRegistry ProviderClusterReconcilerRegistry
 	awsIamAuth                 AWSIamConfigReconciler
 	clusterValidator           ClusterValidator
+	packageControllerClient    PackageControllerClient
 }
 
 type ProviderClusterReconcilerRegistry interface {
@@ -61,12 +68,13 @@ type ClusterValidator interface {
 }
 
 // NewClusterReconciler constructs a new ClusterReconciler.
-func NewClusterReconciler(client client.Client, registry ProviderClusterReconcilerRegistry, awsIamAuth AWSIamConfigReconciler, clusterValidator ClusterValidator) *ClusterReconciler {
+func NewClusterReconciler(client client.Client, registry ProviderClusterReconcilerRegistry, awsIamAuth AWSIamConfigReconciler, clusterValidator ClusterValidator, packageControllerClient PackageControllerClient) *ClusterReconciler {
 	return &ClusterReconciler{
 		client:                     client,
 		providerReconcilerRegistry: registry,
 		awsIamAuth:                 awsIamAuth,
 		clusterValidator:           clusterValidator,
+		packageControllerClient:    packageControllerClient,
 	}
 }
 
@@ -262,9 +270,99 @@ func (r *ClusterReconciler) postClusterProviderReconcile(ctx context.Context, lo
 		}
 	}
 
+	if !cluster.IsSelfManaged() {
+		if err := r.postReconcilePackagesForWorkloadCluster(ctx, log, cluster); err != nil {
+			return controller.Result{}, err
+		}
+	}
+
 	return controller.Result{}, nil
 }
 
+type PackageControllerClient interface {
+	EnableCuratedPackagesFullLifecycle(context.Context, logr.Logger, string, string, *v1alpha1.Image, *registrymirror.RegistryMirror, ...curatedpackages.PackageControllerClientOpt) error
+	ReconcileDelete(context.Context, logr.Logger, client.Client, *anywherev1.Cluster) error
+}
+
+var _ PackageControllerClient = (*curatedpackages.PackageControllerClient)(nil)
+
+func (r *ClusterReconciler) postReconcilePackagesForWorkloadCluster(ctx context.Context, log logr.Logger, cluster *anywherev1.Cluster) (err error) {
+	bundles := &v1alpha1.Bundles{}
+	nn := types.NamespacedName{
+		Name:      cluster.Spec.BundlesRef.Name,
+		Namespace: cluster.Spec.BundlesRef.Namespace,
+	}
+	if err := r.client.Get(ctx, nn, bundles); err != nil {
+		return err
+	}
+
+	verBundle, err := r.findMatchingBundle(bundles, string(cluster.Spec.KubernetesVersion))
+	if err != nil {
+		return err
+	}
+
+	image, ok := verBundle.Charts()["eks-anywhere-packages"]
+	if !ok {
+		return fmt.Errorf("no chart image")
+	}
+
+	kubeConfigNN := types.NamespacedName{
+		Namespace: constants.EksaSystemNamespace,
+		Name:      cluster.Name + "-kubeconfig",
+	}
+	kubeConfigSecret := &corev1.Secret{}
+	if err := r.client.Get(ctx, kubeConfigNN, kubeConfigSecret); err != nil {
+		return fmt.Errorf("getting kubeconfig secret: %w", err)
+	}
+	secretBytes, err := yaml.Marshal(kubeConfigSecret)
+	if err != nil {
+		return fmt.Errorf("marshaling secret %w", err)
+	}
+	f, err := os.CreateTemp("", "kubeconfig-*.yaml")
+	if err != nil {
+		return fmt.Errorf("opening kubeconfig file %w", err)
+	}
+	defer f.Close()
+	// TODO unlink
+
+	if _, err := f.Write(secretBytes); err != nil {
+		return fmt.Errorf("writing kubeconfig file %w", err)
+	}
+	f.Close()
+
+	rm := registrymirror.FromCluster(cluster)
+	var options []curatedpackages.PackageControllerClientOpt
+	err = r.packageControllerClient.EnableCuratedPackagesFullLifecycle(ctx, log,
+		cluster.Name,
+		f.Name(),
+		image,
+		rm,
+		options...,
+	)
+	if err != nil {
+		return fmt.Errorf("package controller client error: %w", err)
+	}
+
+	log.V(6).Info("Installed curated packages on workload cluster", "cluster", cluster.Name)
+
+	return nil
+}
+
+func (r *ClusterReconciler) findMatchingBundle(bundles *v1alpha1.Bundles, kubeVersion string) (*v1alpha1.VersionsBundle, error) {
+	var verBundle *v1alpha1.VersionsBundle
+	for _, b := range bundles.Spec.VersionsBundles {
+		if b.KubeVersion == string(kubeVersion) {
+			verBundle = &b
+			break
+		}
+	}
+	if verBundle == nil {
+		return nil, fmt.Errorf("no bundle for kube version %q", kubeVersion)
+	}
+
+	return verBundle, nil
+}
+
 func (r *ClusterReconciler) reconcileDelete(ctx context.Context, log logr.Logger, cluster *anywherev1.Cluster) (ctrl.Result, error) {
 	if cluster.IsSelfManaged() {
 		return ctrl.Result{}, errors.New("deleting self-managed clusters is not supported")
@@ -310,6 +408,17 @@ func (r *ClusterReconciler) reconcileDelete(ctx context.Context, log logr.Logger
 		}
 	}
 
+	if !cluster.IsSelfManaged() {
+		// TODO all those tests that pass a nil package controller client will bork here, as the packageControllerClient will be nil.
+		if r.packageControllerClient == nil {
+			return ctrl.Result{}, nil
+			// return ctrl.Result{}, fmt.Errorf("controller has a nil package controller client, cannot delete package bundle controller")
+		}
+		if err := r.packageControllerClient.ReconcileDelete(ctx, log, r.client, cluster); err != nil {
+			return ctrl.Result{}, fmt.Errorf("deleting packages controller for cluster %q %w", cluster.Name, err)
+		}
+	}
+
 	return ctrl.Result{}, nil
 }
 

controllers/cluster_controller_legacy.go

@@ -60,6 +60,7 @@ func NewClusterReconcilerLegacy(client client.Client, log logr.Logger, scheme *r
 // +kubebuilder:rbac:groups="",namespace=eksa-system,resources=secrets,verbs=delete;
 // +kubebuilder:rbac:groups=tinkerbell.org,resources=hardware;hardware/status,verbs=get;list;watch;update;patch
 // +kubebuilder:rbac:groups=bmc.tinkerbell.org,resources=machines;machines/status,verbs=get;list;watch
+// +kubebuilder:rbac:groups=packages.eks.amazonaws.com,resources=packagebundlecontrollers,verbs=get;list;watch;create;update;patch;delete
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.

controllers/cluster_controller_test.go

@@ -68,7 +68,7 @@ func newVsphereClusterReconcilerTest(t *testing.T, objs ...runtime.Object) *vsph
 		Add(anywherev1.VSphereDatacenterKind, reconciler).
 		Build()
 
-	r := controllers.NewClusterReconciler(cl, &registry, iam, clusterValidator)
+	r := controllers.NewClusterReconciler(cl, &registry, iam, clusterValidator, nil)
 
 	return &vsphereClusterReconcilerTest{
 		govcClient: govcClient,
@@ -101,9 +101,9 @@ func TestClusterReconcilerReconcileSelfManagedCluster(t *testing.T) {
 
 	providerReconciler.EXPECT().ReconcileWorkerNodes(ctx, gomock.AssignableToTypeOf(logr.Logger{}), sameName(selfManagedCluster))
 
-	r := controllers.NewClusterReconciler(c, registry, iam, clusterValidator)
+	r := controllers.NewClusterReconciler(c, registry, iam, clusterValidator, nil)
 	result, err := r.Reconcile(ctx, clusterRequest(selfManagedCluster))
-	g.Expect(err).ToNot(HaveOccurred())
+	g.Expect(err).NotTo(HaveOccurred())
 	g.Expect(result).To(Equal(ctrl.Result{}))
 }
 
@@ -128,7 +128,7 @@ func TestClusterReconcilerReconcilePausedCluster(t *testing.T) {
 	iam := mocks.NewMockAWSIamConfigReconciler(ctrl)
 	clusterValidator := mocks.NewMockClusterValidator(ctrl)
 	registry := newRegistryMock(providerReconciler)
-	r := controllers.NewClusterReconciler(c, registry, iam, clusterValidator)
+	r := controllers.NewClusterReconciler(c, registry, iam, clusterValidator, nil)
 	g.Expect(r.Reconcile(ctx, clusterRequest(cluster))).To(Equal(reconcile.Result{}))
 	api := envtest.NewAPIExpecter(t, c)
 
@@ -164,7 +164,7 @@ func TestClusterReconcilerReconcileDeletedSelfManagedCluster(t *testing.T) {
 	registry := newRegistryMock(providerReconciler)
 	c := fake.NewClientBuilder().WithRuntimeObjects(selfManagedCluster).Build()
 
-	r := controllers.NewClusterReconciler(c, registry, iam, clusterValidator)
+	r := controllers.NewClusterReconciler(c, registry, iam, clusterValidator, nil)
 	_, err := r.Reconcile(ctx, clusterRequest(selfManagedCluster))
 	g.Expect(err).To(MatchError(ContainSubstring("deleting self-managed clusters is not supported")))
 }
@@ -233,7 +233,7 @@ func TestClusterReconcilerReconcileDeletePausedCluster(t *testing.T) {
 		managementCluster, cluster, capiCluster,
 	).Build()
 
-	r := controllers.NewClusterReconciler(c, newRegistryForDummyProviderReconciler(), iam, clusterValidator)
+	r := controllers.NewClusterReconciler(c, newRegistryForDummyProviderReconciler(), iam, clusterValidator, nil)
 	g.Expect(r.Reconcile(ctx, clusterRequest(cluster))).To(Equal(reconcile.Result{}))
 	api := envtest.NewAPIExpecter(t, c)
 
@@ -276,7 +276,7 @@ func TestClusterReconcilerReconcileDeleteClusterManagedByCLI(t *testing.T) {
 	iam := mocks.NewMockAWSIamConfigReconciler(controller)
 	clusterValidator := mocks.NewMockClusterValidator(controller)
 
-	r := controllers.NewClusterReconciler(c, newRegistryForDummyProviderReconciler(), iam, clusterValidator)
+	r := controllers.NewClusterReconciler(c, newRegistryForDummyProviderReconciler(), iam, clusterValidator, nil)
 	g.Expect(r.Reconcile(ctx, clusterRequest(cluster))).To(Equal(reconcile.Result{}))
 	api := envtest.NewAPIExpecter(t, c)