wip on Get k8s resources using kapp

Signed-off-by: Antonio Gamez Diaz <agamez@vmware.com>

cmd/kubeapps-apis/plugins/kapp_controller/packages/v1alpha1/server.go

@@ -16,6 +16,11 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/cppforlife/go-cli-ui/ui"
+	ctlapp "github.com/k14s/kapp/pkg/kapp/app"
+	kappcmdapp "github.com/k14s/kapp/pkg/kapp/cmd/app"
+	kappcmdcore "github.com/k14s/kapp/pkg/kapp/cmd/core"
+	"github.com/k14s/kapp/pkg/kapp/logger"
 	"github.com/kubeapps/kubeapps/cmd/kubeapps-apis/core"
 	corev1 "github.com/kubeapps/kubeapps/cmd/kubeapps-apis/gen/core/packages/v1alpha1"
 	"github.com/kubeapps/kubeapps/cmd/kubeapps-apis/gen/plugins/kapp_controller/packages/v1alpha1"
@@ -26,6 +31,7 @@ import (
 )
 
 type clientGetter func(context.Context, string) (kubernetes.Interface, dynamic.Interface, error)
+type kappFactoryGetter func(ctx context.Context, cluster string, appFlags kappcmdapp.Flags) (ctlapp.App, kappcmdapp.FactorySupportObjs, error)
 
 const (
 	globalPackagingNamespace = "kapp-controller-packaging-global"
@@ -43,6 +49,7 @@ type Server struct {
 	clientGetter             clientGetter
 	globalPackagingNamespace string
 	globalPackagingCluster   string
+	kappFactoryGetter        kappFactoryGetter
 }
 
 // NewServer returns a Server automatically configured with a function to obtain
@@ -69,6 +76,30 @@ func NewServer(configGetter core.KubernetesConfigGetter, globalPackagingCluster
 		},
 		globalPackagingNamespace: globalPackagingNamespace,
 		globalPackagingCluster:   globalPackagingCluster,
+		kappFactoryGetter: func(ctx context.Context, cluster string, appFlags kappcmdapp.Flags) (ctlapp.App, kappcmdapp.FactorySupportObjs, error) {
+			if configGetter == nil {
+				return nil, kappcmdapp.FactorySupportObjs{}, status.Errorf(codes.Internal, "configGetter arg required")
+			}
+			config, err := configGetter(ctx, cluster)
+			if err != nil {
+				return nil, kappcmdapp.FactorySupportObjs{}, status.Errorf(codes.FailedPrecondition, "unable to get config due to: %v", err)
+			}
+			configFactory := NewConfigurableConfigFactoryImpl()
+			configFactory.ConfigureRESTConfig(config)
+
+			resourceTypesFlags := kappcmdapp.ResourceTypesFlags{
+				IgnoreFailingAPIServices:         true,
+				ScopeToFallbackAllowedNamespaces: true,
+			}
+			depsFactory := kappcmdcore.NewDepsFactoryImpl(configFactory, ui.NewNoopUI())
+
+			app, supportObjs, err := kappcmdapp.Factory(depsFactory, appFlags, resourceTypesFlags, logger.NewNoopLogger())
+			if err != nil {
+				return nil, kappcmdapp.FactorySupportObjs{}, err
+			}
+
+			return app, supportObjs, nil
+		},
 	}
 }
 
@@ -83,3 +114,15 @@ func (s *Server) GetClients(ctx context.Context, cluster string) (kubernetes.Int
 	}
 	return typedClient, dynamicClient, nil
 }
+
+// GetKappFactory ensures a client getter is available and uses it to return a Kapp Factory.
+func (s *Server) GetKappFactory(ctx context.Context, cluster string, appFlags kappcmdapp.Flags) (ctlapp.App, kappcmdapp.FactorySupportObjs, error) {
+	if s.clientGetter == nil {
+		return nil, kappcmdapp.FactorySupportObjs{}, status.Errorf(codes.Internal, "server not configured with configGetter")
+	}
+	app, supportObjs, err := s.kappFactoryGetter(ctx, cluster, appFlags)
+	if err != nil {
+		return nil, kappcmdapp.FactorySupportObjs{}, status.Errorf(codes.FailedPrecondition, fmt.Sprintf("unable to get Kapp Factory : %v", err))
+	}
+	return app, supportObjs, nil
+}

cmd/kubeapps-apis/plugins/kapp_controller/packages/v1alpha1/server_ctrl_packages.go

@@ -735,90 +735,10 @@ func (s *Server) GetInstalledPackageResourceRefs(ctx context.Context, request *c
 		cluster = s.globalPackagingCluster
 	}
 
-	typedClient, _, err := s.GetClients(ctx, cluster)
+	// get the list of every k8s resource matching ResourceRef
+	refs, err := s.findKappK8sResources(ctx, cluster, namespace, installedPackageRefId)
 	if err != nil {
-		return nil, status.Errorf(codes.Internal, "unable to get the k8s client: '%v'", err)
-	}
-
-	refs := []*corev1.ResourceRef{}
-
-	// TODO(agamez): apparently, App CRs not being created by a "kapp deploy"
-	// don't have the proper annotations. So, in order to retrieve the annotation value,
-	// we have to get the ConfigMap <AppName>-ctrl and, then, fetch the
-	// vaulue of the key "labelValue" in data.spec.
-	// See https://kubernetes.slack.com/archives/CH8KCCKA5/p1637842398026700
-	// https://github.com/vmware-tanzu/carvel-kapp-controller/issues/430
-
-	// the ConfigMap name is, by convention, "<appname>-ctrl", but it will change in the near future
-	cmName := fmt.Sprintf("%s-ctrl", installedPackageRefId)
-	cm, err := typedClient.CoreV1().ConfigMaps(namespace).Get(ctx, cmName, metav1.GetOptions{})
-	if err == nil && cm.Data["spec"] != "" {
-
-		appLabelValue := extractValue(cm.Data["spec"], "labelValue")
-		appLabelSelector := fmt.Sprintf("%s=%s", appLabelKey, appLabelValue)
-		listOptions := metav1.ListOptions{LabelSelector: appLabelSelector}
-
-		// TODO(agamez): perform an actual query over all the resources available in the cluster
-		// this is currently just a PoC getting the bare minimum: pods, deployments, services and secrets.
-		// Also, the xxx.Items[i] are not populating the Kind and APIVersion fields. Check why.
-
-		// Fetching all the matching pods
-		pods, err := typedClient.CoreV1().Pods(namespace).List(ctx, listOptions)
-		if err != nil {
-			return nil, statuserror.FromK8sError("get", "Pods", "", err)
-		}
-		for _, resource := range pods.Items {
-			refs = append(refs, &corev1.ResourceRef{
-				ApiVersion: "core/v1",
-				Kind:       "Pod",
-				Name:       resource.Name,
-				Namespace:  resource.Namespace,
-			})
-		}
-
-		// Fetching all the matching deployments
-		deployments, err := typedClient.AppsV1().Deployments(namespace).List(ctx, listOptions)
-		if err != nil {
-			return nil, statuserror.FromK8sError("get", "Deployments", "", err)
-		}
-		for _, resource := range deployments.Items {
-			refs = append(refs, &corev1.ResourceRef{
-				ApiVersion: "apps/v1",
-				Kind:       "Deployment",
-				Name:       resource.ObjectMeta.Name,
-				Namespace:  resource.ObjectMeta.Namespace,
-			})
-		}
-
-		// Fetching all the matching services
-		services, err := typedClient.CoreV1().Services(namespace).List(ctx, listOptions)
-		if err != nil {
-			return nil, statuserror.FromK8sError("get", "services", "", err)
-		}
-		for _, resource := range services.Items {
-			refs = append(refs, &corev1.ResourceRef{
-				ApiVersion: "core/v1",
-				Kind:       "Service",
-				Name:       resource.ObjectMeta.Name,
-				Namespace:  resource.ObjectMeta.Namespace,
-			})
-		}
-
-		// Fetching all the matching secrets
-		secrets, err := typedClient.CoreV1().Secrets(namespace).List(ctx, listOptions)
-		if err != nil {
-			return nil, statuserror.FromK8sError("get", "Secrets", "", err)
-		}
-		for _, resource := range secrets.Items {
-			refs = append(refs, &corev1.ResourceRef{
-				ApiVersion: "core/v1",
-				Kind:       "Secret",
-				Name:       resource.ObjectMeta.Name,
-				Namespace:  resource.ObjectMeta.Namespace,
-			})
-		}
-	} else {
-		log.Warning(statuserror.FromK8sError("get", "ConfigMap", cmName, err))
+		return nil, err
 	}
 
 	return &corev1.GetInstalledPackageResourceRefsResponse{

cmd/kubeapps-apis/plugins/kapp_controller/packages/v1alpha1/server_data_resources.go

@@ -15,6 +15,10 @@ package main
 import (
 	"context"
 
+	kappcmdapp "github.com/k14s/kapp/pkg/kapp/cmd/app"
+	kappcmdcore "github.com/k14s/kapp/pkg/kapp/cmd/core"
+	kappcmdtools "github.com/k14s/kapp/pkg/kapp/cmd/tools"
+	corev1 "github.com/kubeapps/kubeapps/cmd/kubeapps-apis/gen/core/packages/v1alpha1"
 	kappctrlv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1"
 	packagingv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/packaging/v1alpha1"
 	datapackagingv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/apis/datapackaging/v1alpha1"
@@ -399,3 +403,60 @@ func (s *Server) updatePkgInstall(ctx context.Context, cluster, namespace string
 	}
 	return &pkgInstall, nil
 }
+
+// findKappK8sResources returns the list of k8s resources matching the given listOptions
+func (s *Server) findKappK8sResources(ctx context.Context, cluster, namespace, packageId string) ([]*corev1.ResourceRef, error) {
+	refs := []*corev1.ResourceRef{}
+
+	appFlags := kappcmdapp.Flags{
+		NamespaceFlags: kappcmdcore.NamespaceFlags{
+			Name: namespace,
+		},
+		Name: packageId + "-ctrl",
+	}
+
+	app, supportObjs, err := s.GetKappFactory(ctx, cluster, appFlags)
+	if err != nil {
+		return nil, err
+	}
+
+	usedGVs, err := app.UsedGVs()
+	if err != nil {
+		return nil, err
+	}
+
+	resourceTypesFlags := kappcmdapp.ResourceTypesFlags{
+		IgnoreFailingAPIServices:         true,
+		ScopeToFallbackAllowedNamespaces: true,
+	}
+	failingAPIServicesPolicy := resourceTypesFlags.FailingAPIServicePolicy()
+	failingAPIServicesPolicy.MarkRequiredGVs(usedGVs)
+
+	labelSelector, err := app.LabelSelector()
+	if err != nil {
+		return nil, err
+	}
+
+	resources, err := supportObjs.IdentifiedResources.List(labelSelector, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resourceFilterFlags := kappcmdtools.ResourceFilterFlags{}
+	resourceFilter, err := resourceFilterFlags.ResourceFilter()
+	if err != nil {
+		return nil, err
+	}
+
+	resources = resourceFilter.Apply(resources)
+
+	for _, resource := range resources {
+		refs = append(refs, &corev1.ResourceRef{
+			ApiVersion: resource.GroupVersion().String(),
+			Kind:       resource.Kind(),
+			Name:       resource.Name(),
+			Namespace:  resource.Namespace(),
+		})
+	}
+	return refs, nil
+}

cmd/kubeapps-apis/plugins/kapp_controller/packages/v1alpha1/server_utils.go

@@ -22,13 +22,15 @@ import (
 	"strings"
 
 	"github.com/Masterminds/semver/v3"
+	kappcmdcore "github.com/k14s/kapp/pkg/kapp/cmd/core"
 	corev1 "github.com/kubeapps/kubeapps/cmd/kubeapps-apis/gen/core/packages/v1alpha1"
 	kappctrlv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1"
 	datapackagingv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/apis/datapackaging/v1alpha1"
 	"gopkg.in/yaml.v3"
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
 	structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/rest"
 )
 
 type pkgSemver struct {
@@ -256,3 +258,24 @@ func isNonNullableNull(x interface{}, s *structuralschema.Structural) bool {
 func isKindInt(src interface{}) bool {
 	return src != nil && reflect.TypeOf(src).Kind() == reflect.Int
 }
+
+// implementing a custom ConfigFactory to allow for customizing the *rest.Config
+// https://kubernetes.slack.com/archives/CH8KCCKA5/p1642015047046200
+type ConfigurableConfigFactoryImpl struct {
+	kappcmdcore.ConfigFactoryImpl
+	config *rest.Config
+}
+
+var _ kappcmdcore.ConfigFactory = &ConfigurableConfigFactoryImpl{}
+
+func NewConfigurableConfigFactoryImpl() *ConfigurableConfigFactoryImpl {
+	return &ConfigurableConfigFactoryImpl{}
+}
+
+func (f *ConfigurableConfigFactoryImpl) ConfigureRESTConfig(config *rest.Config) {
+	f.config = config
+}
+
+func (f *ConfigurableConfigFactoryImpl) RESTConfig() (*rest.Config, error) {
+	return f.config, nil
+}