Merge pull request #2417 from jmprusi/2415-when-the-kcpsyncertunnel-f…

…eature-gate-is-enabled-pods-should-be-added-to-the-compute-workspaces-automatically

✨ Add pods resource to the default Kubernetes APIExport

config/rootcompute/kube-1.24/apiexport-kubernetes.k8s.io.yaml

@@ -7,9 +7,10 @@ metadata:
   name: kubernetes
 spec:
   latestResourceSchemas:
-  - v124.ingresses.networking.k8s.io
-  - v124.services.core
-  - v124.deployments.apps
+    - v124.ingresses.networking.k8s.io
+    - v124.services.core
+    - v124.deployments.apps
+    - v124.pods.core
   maximalPermissionPolicy:
     local: {}
 status: {}

config/rootcompute/kube-1.24/bootstrap.go

@@ -33,6 +33,8 @@ var KubeComputeFS embed.FS
 // Bootstrap creates resources in this package by continuously retrying the list.
 // This is blocking, i.e. it only returns (with error) when the context is closed or with nil when
 // the bootstrapping is successfully completed.
+// Note: Any change to the list of resources in the kubernetes apiexport has to be kept consistent with:
+//   - pkg/reconciler/workload/apiexport/workload_apiexport_reconcile.go
 func Bootstrap(ctx context.Context, discoveryClient discovery.DiscoveryInterface, dynamicClient dynamic.Interface, batteriesIncluded sets.String) error {
 	return confighelpers.Bootstrap(ctx, discoveryClient, dynamicClient, batteriesIncluded, KubeComputeFS)
 }

config/rootcompute/kube-1.24/clusterrole-kubernetes-maximal-permission-policy.yaml

@@ -5,24 +5,42 @@ metadata:
     bootstrap.kcp.io/battery: root-compute-workspace
   name: compute:apiexport:kubernetes:maximal-permission-policy
 rules:
-- apiGroups: [""]
-  resources: ["services"]
-  verbs: ["*"]
-- apiGroups: ["apps"]
-  resources: ["deployments"]
-  verbs: ["*"]
-- apiGroups: ["networking.k8s.io"]
-  resources: ["ingresses"]
-  verbs: ["*"]
-- apiGroups: [""]
-  resources: ["services/status"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["apps"]
-  resources: ["deployments/status"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["apps"]
-  resources: ["deployments/scale"]
-  verbs: ["get", "update", "patch"]
-- apiGroups: ["networking.k8s.io"]
-  resources: ["ingresses/status"]
-  verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["services"]
+    verbs: ["*"]
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["*"]
+  - apiGroups: ["networking.k8s.io"]
+    resources: ["ingresses"]
+    verbs: ["*"]
+  - apiGroups: [""]
+    resources: ["services/status"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["apps"]
+    resources: ["deployments/status"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["apps"]
+    resources: ["deployments/scale"]
+    verbs: ["get", "update", "patch"]
+  - apiGroups: ["networking.k8s.io"]
+    resources: ["ingresses/status"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods", "pods/status"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods/exec", "pods/attach", "pods/portforward"]
+    verbs: ["create", "get"]
+  - apiGroups: [""]
+    resources: ["pods/binding"]
+    verbs: ["create"]
+  - apiGroups: [""]
+    resources: ["pods/ephemeralcontainers"]
+    verbs: ["get", "patch", "update"]
+  - apiGroups: [""]
+    resources: ["pods/log"]
+    verbs: ["get"]
+  - apiGroups: [""]
+    resources: ["pods/proxy"]
+    verbs: ["create", "delete", "get", "patch", "update"]

pkg/cliplugins/workload/plugin/sync.go

@@ -806,6 +806,20 @@ func getGroupMappings(resourcesToSync []string) []groupMapping {
 		} else {
 			groupMap[apiGroup] = append(groupMap[apiGroup], name)
 		}
+		// If pods are being synced, add the subresources that are required to
+		// support the pod subresources.
+		if apiGroup == "" && name == "pods" {
+			podSubresources := []string{
+				"pods/log",
+				"pods/exec",
+				"pods/attach",
+				"pods/binding",
+				"pods/portforward",
+				"pods/proxy",
+				"pods/ephemeralcontainers",
+			}
+			groupMap[apiGroup] = append(groupMap[apiGroup], podSubresources...)
+		}
 	}
 
 	groupMappings := make([]groupMapping, 0, len(groupMap))

pkg/reconciler/workload/apiexport/workload_apiexport_reconcile.go

@@ -53,6 +53,7 @@ var rootComputeResourceSchema = sets.NewString(
 	"deployments.apps",
 	"services.core",
 	"ingresses.networking.k8s.io",
+	"pods.core",
 )
 
 type reconciler interface {

pkg/syncer/resourcesync/controller.go

@@ -73,7 +73,7 @@ const (
 // (one for downstream and 2 for upstream, for syncing and upsyncing).
 func NewSyncTargetGVRSource(
 	syncerLogger logr.Logger,
-	upstreamSyncerDiscovery discovery.DiscoveryInterface,
+	downstreamSyncerDiscoveryClient discovery.DiscoveryInterface,
 	upstreamDynamicClusterClient kcpdynamic.ClusterInterface,
 	downstreamDynamicClient dynamic.Interface,
 	downstreamKubeClient kubernetes.Interface,
@@ -84,14 +84,14 @@ func NewSyncTargetGVRSource(
 	syncTargetUID types.UID,
 ) (*controller, error) {
 	c := &controller{
-		queue:                         workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), controllerName),
-		upstreamSyncerDiscoveryClient: memory.NewMemCacheClient(upstreamSyncerDiscovery),
-		downstreamKubeClient:          downstreamKubeClient,
-		syncTargetClient:              syncTargetClient,
-		syncTargetUID:                 syncTargetUID,
-		syncTargetLister:              syncTargetInformer.Lister(),
-		synctargetInformerHasSynced:   syncTargetInformer.Informer().HasSynced,
-		gvrsToWatch:                   map[schema.GroupVersionResource]informer.GVRPartialMetadata{},
+		queue:                           workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), controllerName),
+		downstreamSyncerDiscoveryClient: memory.NewMemCacheClient(downstreamSyncerDiscoveryClient),
+		downstreamKubeClient:            downstreamKubeClient,
+		syncTargetClient:                syncTargetClient,
+		syncTargetUID:                   syncTargetUID,
+		syncTargetLister:                syncTargetInformer.Lister(),
+		synctargetInformerHasSynced:     syncTargetInformer.Informer().HasSynced,
+		gvrsToWatch:                     map[schema.GroupVersionResource]informer.GVRPartialMetadata{},
 	}
 
 	logger := logging.WithReconciler(syncerLogger, controllerName)
@@ -128,7 +128,7 @@ type controller struct {
 	queue                workqueue.RateLimitingInterface
 	downstreamKubeClient kubernetes.Interface
 
-	upstreamSyncerDiscoveryClient discovery.CachedDiscoveryInterface
+	downstreamSyncerDiscoveryClient discovery.CachedDiscoveryInterface
 
 	syncTargetUID               types.UID
 	syncTargetLister            workloadv1alpha1listers.SyncTargetLister
@@ -273,7 +273,7 @@ func (c *controller) process(ctx context.Context, key string) error {
 
 	requiredGVRs := getAllGVRs(syncTarget)
 
-	c.upstreamSyncerDiscoveryClient.Invalidate()
+	c.downstreamSyncerDiscoveryClient.Invalidate()
 
 	var errs []error
 	var unauthorizedGVRs []string
@@ -432,7 +432,7 @@ func (c *controller) addGVR(ctx context.Context, gvr schema.GroupVersionResource
 }
 
 func (c *controller) getGVRPartialMetadata(gvr schema.GroupVersionResource) (*informer.GVRPartialMetadata, error) {
-	apiResourceList, err := c.upstreamSyncerDiscoveryClient.ServerResourcesForGroupVersion(gvr.GroupVersion().String())
+	apiResourceList, err := c.downstreamSyncerDiscoveryClient.ServerResourcesForGroupVersion(gvr.GroupVersion().String())
 	if err != nil {
 		return nil, err
 	}

pkg/syncer/syncer.go

@@ -22,7 +22,6 @@ import (
 	"net/url"
 	"time"
 
-	kcpdiscovery "github.com/kcp-dev/client-go/discovery"
 	kcpdynamic "github.com/kcp-dev/client-go/dynamic"
 	"github.com/kcp-dev/logicalcluster/v3"
 
@@ -33,6 +32,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/dynamic"
 	kubernetesinformers "k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
@@ -198,14 +198,10 @@ func StartSyncer(ctx context.Context, cfg *SyncerConfig, numSyncerThreads int, i
 	// syncerNamespaceInformerFactory to watch some DNS-related resources in the dns namespace
 	syncerNamespaceInformerFactory := kubernetesinformers.NewSharedInformerFactoryWithOptions(downstreamKubeClient, resyncPeriod, kubernetesinformers.WithNamespace(syncerNamespace))
 
-	upstreamSyncerDiscoveryClient, err := kcpdiscovery.NewForConfig(upstreamConfig)
-	if err != nil {
-		return err
-	}
-
+	downstreamSyncerDiscoveryClient := discovery.NewDiscoveryClient(downstreamKubeClient.RESTClient())
 	syncTargetGVRSource, err := resourcesync.NewSyncTargetGVRSource(
 		logger,
-		upstreamSyncerDiscoveryClient.DiscoveryInterface,
+		downstreamSyncerDiscoveryClient,
 		upstreamSyncerClusterClient,
 		downstreamDynamicClient,
 		downstreamKubeClient,
@@ -219,7 +215,18 @@ func StartSyncer(ctx context.Context, cfg *SyncerConfig, numSyncerThreads int, i
 		return err
 	}
 
-	ddsifForUpstreamSyncer, err := ddsif.NewDiscoveringDynamicSharedInformerFactory(upstreamSyncerClusterClient, nil, nil, syncTargetGVRSource, cache.Indexers{})
+	ddsifForUpstreamSyncer, err := ddsif.NewDiscoveringDynamicSharedInformerFactory(upstreamSyncerClusterClient, nil, nil,
+		&filteredGVRSource{
+			GVRSource: syncTargetGVRSource,
+			keepGVR: func(gvr schema.GroupVersionResource) bool {
+				// Don't expose pods or endpoints via the syncer vw
+				if gvr.Group == corev1.GroupName && (gvr.Resource == "pods" || gvr.Resource == "endpoints") {
+					return false
+				}
+				return true
+			},
+		},
+		cache.Indexers{})
 	if err != nil {
 		return err
 	}

pkg/virtual/syncer/builder/build.go

@@ -83,7 +83,13 @@ func BuildVirtualWorkspace(
 				virtualWorkspaceName:  SyncerVirtualWorkspaceName,
 				filteredResourceState: workloadv1alpha1.ResourceStateSync,
 				restProviderBuilder:   NewSyncerRestProvider,
-				allowedAPIFilter:      nil,
+				allowedAPIFilter: func(apiGroupResource schema.GroupResource) bool {
+					// Don't expose Endpoints or Pods via the Syncer VirtualWorkspace.
+					if apiGroupResource.Group == "" && (apiGroupResource.Resource == "pods" || apiGroupResource.Resource == "endpoints") {
+						return false
+					}
+					return true
+				},
 				transformer: &transformations.SyncerResourceTransformer{
 					TransformationProvider:   &transformations.SpecDiffTransformation{},
 					SummarizingRulesProvider: &transformations.DefaultSummarizingRules{},
@@ -99,7 +105,7 @@ func BuildVirtualWorkspace(
 				restProviderBuilder:   NewUpSyncerRestProvider,
 				allowedAPIFilter: func(apiGroupResource schema.GroupResource) bool {
 					// Only allow persistentvolumes and Pods to be Upsynced.
-					return apiGroupResource.Group == "" && apiGroupResource.Resource == "persistentvolumes" || apiGroupResource.Group == "" && apiGroupResource.Resource == "pods"
+					return apiGroupResource.Group == "" && (apiGroupResource.Resource == "persistentvolumes" || apiGroupResource.Resource == "pods")
 				},
 				transformer:           &upsyncer.UpsyncerResourceTransformer{},
 				storageWrapperBuilder: upsyncer.WithStaticLabelSelectorAndInWriteCallsCheck,