test/e2e: create privileged workspaces as shard admin

In order to set the required-groups annotation, we must be the shard
admin. However, by being that powerful, we side-step admission and any
workspaces created with that privilege do not have owner annotations,
which are required for workspace initialization. Therefore, when we use
that privilege we also need to hard-code a workspace owner annotation
ourselves as well.

Signed-off-by: Steve Kuznetsov <skuznets@redhat.com>

pkg/server/config.go

@@ -153,7 +153,7 @@ func (c *Config) Complete() (CompletedConfig, error) {
 	}}, nil
 }
 
-const kcpBootstrapperUserName = "system:kcp:bootstrapper"
+const KcpBootstrapperUserName = "system:kcp:bootstrapper"
 
 func NewConfig(opts *kcpserveroptions.CompletedOptions) (*Config, error) {
 	c := &Config{
@@ -295,7 +295,7 @@ func NewConfig(opts *kcpserveroptions.CompletedOptions) (*Config, error) {
 	}
 
 	bootstrapConfig := rest.CopyConfig(c.GenericConfig.LoopbackClientConfig)
-	bootstrapConfig.Impersonate.UserName = kcpBootstrapperUserName
+	bootstrapConfig.Impersonate.UserName = KcpBootstrapperUserName
 	bootstrapConfig.Impersonate.Groups = []string{bootstrappolicy.SystemKcpWorkspaceBootstrapper}
 	bootstrapConfig = rest.AddUserAgent(bootstrapConfig, "kcp-bootstrapper")
 

pkg/server/controllers.go

@@ -504,7 +504,7 @@ func (s *Server) installWorkspaceScheduler(ctx context.Context, config *rest.Con
 	bootstrapConfig := rest.CopyConfig(config)
 	universalControllerName := fmt.Sprintf("%s-%s", bootstrap.ControllerNameBase, "universal")
 	bootstrapConfig = rest.AddUserAgent(bootstrapConfig, universalControllerName)
-	bootstrapConfig.Impersonate.UserName = kcpBootstrapperUserName
+	bootstrapConfig.Impersonate.UserName = KcpBootstrapperUserName
 	bootstrapConfig.Impersonate.Groups = []string{bootstrappolicy.SystemKcpWorkspaceBootstrapper}
 
 	dynamicClusterClient, err := kcpdynamic.NewForConfig(bootstrapConfig)

test/e2e/authorizer/authorizer_test.go

@@ -69,7 +69,7 @@ func TestAuthorizer(t *testing.T) {
 	require.NoError(t, err)
 
 	org1 := framework.NewOrganizationFixture(t, server)
-	org2 := framework.NewOrganizationFixture(t, server, framework.WithRequiredGroups("empty-group"))
+	org2 := framework.NewPrivilegedOrganizationFixture(t, server, framework.WithRequiredGroups("empty-group"))
 
 	framework.NewWorkspaceFixture(t, server, org1.Path(), framework.WithName("workspace1"))
 	framework.NewWorkspaceFixture(t, server, org1.Path(), framework.WithName("workspace2"))

test/e2e/framework/workspaces.go

@@ -29,45 +29,68 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/authentication/user"
 
+	"github.com/kcp-dev/kcp/pkg/admission/workspace"
 	"github.com/kcp-dev/kcp/pkg/apis/core"
 	corev1alpha1 "github.com/kcp-dev/kcp/pkg/apis/core/v1alpha1"
 	tenancyv1alpha1 "github.com/kcp-dev/kcp/pkg/apis/tenancy/v1alpha1"
 	tenancyv1beta1 "github.com/kcp-dev/kcp/pkg/apis/tenancy/v1beta1"
 	"github.com/kcp-dev/kcp/pkg/authorization"
+	bootstrappolicy "github.com/kcp-dev/kcp/pkg/authorization/bootstrap"
 	kcpclientset "github.com/kcp-dev/kcp/pkg/client/clientset/versioned/cluster"
+	"github.com/kcp-dev/kcp/pkg/server"
 )
 
-type ClusterWorkspaceOption func(ws *tenancyv1beta1.Workspace)
+type ClusterWorkspaceOption interface {
+	PrivilegedClusterWorkspaceOption | UnprivilegedClusterWorkspaceOption
+}
+
+type PrivilegedClusterWorkspaceOption func(ws *tenancyv1beta1.Workspace)
 
-func WithRootShard() ClusterWorkspaceOption {
+type UnprivilegedClusterWorkspaceOption func(ws *tenancyv1beta1.Workspace)
+
+func WithRootShard() UnprivilegedClusterWorkspaceOption {
 	return WithShard(corev1alpha1.RootShard)
 }
 
-func WithShard(name string) ClusterWorkspaceOption {
+func WithShard(name string) UnprivilegedClusterWorkspaceOption {
 	return WithLocation(tenancyv1beta1.WorkspaceLocation{Selector: &metav1.LabelSelector{
 		MatchLabels: map[string]string{
 			"name": name,
 		},
 	}})
 }
 
-func WithLocation(c tenancyv1beta1.WorkspaceLocation) ClusterWorkspaceOption {
+func WithLocation(c tenancyv1beta1.WorkspaceLocation) UnprivilegedClusterWorkspaceOption {
 	return func(ws *tenancyv1beta1.Workspace) {
 		ws.Spec.Location = &c
 	}
 }
 
-func WithRequiredGroups(groups ...string) ClusterWorkspaceOption {
+// WithRequiredGroups is a privileged action, so we return a privileged option type, and only the helpers that
+// use the system:master config can consume this. However, workspace initialization requires a valid user annotation
+// on the workspace object to impersonate during initialization, and system:master bypasses setting that, so we
+// end up needing to hard-code something conceivable.
+func WithRequiredGroups(groups ...string) PrivilegedClusterWorkspaceOption {
 	return func(ws *tenancyv1beta1.Workspace) {
 		if ws.Annotations == nil {
 			ws.Annotations = map[string]string{}
 		}
 		ws.Annotations[authorization.RequiredGroupsAnnotationKey] = strings.Join(groups, ",")
+		userInfo, err := workspace.WorkspaceOwnerAnnotationValue(&user.DefaultInfo{
+			Name:   server.KcpBootstrapperUserName,
+			Groups: []string{user.AllAuthenticated, bootstrappolicy.SystemKcpWorkspaceBootstrapper},
+		})
+		if err != nil {
+			// should never happen
+			panic(err)
+		}
+		ws.Annotations[tenancyv1alpha1.ExperimentalWorkspaceOwnerAnnotationKey] = userInfo
 	}
 }
 
-func WithType(path logicalcluster.Path, name tenancyv1alpha1.WorkspaceTypeName) ClusterWorkspaceOption {
+func WithType(path logicalcluster.Path, name tenancyv1alpha1.WorkspaceTypeName) UnprivilegedClusterWorkspaceOption {
 	return func(ws *tenancyv1beta1.Workspace) {
 		ws.Spec.Type = tenancyv1alpha1.WorkspaceTypeReference{
 			Name: name,
@@ -76,23 +99,19 @@ func WithType(path logicalcluster.Path, name tenancyv1alpha1.WorkspaceTypeName)
 	}
 }
 
-func WithName(s string, formatArgs ...interface{}) ClusterWorkspaceOption {
+func WithName(s string, formatArgs ...interface{}) UnprivilegedClusterWorkspaceOption {
 	return func(ws *tenancyv1beta1.Workspace) {
 		ws.Name = fmt.Sprintf(s, formatArgs...)
 		ws.GenerateName = ""
 	}
 }
 
-func NewWorkspaceFixtureObject(t *testing.T, server RunningServer, parent logicalcluster.Path, options ...ClusterWorkspaceOption) *tenancyv1beta1.Workspace {
+func NewWorkspaceFixtureObject[O ClusterWorkspaceOption](t *testing.T, clusterClient kcpclientset.ClusterInterface, parent logicalcluster.Path, options ...O) *tenancyv1beta1.Workspace {
 	t.Helper()
 
 	ctx, cancelFunc := context.WithCancel(context.Background())
 	t.Cleanup(cancelFunc)
 
-	cfg := server.RootShardSystemMasterBaseConfig(t)
-	clusterClient, err := kcpclientset.NewForConfig(cfg)
-	require.NoError(t, err, "failed to construct client for server")
-
 	tmpl := &tenancyv1beta1.Workspace{
 		ObjectMeta: metav1.ObjectMeta{
 			GenerateName: "e2e-workspace-",
@@ -135,6 +154,7 @@ func NewWorkspaceFixtureObject(t *testing.T, server RunningServer, parent logica
 	})
 
 	Eventually(t, func() (bool, string) {
+		var err error
 		ws, err = clusterClient.Cluster(parent).TenancyV1beta1().Workspaces().Get(ctx, ws.Name, metav1.GetOptions{})
 		require.Falsef(t, apierrors.IsNotFound(err), "workspace %s was deleted", parent.Join(ws.Name))
 		require.NoError(t, err, "failed to get workspace %s", parent.Join(ws.Name))
@@ -145,6 +165,7 @@ func NewWorkspaceFixtureObject(t *testing.T, server RunningServer, parent logica
 	}, wait.ForeverTestTimeout, time.Millisecond*100, "failed to wait for %s workspace %s to become ready", ws.Spec.Type, parent.Join(ws.Name))
 
 	Eventually(t, func() (bool, string) {
+		var err error
 		_, err = clusterClient.Cluster(parent.Join(ws.Name)).CoreV1alpha1().LogicalClusters().Get(ctx, corev1alpha1.LogicalClusterName, metav1.GetOptions{})
 		require.Falsef(t, apierrors.IsNotFound(err), "workspace %s was deleted", parent.Join(ws.Name))
 		require.NoError(t, err, "failed to get LogicalCluster %s", parent.Join(ws.Name).Join(corev1alpha1.LogicalClusterName))
@@ -155,19 +176,40 @@ func NewWorkspaceFixtureObject(t *testing.T, server RunningServer, parent logica
 	return ws
 }
 
-func NewWorkspaceFixture(t *testing.T, server RunningServer, orgClusterName logicalcluster.Path, options ...ClusterWorkspaceOption) (clusterName logicalcluster.Name) {
+func NewWorkspaceFixture(t *testing.T, server RunningServer, orgClusterName logicalcluster.Path, options ...UnprivilegedClusterWorkspaceOption) (clusterName logicalcluster.Name) {
 	t.Helper()
-	ws := NewWorkspaceFixtureObject(t, server, orgClusterName, options...)
+
+	cfg := server.BaseConfig(t)
+	clusterClient, err := kcpclientset.NewForConfig(cfg)
+	require.NoError(t, err, "failed to construct client for server")
+
+	ws := NewWorkspaceFixtureObject(t, clusterClient, orgClusterName, options...)
 	return logicalcluster.Name(ws.Spec.Cluster)
 }
 
-func NewOrganizationFixtureObject(t *testing.T, server RunningServer, options ...ClusterWorkspaceOption) *tenancyv1beta1.Workspace {
+func NewOrganizationFixtureObject(t *testing.T, server RunningServer, options ...UnprivilegedClusterWorkspaceOption) *tenancyv1beta1.Workspace {
 	t.Helper()
-	return NewWorkspaceFixtureObject(t, server, core.RootCluster.Path(), append(options, WithType(core.RootCluster.Path(), "organization"))...)
+
+	cfg := server.BaseConfig(t)
+	clusterClient, err := kcpclientset.NewForConfig(cfg)
+	require.NoError(t, err, "failed to construct client for server")
+
+	return NewWorkspaceFixtureObject(t, clusterClient, core.RootCluster.Path(), append(options, WithType(core.RootCluster.Path(), "organization"))...)
 }
 
-func NewOrganizationFixture(t *testing.T, server RunningServer, options ...ClusterWorkspaceOption) (orgClusterName logicalcluster.Name) {
+func NewOrganizationFixture(t *testing.T, server RunningServer, options ...UnprivilegedClusterWorkspaceOption) (orgClusterName logicalcluster.Name) {
 	t.Helper()
 	org := NewOrganizationFixtureObject(t, server, options...)
 	return logicalcluster.Name(org.Spec.Cluster)
 }
+
+func NewPrivilegedOrganizationFixture[O ClusterWorkspaceOption](t *testing.T, server RunningServer, options ...O) (orgClusterName logicalcluster.Name) {
+	t.Helper()
+
+	cfg := server.RootShardSystemMasterBaseConfig(t)
+	clusterClient, err := kcpclientset.NewForConfig(cfg)
+	require.NoError(t, err, "failed to construct client for server")
+
+	org := NewWorkspaceFixtureObject(t, clusterClient, core.RootCluster.Path(), append(options, O(WithType(core.RootCluster.Path(), "organization")))...)
+	return logicalcluster.Name(org.Spec.Cluster)
+}

test/e2e/reconciler/clusterworkspace/apibinding_initializer_test.go

@@ -46,10 +46,13 @@ func TestWorkspaceTypesAPIBindingInitialization(t *testing.T) {
 
 	org := framework.NewOrganizationFixtureObject(t, server)
 	orgPath := core.RootCluster.Path().Join(org.Name)
-	cowboysProviderWorkspace := framework.NewWorkspaceFixtureObject(t, server, orgPath, framework.WithName("cowboys-provider"))
-	cowboysProviderPath := orgPath.Join(cowboysProviderWorkspace.Name)
 
 	cfg := server.BaseConfig(t)
+	clusterClient, err := kcpclientset.NewForConfig(cfg)
+	require.NoError(t, err, "failed to construct client for server")
+
+	cowboysProviderWorkspace := framework.NewWorkspaceFixtureObject(t, clusterClient, orgPath, framework.WithName("cowboys-provider"))
+	cowboysProviderPath := orgPath.Join(cowboysProviderWorkspace.Name)
 
 	kcpClusterClient, err := kcpclientset.NewForConfig(cfg)
 	require.NoError(t, err, "error creating kcp cluster client")
@@ -85,7 +88,7 @@ func TestWorkspaceTypesAPIBindingInitialization(t *testing.T) {
 	cowboysAPIExport, err = kcpClusterClient.Cluster(cowboysProviderPath).ApisV1alpha1().APIExports().Create(ctx, cowboysAPIExport, metav1.CreateOptions{})
 	require.NoError(t, err, "error creating APIExport")
 
-	universal := framework.NewWorkspaceFixtureObject(t, server, orgPath, framework.WithName("universal"))
+	universal := framework.NewWorkspaceFixtureObject(t, clusterClient, orgPath, framework.WithName("universal"))
 	universalPath := orgPath.Join(universal.Name)
 
 	cwtParent1 := &tenancyv1alpha1.WorkspaceType{

test/e2e/reconciler/workspacedeletion/controller_test.go

@@ -192,7 +192,7 @@ func TestWorkspaceDeletion(t *testing.T) {
 				t.Helper()
 
 				org := framework.NewOrganizationFixtureObject(t, server, framework.WithRootShard())
-				orgClusterName := logicalcluster.Name(org.Status.Cluster)
+				orgClusterName := logicalcluster.Name(org.Spec.Cluster)
 
 				t.Logf("Should have finalizer in org workspace")
 				require.Eventually(t, func() bool {
@@ -201,9 +201,13 @@ func TestWorkspaceDeletion(t *testing.T) {
 					return len(orgWorkspace.Finalizers) > 0
 				}, wait.ForeverTestTimeout, 100*time.Millisecond)
 
+				cfg := server.RunningServer.BaseConfig(t)
+				clusterClient, err := kcpclientset.NewForConfig(cfg)
+				require.NoError(t, err, "failed to construct client for server")
+
 				t.Logf("Create a workspace with in the org workspace")
-				ws := framework.NewWorkspaceFixtureObject(t, server.RunningServer, orgClusterName.Path(), framework.WithName("org-ws-cleanup"), framework.WithRootShard())
-				wsClusterName := logicalcluster.Name(ws.Status.Cluster)
+				ws := framework.NewWorkspaceFixtureObject(t, clusterClient, orgClusterName.Path(), framework.WithName("org-ws-cleanup"), framework.WithRootShard())
+				wsClusterName := logicalcluster.Name(ws.Spec.Cluster)
 
 				t.Logf("Should have finalizer added in workspace")
 				require.Eventually(t, func() bool {
@@ -219,7 +223,7 @@ func TestWorkspaceDeletion(t *testing.T) {
 					},
 				}
 
-				_, err := server.kubeClusterClient.Cluster(wsClusterName.Path()).CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
+				_, err = server.kubeClusterClient.Cluster(wsClusterName.Path()).CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
 				require.NoError(t, err, "failed to create ns in workspace root:%s", ws)
 
 				_, err = server.kubeClusterClient.Cluster(orgClusterName.Path()).CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})