From 9ce775d782918e83f816e204054a2e40efc1dada Mon Sep 17 00:00:00 2001 From: Adem Baccara <71262172+Adembc@users.noreply.github.com> Date: Mon, 12 Aug 2024 16:54:06 +0100 Subject: [PATCH 1/8] add validating webhook Signed-off-by: Adem Baccara <71262172+Adembc@users.noreply.github.com> --- workspaces/controller/Makefile | 2 +- workspaces/controller/PROJECT | 6 + .../api/v1beta1/webhook_suite_test.go | 404 ++++++++++++++++++ .../api/v1beta1/workspace_webhook.go | 78 ++++ .../api/v1beta1/workspace_webhook_test.go | 39 ++ .../api/v1beta1/workspacekind_webhook.go | 315 ++++++++++++++ .../api/v1beta1/workspacekind_webhook_test.go | 77 ++++ .../api/v1beta1/zz_generated.deepcopy.go | 2 +- workspaces/controller/cmd/main.go | 12 + .../config/certmanager/certificate.yaml | 39 ++ .../config/certmanager/kustomization.yaml | 5 + .../config/certmanager/kustomizeconfig.yaml | 8 + .../controller/config/crd/kustomization.yaml | 4 +- .../cainjection_in_workspacekinds.yaml | 7 + .../patches/cainjection_in_workspaces.yaml | 7 + .../patches/webhook_in_workspacekinds.yaml | 16 + .../crd/patches/webhook_in_workspaces.yaml | 16 + .../config/default/kustomization.yaml | 186 ++++---- .../config/default/manager_webhook_patch.yaml | 23 + .../default/webhookcainjection_patch.yaml | 15 + .../config/webhook/kustomization.yaml | 6 + .../config/webhook/kustomizeconfig.yaml | 22 + .../controller/config/webhook/manifests.yaml | 46 ++ .../controller/config/webhook/service.yaml | 15 + workspaces/controller/go.mod | 2 +- 25 files changed, 1246 insertions(+), 106 deletions(-) create mode 100644 workspaces/controller/api/v1beta1/webhook_suite_test.go create mode 100644 workspaces/controller/api/v1beta1/workspace_webhook.go create mode 100644 workspaces/controller/api/v1beta1/workspace_webhook_test.go create mode 100644 workspaces/controller/api/v1beta1/workspacekind_webhook.go create mode 100644 workspaces/controller/api/v1beta1/workspacekind_webhook_test.go create mode 100644 workspaces/controller/config/certmanager/certificate.yaml create mode 100644 workspaces/controller/config/certmanager/kustomization.yaml create mode 100644 workspaces/controller/config/certmanager/kustomizeconfig.yaml create mode 100644 workspaces/controller/config/crd/patches/cainjection_in_workspacekinds.yaml create mode 100644 workspaces/controller/config/crd/patches/cainjection_in_workspaces.yaml create mode 100644 workspaces/controller/config/crd/patches/webhook_in_workspacekinds.yaml create mode 100644 workspaces/controller/config/crd/patches/webhook_in_workspaces.yaml create mode 100644 workspaces/controller/config/default/manager_webhook_patch.yaml create mode 100644 workspaces/controller/config/default/webhookcainjection_patch.yaml create mode 100644 workspaces/controller/config/webhook/kustomization.yaml create mode 100644 workspaces/controller/config/webhook/kustomizeconfig.yaml create mode 100644 workspaces/controller/config/webhook/manifests.yaml create mode 100644 workspaces/controller/config/webhook/service.yaml diff --git a/workspaces/controller/Makefile b/workspaces/controller/Makefile index 2178694e..6ce3d20e 100644 --- a/workspaces/controller/Makefile +++ b/workspaces/controller/Makefile @@ -199,7 +199,7 @@ endef define prompt_for_e2e_test_execution if [ "$$(echo "$(KUBEFLOW_TEST_PROMPT)" | tr '[:upper:]' '[:lower:]')" = "false" ]; then \ - echo "Skipping E2E test confirmation prompt (KUBEFLOW_TEST_PROMPT is set to true)"; \ + echo "Skipping E2E test confirmation prompt (KUBEFLOW_TEST_PROMPT is set to false)"; \ else \ current_k8s_context=$$(kubectl config current-context); \ echo "================================ WARNING ================================"; \ diff --git a/workspaces/controller/PROJECT b/workspaces/controller/PROJECT index a26562f2..4a01c524 100644 --- a/workspaces/controller/PROJECT +++ b/workspaces/controller/PROJECT @@ -16,6 +16,9 @@ resources: kind: Workspace path: github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1 version: v1beta1 + webhooks: + validation: true + webhookVersion: v1 - api: crdVersion: v1 controller: true @@ -23,4 +26,7 @@ resources: kind: WorkspaceKind path: github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1 version: v1beta1 + webhooks: + validation: true + webhookVersion: v1 version: "3" diff --git a/workspaces/controller/api/v1beta1/webhook_suite_test.go b/workspaces/controller/api/v1beta1/webhook_suite_test.go new file mode 100644 index 00000000..3c956830 --- /dev/null +++ b/workspaces/controller/api/v1beta1/webhook_suite_test.go @@ -0,0 +1,404 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "path/filepath" + "runtime" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + admissionv1 "k8s.io/api/admission/v1" + //+kubebuilder:scaffold:imports + apimachineryruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + cfg *rest.Config + k8sTestClient client.Client + testEnv *envtest.Environment + ctx context.Context + cancel context.CancelFunc +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Webhook Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: false, + + // The BinaryAssetsDirectory is only required if you want to run the tests directly + // without call the makefile target test. If not informed it will look for the + // default path defined in controller-runtime which is /usr/local/kubebuilder/. + // Note that you must have the required binaries setup under the bin directory to perform + // the tests directly. When we run make test it will be setup and used automatically. + BinaryAssetsDirectory: filepath.Join("..", "..", "bin", "k8s", + fmt.Sprintf("1.29.0-%s-%s", runtime.GOOS, runtime.GOARCH)), + + WebhookInstallOptions: envtest.WebhookInstallOptions{ + Paths: []string{filepath.Join("..", "..", "config", "webhook")}, + }, + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + scheme := apimachineryruntime.NewScheme() + err = AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + err = admissionv1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + k8sTestClient, err = client.New(cfg, client.Options{Scheme: scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sTestClient).NotTo(BeNil()) + + // start webhook server using Manager + webhookInstallOptions := &testEnv.WebhookInstallOptions + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: webhookInstallOptions.LocalServingHost, + Port: webhookInstallOptions.LocalServingPort, + CertDir: webhookInstallOptions.LocalServingCertDir, + }), + LeaderElection: false, + Metrics: metricsserver.Options{BindAddress: "0"}, + }) + Expect(err).NotTo(HaveOccurred()) + + err = (&WorkspaceKind{}).SetupWebhookWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&Workspace{}).SetupWebhookWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:webhook + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() + + // wait for the webhook server to get ready + dialer := &net.Dialer{Timeout: time.Second} + addrPort := fmt.Sprintf("%s:%d", webhookInstallOptions.LocalServingHost, webhookInstallOptions.LocalServingPort) + Eventually(func() error { + conn, err := tls.DialWithDialer(dialer, "tcp", addrPort, &tls.Config{InsecureSkipVerify: true}) + if err != nil { + return err + } + return conn.Close() + }).Should(Succeed()) + +}) + +var _ = AfterSuite(func() { + cancel() + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +// NewExampleWorkspaceKind1 returns the common "WorkspaceKind 1" object used in tests. +func NewExampleWorkspaceKind1(name string) *WorkspaceKind { + return &WorkspaceKind{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: WorkspaceKindSpec{ + Spawner: WorkspaceKindSpawner{ + DisplayName: "JupyterLab Notebook", + Description: "A Workspace which runs JupyterLab in a Pod", + Hidden: ptr.To(false), + Deprecated: ptr.To(false), + DeprecationMessage: ptr.To("This WorkspaceKind will be removed on 20XX-XX-XX, please use another WorkspaceKind."), + Icon: WorkspaceKindIcon{ + Url: ptr.To("https://jupyter.org/assets/favicons/apple-touch-icon-152x152.png"), + }, + Logo: WorkspaceKindIcon{ + ConfigMap: &WorkspaceKindConfigMap{ + Name: "my-logos", + Key: "apple-touch-icon-152x152.png", + }, + }, + }, + PodTemplate: WorkspaceKindPodTemplate{ + PodMetadata: &WorkspaceKindPodMetadata{}, + ServiceAccount: WorkspaceKindServiceAccount{ + Name: "default-editor", + }, + Culling: &WorkspaceKindCullingConfig{ + Enabled: ptr.To(true), + MaxInactiveSeconds: ptr.To(int32(86400)), + ActivityProbe: ActivityProbe{ + Jupyter: &ActivityProbeJupyter{ + LastActivity: true, + }, + }, + }, + Probes: &WorkspaceKindProbes{}, + VolumeMounts: WorkspaceKindVolumeMounts{ + Home: "/home/jovyan", + }, + HTTPProxy: &HTTPProxy{ + RemovePathPrefix: ptr.To(false), + RequestHeaders: &IstioHeaderOperations{ + Set: map[string]string{"X-RStudio-Root-Path": "{{ .PathPrefix }}"}, + Add: map[string]string{}, + Remove: []string{}, + }, + }, + ExtraEnv: []v1.EnvVar{ + { + Name: "NB_PREFIX", + Value: `{{ httpPathPrefix "jupyterlab" }}`, + }, + }, + ExtraVolumeMounts: []v1.VolumeMount{ + { + Name: "dshm", + MountPath: "/dev/shm", + }, + }, + ExtraVolumes: []v1.Volume{ + { + Name: "dshm", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: v1.StorageMediumMemory, + }, + }, + }, + }, + SecurityContext: &v1.PodSecurityContext{ + FSGroup: ptr.To(int64(100)), + }, + ContainerSecurityContext: &v1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &v1.Capabilities{ + Drop: []v1.Capability{"ALL"}, + }, + RunAsNonRoot: ptr.To(true), + }, + Options: WorkspaceKindPodOptions{ + ImageConfig: ImageConfig{ + Spawner: OptionsSpawnerConfig{ + Default: "jupyterlab_scipy_190", + }, + Values: []ImageConfigValue{ + { + Id: "jupyterlab_scipy_180", + Spawner: OptionSpawnerInfo{ + DisplayName: "jupyter-scipy:v1.8.0", + Description: ptr.To("JupyterLab, with SciPy Packages"), + Labels: []OptionSpawnerLabel{ + { + Key: "python_version", + Value: "3.11", + }, + }, + Hidden: ptr.To(true), + }, + Redirect: &OptionRedirect{ + To: "jupyterlab_scipy_190", + Message: &RedirectMessage{ + Level: "Info", + Text: "This update will change...", + }, + }, + Spec: ImageConfigSpec{ + Image: "docker.io/kubeflownotebookswg/jupyter-scipy:v1.8.0", + Ports: []ImagePort{ + { + Id: "jupyterlab", + DisplayName: "JupyterLab", + Port: 8888, + Protocol: "HTTP", + }, + }, + }, + }, + { + Id: "jupyterlab_scipy_190", + Spawner: OptionSpawnerInfo{ + DisplayName: "jupyter-scipy:v1.9.0", + Description: ptr.To("JupyterLab, with SciPy Packages"), + Labels: []OptionSpawnerLabel{ + { + Key: "python_version", + Value: "3.11", + }, + }, + }, + Spec: ImageConfigSpec{ + Image: "docker.io/kubeflownotebookswg/jupyter-scipy:v1.9.0", + Ports: []ImagePort{ + { + Id: "jupyterlab", + DisplayName: "JupyterLab", + Port: 8888, + Protocol: "HTTP", + }, + }, + }, + }, + }, + }, + PodConfig: PodConfig{ + Spawner: OptionsSpawnerConfig{ + Default: "tiny_cpu", + }, + Values: []PodConfigValue{ + { + Id: "tiny_cpu", + Spawner: OptionSpawnerInfo{ + DisplayName: "Tiny CPU", + Description: ptr.To("Pod with 0.1 CPU, 128 MB RAM"), + Labels: []OptionSpawnerLabel{ + { + Key: "cpu", + Value: "100m", + }, + { + Key: "memory", + Value: "128Mi", + }, + }, + }, + Spec: PodConfigSpec{ + Resources: &v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + }, + }, + }, + { + Id: "small_cpu", + Spawner: OptionSpawnerInfo{ + DisplayName: "Small CPU", + Description: ptr.To("Pod with 1 CPU, 2 GB RAM"), + Labels: []OptionSpawnerLabel{ + { + Key: "cpu", + Value: "1000m", + }, + { + Key: "memory", + Value: "2Gi", + }, + }, + }, + Spec: PodConfigSpec{ + Resources: &v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1000m"), + v1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + }, + }, + { + Id: "big_gpu", + Spawner: OptionSpawnerInfo{ + DisplayName: "Big GPU", + Description: ptr.To("Pod with 4 CPU, 16 GB RAM, and 1 GPU"), + Labels: []OptionSpawnerLabel{ + { + Key: "cpu", + Value: "4000m", + }, + { + Key: "memory", + Value: "16Gi", + }, + { + Key: "gpu", + Value: "1", + }, + }, + }, + Spec: PodConfigSpec{ + Affinity: nil, + NodeSelector: nil, + Tolerations: []v1.Toleration{ + { + Key: "nvidia.com/gpu", + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoSchedule, + }, + }, + Resources: &v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("4000m"), + v1.ResourceMemory: resource.MustParse("16Gi"), + }, + Limits: map[v1.ResourceName]resource.Quantity{ + "nvidia.com/gpu": resource.MustParse("1"), + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} diff --git a/workspaces/controller/api/v1beta1/workspace_webhook.go b/workspaces/controller/api/v1beta1/workspace_webhook.go new file mode 100644 index 00000000..f2d34d19 --- /dev/null +++ b/workspaces/controller/api/v1beta1/workspace_webhook.go @@ -0,0 +1,78 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + "fmt" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// log is for logging in this package. +var ( + workspacelog = logf.Log.WithName("workspace-resource") + k8sClient client.Client +) + +// SetupWebhookWithManager will setup the manager to manage the webhooks +func (r *Workspace) SetupWebhookWithManager(mgr ctrl.Manager) error { + k8sClient = mgr.GetClient() + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +// TODO(user): EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! + +// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation. +//+kubebuilder:webhook:path=/validate-kubeflow-org-v1beta1-workspace,mutating=false,failurePolicy=fail,sideEffects=None,groups=kubeflow.org,resources=workspaces,verbs=create;update,versions=v1beta1,name=vworkspace.kb.io,admissionReviewVersions=v1 + +var _ webhook.Validator = &Workspace{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type +func (r *Workspace) ValidateCreate() (admission.Warnings, error) { + workspacelog.Info("validate create", "name", r.Name) + + workspaceKindName := r.Spec.Kind + workspaceKind := &WorkspaceKind{} + if err := k8sClient.Get(context.Background(), client.ObjectKey{Name: workspaceKindName}, workspaceKind); err != nil { + return nil, fmt.Errorf("workspace kind %s not found", workspaceKindName) + } + + return nil, nil +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type +func (r *Workspace) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { + workspacelog.Info("validate update", "name", r.Name) + + // TODO(user): fill in your validation logic upon object update. + return nil, nil +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type +func (r *Workspace) ValidateDelete() (admission.Warnings, error) { + workspacelog.Info("validate delete", "name", r.Name) + + // TODO(user): fill in your validation logic upon object deletion. + return nil, nil +} diff --git a/workspaces/controller/api/v1beta1/workspace_webhook_test.go b/workspaces/controller/api/v1beta1/workspace_webhook_test.go new file mode 100644 index 00000000..99fe7575 --- /dev/null +++ b/workspaces/controller/api/v1beta1/workspace_webhook_test.go @@ -0,0 +1,39 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + . "github.com/onsi/ginkgo/v2" +) + +var _ = Describe("Workspace Webhook", func() { + + Context("When creating Workspace under Validating Webhook", func() { + It("Should deny if a required field is empty", func() { + + // TODO(user): Add your logic here + + }) + + It("Should admit if all required fields are provided", func() { + + // TODO(user): Add your logic here + + }) + }) + +}) diff --git a/workspaces/controller/api/v1beta1/workspacekind_webhook.go b/workspaces/controller/api/v1beta1/workspacekind_webhook.go new file mode 100644 index 00000000..59adfdcd --- /dev/null +++ b/workspaces/controller/api/v1beta1/workspacekind_webhook.go @@ -0,0 +1,315 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + "errors" + "fmt" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "reflect" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// log is for logging in this package. +var workspacekindlog = logf.Log.WithName("workspacekind-resource") + +// SetupWebhookWithManager will setup the manager to manage the webhooks +func (r *WorkspaceKind) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +// TODO(user): EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! + +// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation. +//+kubebuilder:webhook:path=/validate-kubeflow-org-v1beta1-workspacekind,mutating=false,failurePolicy=fail,sideEffects=None,groups=kubeflow.org,resources=workspacekinds,verbs=create;update,versions=v1beta1,name=vworkspacekind.kb.io,admissionReviewVersions=v1 + +var _ webhook.Validator = &WorkspaceKind{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type +func (r *WorkspaceKind) ValidateCreate() (admission.Warnings, error) { + workspacekindlog.Info("validate create", "name", r.Name) + + // Reject cycles in image options + imageConfigIdMap := make(map[string]ImageConfigValue) + for _, v := range r.Spec.PodTemplate.Options.ImageConfig.Values { + // Ensure ports are unique + ports := make(map[int32]bool) + for _, port := range v.Spec.Ports { + if _, exists := ports[port.Port]; exists { + return nil, fmt.Errorf("duplicate port %d in imageConfig with id '%s'", port.Port, v.Id) + } + ports[port.Port] = true + } + + imageConfigIdMap[v.Id] = v + } + for _, currentImageConfig := range imageConfigIdMap { + // follow any redirects to get the desired imageConfig + desiredImageConfig := currentImageConfig + visitedNodes := map[string]bool{currentImageConfig.Id: true} + for { + if desiredImageConfig.Redirect == nil { + break + } + if visitedNodes[desiredImageConfig.Redirect.To] { + return nil, fmt.Errorf("imageConfig with id '%s' has a circular redirect", desiredImageConfig.Id) + } + nextNode, ok := imageConfigIdMap[desiredImageConfig.Redirect.To] + if !ok { + return nil, fmt.Errorf("imageConfig with id '%s' not found, was redirected from '%s'", desiredImageConfig.Redirect.To, desiredImageConfig.Id) + } + desiredImageConfig = nextNode + visitedNodes[desiredImageConfig.Id] = true + } + } + + // Reject cycles in pod options + podConfigIdMap := make(map[string]PodConfigValue) + for _, v := range r.Spec.PodTemplate.Options.PodConfig.Values { + podConfigIdMap[v.Id] = v + } + for _, currentPodConfig := range podConfigIdMap { + // follow any redirects to get the desired podConfig + desiredPodConfig := currentPodConfig + visitedNodes := map[string]bool{currentPodConfig.Id: true} + for { + if desiredPodConfig.Redirect == nil { + break + } + if visitedNodes[desiredPodConfig.Redirect.To] { + return nil, fmt.Errorf("podConfig with id '%s' has a circular redirect", desiredPodConfig.Id) + } + nextNode, ok := podConfigIdMap[desiredPodConfig.Redirect.To] + if !ok { + return nil, fmt.Errorf("podConfig with id '%s' not found, was redirected from '%s'", desiredPodConfig.Redirect.To, desiredPodConfig.Id) + } + desiredPodConfig = nextNode + visitedNodes[desiredPodConfig.Id] = true + } + } + + // Ensure the default image config is present + if _, ok := imageConfigIdMap[r.Spec.PodTemplate.Options.ImageConfig.Spawner.Default]; !ok { + return nil, fmt.Errorf("default image config with id '%s' is not present in spec.podTemplate.options.imageConfig.values", r.Spec.PodTemplate.Options.ImageConfig.Spawner.Default) + } + + // Ensure the default pod config is present + if _, ok := podConfigIdMap[r.Spec.PodTemplate.Options.PodConfig.Spawner.Default]; !ok { + return nil, fmt.Errorf("default pod config with id '%s' is not present in spec.podTemplate.options.podConfig.values", r.Spec.PodTemplate.Options.PodConfig.Spawner.Default) + } + + // TODO: Ensure that `spec.podTemplate.extraEnv[].value` is a valid go template + + return nil, nil +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type +func (r *WorkspaceKind) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { + workspacekindlog.Info("validate update", "name", r.Name) + + // Type assertion to convert the old runtime.Object to WorkspaceKind + oldWorkspaceKind, ok := old.(*WorkspaceKind) + if !ok { + return nil, errors.New("old object is not a WorkspaceKind") + } + + // Validate ImageConfig is immutable + imageConfigSpecMap := make(map[string]ImageConfigSpec) + for _, v := range oldWorkspaceKind.Spec.PodTemplate.Options.ImageConfig.Values { + imageConfigSpecMap[v.Id] = v.Spec + } + updatedImageConfigSpecMap := make(map[string]ImageConfigSpec) + for _, v := range r.Spec.PodTemplate.Options.ImageConfig.Values { + updatedImageConfigSpecMap[v.Id] = v.Spec + if oldSpec, exists := imageConfigSpecMap[v.Id]; exists { + if !reflect.DeepEqual(oldSpec, v.Spec) { + return nil, fmt.Errorf("spec.podTemplate.options.imageConfig.values with id '%s' is immutable", v.Id) + } + } + } + + // Validate PodConfig is immutable + podConfigSpecMap := make(map[string]PodConfigSpec) + for _, v := range oldWorkspaceKind.Spec.PodTemplate.Options.PodConfig.Values { + podConfigSpecMap[v.Id] = v.Spec + } + updatedPodConfigSpecMap := make(map[string]PodConfigSpec) + for _, v := range r.Spec.PodTemplate.Options.PodConfig.Values { + updatedPodConfigSpecMap[v.Id] = v.Spec + if oldSpec, exists := podConfigSpecMap[v.Id]; exists { + normalizePodConfigSpec(&oldSpec) + normalizePodConfigSpec(&v.Spec) + + if !reflect.DeepEqual(oldSpec, v.Spec) { + return nil, fmt.Errorf("spec.podTemplate.options.podConfig.values with id '%s' is immutable", v.Id) + } + } + } + + kbCacheWorkspaceKindField := ".spec.kind" + workspaces := &WorkspaceList{} + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(kbCacheWorkspaceKindField, r.Name), + Namespace: corev1.NamespaceAll, + } + if err := k8sClient.List(context.Background(), workspaces, listOpts); err != nil { + return nil, err + } + + usedImageConfig := make(map[string]int) + usedPodConfig := make(map[string]int) + for _, ws := range workspaces.Items { + usedImageConfig[ws.Spec.PodTemplate.Options.ImageConfig]++ + usedPodConfig[ws.Spec.PodTemplate.Options.PodConfig]++ + } + // Only allow removing option ids which are not used + for id, _ := range imageConfigSpecMap { + if _, exists := updatedImageConfigSpecMap[id]; !exists { + // check if this option is used by any workspace + if usedImageConfig[id] > 0 { + errMsg := fmt.Sprintf("spec.podTemplate.options.imageConfig.values with id '%s' is used by %d workspace", id, usedImageConfig[id]) + if usedImageConfig[id] > 1 { + errMsg += "s" + } + return nil, fmt.Errorf(errMsg) + } + } + } + for id, _ := range podConfigSpecMap { + if _, exists := updatedPodConfigSpecMap[id]; !exists { + // check if this option is used by any workspace + if usedPodConfig[id] > 0 { + errMsg := fmt.Sprintf("spec.podTemplate.options.podConfig.values with id '%s' is used by %d workspace", id, usedPodConfig[id]) + if usedPodConfig[id] > 1 { + errMsg += "s" + } + return nil, fmt.Errorf(errMsg) + } + } + } + + // Reject cycles in image options + imageConfigIdMap := make(map[string]ImageConfigValue) + for _, v := range r.Spec.PodTemplate.Options.ImageConfig.Values { + imageConfigIdMap[v.Id] = v + } + for _, currentImageConfig := range imageConfigIdMap { + // follow any redirects to get the desired imageConfig + desiredImageConfig := currentImageConfig + visitedNodes := map[string]bool{currentImageConfig.Id: true} + for { + if desiredImageConfig.Redirect == nil { + break + } + if visitedNodes[desiredImageConfig.Redirect.To] { + return nil, fmt.Errorf("imageConfig with id '%s' has a circular redirect", desiredImageConfig.Id) + } + nextNode, ok := imageConfigIdMap[desiredImageConfig.Redirect.To] + if !ok { + return nil, fmt.Errorf("imageConfig with id '%s' not found, was redirected from '%s'", desiredImageConfig.Redirect.To, desiredImageConfig.Id) + } + desiredImageConfig = nextNode + visitedNodes[desiredImageConfig.Id] = true + } + } + + // Reject cycles in pod options + podConfigIdMap := make(map[string]PodConfigValue) + for _, v := range r.Spec.PodTemplate.Options.PodConfig.Values { + podConfigIdMap[v.Id] = v + } + for _, currentPodConfig := range podConfigIdMap { + // follow any redirects to get the desired podConfig + desiredPodConfig := currentPodConfig + visitedNodes := map[string]bool{currentPodConfig.Id: true} + for { + if desiredPodConfig.Redirect == nil { + break + } + if visitedNodes[desiredPodConfig.Redirect.To] { + return nil, fmt.Errorf("podConfig with id '%s' has a circular redirect", desiredPodConfig.Id) + } + nextNode, ok := podConfigIdMap[desiredPodConfig.Redirect.To] + if !ok { + return nil, fmt.Errorf("podConfig with id '%s' not found, was redirected from '%s'", desiredPodConfig.Redirect.To, desiredPodConfig.Id) + } + desiredPodConfig = nextNode + visitedNodes[desiredPodConfig.Id] = true + } + } + + // Ensure the default image config is present + if _, ok := imageConfigIdMap[r.Spec.PodTemplate.Options.ImageConfig.Spawner.Default]; !ok { + return nil, fmt.Errorf("default image config with id '%s' is not present in spec.podTemplate.options.imageConfig.values", r.Spec.PodTemplate.Options.ImageConfig.Spawner.Default) + } + + // Ensure the default pod config is present + if _, ok := podConfigIdMap[r.Spec.PodTemplate.Options.PodConfig.Spawner.Default]; !ok { + return nil, fmt.Errorf("default pod config with id '%s' is not present in spec.podTemplate.options.podConfig.values", r.Spec.PodTemplate.Options.PodConfig.Spawner.Default) + } + return nil, nil +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type +func (r *WorkspaceKind) ValidateDelete() (admission.Warnings, error) { + workspacekindlog.Info("validate delete", "name", r.Name) + + // TODO(user): fill in your validation logic upon object deletion. + return nil, nil +} + +func normalizePodConfigSpec(spec *PodConfigSpec) { + // Normalize NodeSelector + if spec.NodeSelector != nil && len(spec.NodeSelector) == 0 { + spec.NodeSelector = nil + } + + // Normalize Tolerations + if spec.Tolerations != nil && len(spec.Tolerations) == 0 { + spec.Tolerations = nil + } + + // Normalize ResourceRequests + if reflect.DeepEqual(spec.Resources.Requests, corev1.ResourceList{}) { + spec.Resources.Requests = nil + } + if spec.Resources.Requests != nil { + for key, value := range spec.Resources.Requests { + spec.Resources.Requests[key] = resource.MustParse(value.String()) + } + } + + // Normalize ResourceLimits + if reflect.DeepEqual(spec.Resources.Limits, corev1.ResourceList{}) { + spec.Resources.Limits = nil + } + if spec.Resources.Limits != nil { + for key, value := range spec.Resources.Limits { + spec.Resources.Limits[key] = resource.MustParse(value.String()) + } + } +} diff --git a/workspaces/controller/api/v1beta1/workspacekind_webhook_test.go b/workspaces/controller/api/v1beta1/workspacekind_webhook_test.go new file mode 100644 index 00000000..c5780e37 --- /dev/null +++ b/workspaces/controller/api/v1beta1/workspacekind_webhook_test.go @@ -0,0 +1,77 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "fmt" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ = Describe("WorkspaceKind Webhook", func() { + + Context("When updating WorkspaceKind under Validating Webhook", Ordered, func() { + var ( + workspaceKindName string + workspaceKindKey types.NamespacedName + ) + + BeforeAll(func() { + uniqueName := "wsk-webhook-update-test" + workspaceKindName = fmt.Sprintf("workspacekind-%s", uniqueName) + workspaceKindKey = types.NamespacedName{Name: workspaceKindName} + + By("creating the WorkspaceKind") + workspaceKind := NewExampleWorkspaceKind1(workspaceKindName) + Expect(k8sClient.Create(ctx, workspaceKind)).To(Succeed()) + }) + + AfterAll(func() { + By("deleting the WorkspaceKind") + workspaceKind := &WorkspaceKind{ + ObjectMeta: metav1.ObjectMeta{ + Name: workspaceKindName, + }, + } + Expect(k8sClient.Delete(ctx, workspaceKind)).To(Succeed()) + }) + + It("should not allow updating immutable fields", func() { + By("getting the WorkspaceKind") + workspaceKind := &WorkspaceKind{} + Expect(k8sClient.Get(ctx, workspaceKindKey, workspaceKind)).To(Succeed()) + patch := client.MergeFrom(workspaceKind.DeepCopy()) + + By("failing to update the `spec.podTemplate.options.imageConfig.values[0].spec` field") + newWorkspaceKind := workspaceKind.DeepCopy() + newWorkspaceKind.Spec.PodTemplate.Options.ImageConfig.Values[0].Spec.Image = "new-image:latest" + Expect(k8sClient.Patch(ctx, newWorkspaceKind, patch)).NotTo(Succeed()) + + By("failing to update the `spec.podTemplate.options.podConfig.values[0].spec` field") + newWorkspaceKind = workspaceKind.DeepCopy() + newWorkspaceKind.Spec.PodTemplate.Options.PodConfig.Values[0].Spec.Resources.Requests[v1.ResourceCPU] = resource.MustParse("99") + Expect(k8sClient.Patch(ctx, newWorkspaceKind, patch)).NotTo(Succeed()) + }) + + }) + +}) diff --git a/workspaces/controller/api/v1beta1/zz_generated.deepcopy.go b/workspaces/controller/api/v1beta1/zz_generated.deepcopy.go index 1beab4fd..72655956 100644 --- a/workspaces/controller/api/v1beta1/zz_generated.deepcopy.go +++ b/workspaces/controller/api/v1beta1/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ package v1beta1 import ( "k8s.io/api/core/v1" - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. diff --git a/workspaces/controller/cmd/main.go b/workspaces/controller/cmd/main.go index 0c807070..e9f85fd2 100644 --- a/workspaces/controller/cmd/main.go +++ b/workspaces/controller/cmd/main.go @@ -136,6 +136,18 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "WorkspaceKind") os.Exit(1) } + if os.Getenv("ENABLE_WEBHOOKS") != "false" { + if err = (&kubefloworgv1beta1.WorkspaceKind{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "WorkspaceKind") + os.Exit(1) + } + } + if os.Getenv("ENABLE_WEBHOOKS") != "false" { + if err = (&kubefloworgv1beta1.Workspace{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "Workspace") + os.Exit(1) + } + } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/workspaces/controller/config/certmanager/certificate.yaml b/workspaces/controller/config/certmanager/certificate.yaml new file mode 100644 index 00000000..7943460b --- /dev/null +++ b/workspaces/controller/config/certmanager/certificate.yaml @@ -0,0 +1,39 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: certificate + app.kubernetes.io/instance: serving-cert + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: workspace-controller + app.kubernetes.io/part-of: workspace-controller + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: certificate + app.kubernetes.io/instance: serving-cert + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: workspace-controller + app.kubernetes.io/part-of: workspace-controller + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/workspaces/controller/config/certmanager/kustomization.yaml b/workspaces/controller/config/certmanager/kustomization.yaml new file mode 100644 index 00000000..bebea5a5 --- /dev/null +++ b/workspaces/controller/config/certmanager/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- certificate.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/workspaces/controller/config/certmanager/kustomizeconfig.yaml b/workspaces/controller/config/certmanager/kustomizeconfig.yaml new file mode 100644 index 00000000..cf6f89e8 --- /dev/null +++ b/workspaces/controller/config/certmanager/kustomizeconfig.yaml @@ -0,0 +1,8 @@ +# This configuration is for teaching kustomize how to update name ref substitution +nameReference: +- kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name diff --git a/workspaces/controller/config/crd/kustomization.yaml b/workspaces/controller/config/crd/kustomization.yaml index 76f94edd..aac7d81b 100644 --- a/workspaces/controller/config/crd/kustomization.yaml +++ b/workspaces/controller/config/crd/kustomization.yaml @@ -20,5 +20,5 @@ patches: # [WEBHOOK] To enable webhook, uncomment the following section # the following config is for teaching kustomize how to do kustomization for CRDs. -#configurations: -#- kustomizeconfig.yaml +configurations: +- kustomizeconfig.yaml diff --git a/workspaces/controller/config/crd/patches/cainjection_in_workspacekinds.yaml b/workspaces/controller/config/crd/patches/cainjection_in_workspacekinds.yaml new file mode 100644 index 00000000..887f17ff --- /dev/null +++ b/workspaces/controller/config/crd/patches/cainjection_in_workspacekinds.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: CERTIFICATE_NAMESPACE/CERTIFICATE_NAME + name: workspacekinds.kubeflow.org diff --git a/workspaces/controller/config/crd/patches/cainjection_in_workspaces.yaml b/workspaces/controller/config/crd/patches/cainjection_in_workspaces.yaml new file mode 100644 index 00000000..d2fe44b2 --- /dev/null +++ b/workspaces/controller/config/crd/patches/cainjection_in_workspaces.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: CERTIFICATE_NAMESPACE/CERTIFICATE_NAME + name: workspaces.kubeflow.org diff --git a/workspaces/controller/config/crd/patches/webhook_in_workspacekinds.yaml b/workspaces/controller/config/crd/patches/webhook_in_workspacekinds.yaml new file mode 100644 index 00000000..342a98d1 --- /dev/null +++ b/workspaces/controller/config/crd/patches/webhook_in_workspacekinds.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: workspacekinds.kubeflow.org +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/workspaces/controller/config/crd/patches/webhook_in_workspaces.yaml b/workspaces/controller/config/crd/patches/webhook_in_workspaces.yaml new file mode 100644 index 00000000..ed75fd19 --- /dev/null +++ b/workspaces/controller/config/crd/patches/webhook_in_workspaces.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: workspaces.kubeflow.org +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/workspaces/controller/config/default/kustomization.yaml b/workspaces/controller/config/default/kustomization.yaml index b7ddfabc..6ac63df2 100644 --- a/workspaces/controller/config/default/kustomization.yaml +++ b/workspaces/controller/config/default/kustomization.yaml @@ -20,9 +20,9 @@ resources: - ../manager # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml -#- ../webhook +- ../webhook # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. -#- ../certmanager +- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus @@ -34,109 +34,93 @@ patches: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml -#- path: manager_webhook_patch.yaml +- path: manager_webhook_patch.yaml # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. # 'CERTMANAGER' needs to be enabled to use ca injection -#- path: webhookcainjection_patch.yaml +- path: webhookcainjection_patch.yaml # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations -#replacements: -# - source: # Add cert-manager annotation to ValidatingWebhookConfiguration, MutatingWebhookConfiguration and CRDs -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldPath: .metadata.namespace # namespace of the certificate CR -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - select: -# kind: CustomResourceDefinition -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - source: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldPath: .metadata.name -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - select: -# kind: CustomResourceDefinition -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - source: # Add cert-manager annotation to the webhook Service -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.name # namespace of the service -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 0 -# create: true -# - source: -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.namespace # namespace of the service -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 1 -# create: true +replacements: + - source: # Add cert-manager annotation to ValidatingWebhookConfiguration, MutatingWebhookConfiguration and CRDs + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldPath: .metadata.namespace # namespace of the certificate CR + targets: + - select: + kind: ValidatingWebhookConfiguration + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 0 + create: true + - select: + kind: CustomResourceDefinition + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 0 + create: true + - source: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldPath: .metadata.name + targets: + - select: + kind: ValidatingWebhookConfiguration + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 1 + create: true + - select: + kind: CustomResourceDefinition + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 1 + create: true + - source: # Add cert-manager annotation to the webhook Service + kind: Service + version: v1 + name: webhook-service + fieldPath: .metadata.name # namespace of the service + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + fieldPaths: + - .spec.dnsNames.0 + - .spec.dnsNames.1 + options: + delimiter: '.' + index: 0 + create: true + - source: + kind: Service + version: v1 + name: webhook-service + fieldPath: .metadata.namespace # namespace of the service + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + fieldPaths: + - .spec.dnsNames.0 + - .spec.dnsNames.1 + options: + delimiter: '.' + index: 1 + create: true diff --git a/workspaces/controller/config/default/manager_webhook_patch.yaml b/workspaces/controller/config/default/manager_webhook_patch.yaml new file mode 100644 index 00000000..738de350 --- /dev/null +++ b/workspaces/controller/config/default/manager_webhook_patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert diff --git a/workspaces/controller/config/default/webhookcainjection_patch.yaml b/workspaces/controller/config/default/webhookcainjection_patch.yaml new file mode 100644 index 00000000..0124dbc5 --- /dev/null +++ b/workspaces/controller/config/default/webhookcainjection_patch.yaml @@ -0,0 +1,15 @@ +# This patch add annotation to admission webhook config and +# CERTIFICATE_NAMESPACE and CERTIFICATE_NAME will be substituted by kustomize +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/name: validatingwebhookconfiguration + app.kubernetes.io/instance: validating-webhook-configuration + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: workspace-controller + app.kubernetes.io/part-of: workspace-controller + app.kubernetes.io/managed-by: kustomize + name: validating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: CERTIFICATE_NAMESPACE/CERTIFICATE_NAME diff --git a/workspaces/controller/config/webhook/kustomization.yaml b/workspaces/controller/config/webhook/kustomization.yaml new file mode 100644 index 00000000..9cf26134 --- /dev/null +++ b/workspaces/controller/config/webhook/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- manifests.yaml +- service.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/workspaces/controller/config/webhook/kustomizeconfig.yaml b/workspaces/controller/config/webhook/kustomizeconfig.yaml new file mode 100644 index 00000000..206316e5 --- /dev/null +++ b/workspaces/controller/config/webhook/kustomizeconfig.yaml @@ -0,0 +1,22 @@ +# the following config is for teaching kustomize where to look at when substituting nameReference. +# It requires kustomize v2.1.0 or newer to work properly. +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + +namespace: +- kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true +- kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true diff --git a/workspaces/controller/config/webhook/manifests.yaml b/workspaces/controller/config/webhook/manifests.yaml new file mode 100644 index 00000000..04256eb3 --- /dev/null +++ b/workspaces/controller/config/webhook/manifests.yaml @@ -0,0 +1,46 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-kubeflow-org-v1beta1-workspace + failurePolicy: Fail + name: vworkspace.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - workspaces + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-kubeflow-org-v1beta1-workspacekind + failurePolicy: Fail + name: vworkspacekind.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - workspacekinds + sideEffects: None diff --git a/workspaces/controller/config/webhook/service.yaml b/workspaces/controller/config/webhook/service.yaml new file mode 100644 index 00000000..ddabd8e7 --- /dev/null +++ b/workspaces/controller/config/webhook/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: workspace-controller + app.kubernetes.io/managed-by: kustomize + name: webhook-service + namespace: system +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager diff --git a/workspaces/controller/go.mod b/workspaces/controller/go.mod index 6d5c34ac..d01e4128 100644 --- a/workspaces/controller/go.mod +++ b/workspaces/controller/go.mod @@ -3,6 +3,7 @@ module github.com/kubeflow/notebooks/workspaces/controller go 1.21 require ( + github.com/go-logr/logr v1.4.1 github.com/onsi/ginkgo/v2 v2.14.0 github.com/onsi/gomega v1.30.0 k8s.io/api v0.29.2 @@ -19,7 +20,6 @@ require ( github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.8.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect From c832b27e263a7a1b1942ad455191d2e94def17a4 Mon Sep 17 00:00:00 2001 From: Adem Baccara <71262172+Adembc@users.noreply.github.com> Date: Thu, 15 Aug 2024 19:51:49 +0100 Subject: [PATCH 2/8] add tests for workspacekind and workspace webhooks Signed-off-by: Adem Baccara <71262172+Adembc@users.noreply.github.com> --- .../api/v1beta1/webhook_suite_test.go | 124 ++++++++++- .../api/v1beta1/workspace_webhook_test.go | 53 ++++- .../api/v1beta1/workspacekind_webhook.go | 37 +++- .../api/v1beta1/workspacekind_webhook_test.go | 197 +++++++++++++++--- 4 files changed, 371 insertions(+), 40 deletions(-) diff --git a/workspaces/controller/api/v1beta1/webhook_suite_test.go b/workspaces/controller/api/v1beta1/webhook_suite_test.go index 3c956830..4885c0d5 100644 --- a/workspaces/controller/api/v1beta1/webhook_suite_test.go +++ b/workspaces/controller/api/v1beta1/webhook_suite_test.go @@ -123,6 +123,16 @@ var _ = BeforeSuite(func() { err = (&WorkspaceKind{}).SetupWebhookWithManager(mgr) Expect(err).NotTo(HaveOccurred()) + // Indexing `.spec.kind` here, not in SetupWebhookWithManager, to avoid conflicts with existing indexing. + // This indexing is specifically for testing purposes to index `Workspace` by `WorkspaceKind`. + err = mgr.GetFieldIndexer().IndexField(context.Background(), &Workspace{}, kbCacheWorkspaceKindField, func(rawObj client.Object) []string { + ws := rawObj.(*Workspace) + if ws.Spec.Kind == "" { + return nil + } + return []string{ws.Spec.Kind} + }) + Expect(err).NotTo(HaveOccurred()) err = (&Workspace{}).SetupWebhookWithManager(mgr) Expect(err).NotTo(HaveOccurred()) @@ -154,8 +164,8 @@ var _ = AfterSuite(func() { Expect(err).NotTo(HaveOccurred()) }) -// NewExampleWorkspaceKind1 returns the common "WorkspaceKind 1" object used in tests. -func NewExampleWorkspaceKind1(name string) *WorkspaceKind { +// NewExampleWorkspaceKind returns the common "WorkspaceKind" object used in tests. +func NewExampleWorkspaceKind(name string) *WorkspaceKind { return &WorkspaceKind{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -402,3 +412,113 @@ func NewExampleWorkspaceKind1(name string) *WorkspaceKind { }, } } + +// NewExampleWorkspaceKindWithImageConfigCycle returns a WorkspaceKind with a cycle in the ImageConfig options. +func NewExampleWorkspaceKindWithImageConfigCycle(name string) *WorkspaceKind { + workspaceKind := NewExampleWorkspaceKind(name) + workspaceKind.Spec.PodTemplate.Options.ImageConfig.Values[1].Redirect = &OptionRedirect{ + To: "jupyterlab_scipy_180", + } + return workspaceKind +} + +// NewExampleWorkspaceKindWithPodConfigCycle returns a WorkspaceKind with a cycle in the PodConfig options. +func NewExampleWorkspaceKindWithPodConfigCycle(name string) *WorkspaceKind { + workspaceKind := NewExampleWorkspaceKind(name) + workspaceKind.Spec.PodTemplate.Options.PodConfig.Values[0].Redirect = &OptionRedirect{ + To: "small_cpu", + Message: &RedirectMessage{ + Level: "Info", + Text: "This update will change...", + }, + } + workspaceKind.Spec.PodTemplate.Options.PodConfig.Values[1].Redirect = &OptionRedirect{ + To: "tiny_cpu", + } + + return workspaceKind +} + +// NewExampleWorkspaceKindWithInvalidImageConfig returns a WorkspaceKind with an invalid redirect in the ImageConfig options. +func NewExampleWorkspaceKindWithInvalidImageConfig(name string) *WorkspaceKind { + workspaceKind := NewExampleWorkspaceKind(name) + workspaceKind.Spec.PodTemplate.Options.ImageConfig.Values[1].Redirect = &OptionRedirect{ + To: "invalid_image_config", + } + + return workspaceKind +} + +// NewExampleWorkspaceKindWithInvalidPodConfig returns a WorkspaceKind with an invalid redirect in the PodConfig options. +func NewExampleWorkspaceKindWithInvalidPodConfig(name string) *WorkspaceKind { + workspaceKind := NewExampleWorkspaceKind(name) + workspaceKind.Spec.PodTemplate.Options.PodConfig.Values[0].Redirect = &OptionRedirect{ + To: "invalid_pod_config", + } + + return workspaceKind +} + +// NewExampleWorkspaceKindWithMissingDefaultImageConfig returns a WorkspaceKind with missing default image config. +func NewExampleWorkspaceKindWithInvalidDefaultImageConfig(name string) *WorkspaceKind { + workspaceKind := NewExampleWorkspaceKind(name) + workspaceKind.Spec.PodTemplate.Options.ImageConfig.Spawner.Default = "invalid_image_config" + return workspaceKind +} + +// NewExampleWorkspaceKindWithMissingDefaultPodConfig returns a WorkspaceKind with missing default pod config. +func NewExampleWorkspaceKindWithInvalidDefaultPodConfig(name string) *WorkspaceKind { + workspaceKind := NewExampleWorkspaceKind(name) + workspaceKind.Spec.PodTemplate.Options.PodConfig.Spawner.Default = "invalid_pod_config" + return workspaceKind +} + +// NewExampleWorkspaceKindWithInvalidExtraEnvValue returns a WorkspaceKind with an invalid extraEnv value. +func NewExampleWorkspaceKindWithDuplicatePorts(name string) *WorkspaceKind { + workspaceKind := NewExampleWorkspaceKind(name) + workspaceKind.Spec.PodTemplate.Options.ImageConfig.Values[0].Spec.Ports = []ImagePort{ + { + Id: "jupyterlab", + DisplayName: "JupyterLab", + Port: 8888, + Protocol: "HTTP", + }, + { + Id: "jupyterlab2", + DisplayName: "JupyterLab2", + Port: 8888, + Protocol: "HTTP", + }, + } + return workspaceKind +} + +// NewExampleWorkspaceKindWithInvalidExtraEnvValue returns a WorkspaceKind with an invalid extraEnv value. +func NewExampleWorkspaceKindWithInvalidExtraEnvValue(name string) *WorkspaceKind { + workspaceKind := NewExampleWorkspaceKind(name) + workspaceKind.Spec.PodTemplate.ExtraEnv = []v1.EnvVar{ + { + Name: "NB_PREFIX", + Value: `{{ httpPathPrefix "jupyterlab" }`, + }, + } + return workspaceKind +} + +// NewExampleWorkspace returns the common "Workspace" object used in tests. +func NewExampleWorkspace(name, namespace, workspaceKindName string) *Workspace { + return &Workspace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: WorkspaceSpec{ + Kind: workspaceKindName, + PodTemplate: WorkspacePodTemplate{Options: WorkspacePodOptions{ + ImageConfig: "jupyterlab_scipy_180", + PodConfig: "tiny_cpu", + }, + }, + }, + } +} diff --git a/workspaces/controller/api/v1beta1/workspace_webhook_test.go b/workspaces/controller/api/v1beta1/workspace_webhook_test.go index 99fe7575..729bc421 100644 --- a/workspaces/controller/api/v1beta1/workspace_webhook_test.go +++ b/workspaces/controller/api/v1beta1/workspace_webhook_test.go @@ -17,21 +17,64 @@ limitations under the License. package v1beta1 import ( + "fmt" . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var _ = Describe("Workspace Webhook", func() { - Context("When creating Workspace under Validating Webhook", func() { - It("Should deny if a required field is empty", func() { + Context("When creating Workspace under Validating Webhook", Ordered, func() { + var ( + workspaceName string + workspaceKindName string + namespaceName string + ) - // TODO(user): Add your logic here + BeforeAll(func() { + uniqueName := "ws-create-test" + workspaceName = fmt.Sprintf("workspace-%s", uniqueName) + namespaceName = "default" + workspaceKindName = fmt.Sprintf("workspacekind-%s", uniqueName) + + By("creating the WorkspaceKind") + workspaceKind := NewExampleWorkspaceKind(workspaceKindName) + Expect(k8sClient.Create(ctx, workspaceKind)).To(Succeed()) + + }) + + AfterAll(func() { + + By("deleting the WorkspaceKind") + workspaceKind := &WorkspaceKind{ + ObjectMeta: metav1.ObjectMeta{ + Name: workspaceKindName, + }, + } + Expect(k8sClient.Delete(ctx, workspaceKind)).To(Succeed()) }) - It("Should admit if all required fields are provided", func() { + It("should reject workspace creation with an invalid WorkspaceKind", func() { + workspaceKindName := "invalid-workspace-kind" + + By("creating the Workspace") + workspace := NewExampleWorkspace(workspaceName, namespaceName, workspaceKindName) + err := k8sClient.Create(ctx, workspace) + Expect(err).ToNot(Succeed()) + Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("workspace kind %s not found", workspaceKindName))) + + }) + + It("should successfully create workspace with a valid WorkspaceKind", func() { + + By("creating the Workspace") + workspace := NewExampleWorkspace(workspaceName, namespaceName, workspaceKindName) + Expect(k8sClient.Create(ctx, workspace)).To(Succeed()) - // TODO(user): Add your logic here + By("deleting the Workspace") + Expect(k8sClient.Delete(ctx, workspace)).To(Succeed()) }) }) diff --git a/workspaces/controller/api/v1beta1/workspacekind_webhook.go b/workspaces/controller/api/v1beta1/workspacekind_webhook.go index 59adfdcd..138f2ef5 100644 --- a/workspaces/controller/api/v1beta1/workspacekind_webhook.go +++ b/workspaces/controller/api/v1beta1/workspacekind_webhook.go @@ -30,8 +30,11 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + "text/template" ) +const kbCacheWorkspaceKindField = ".spec.kind" + // log is for logging in this package. var workspacekindlog = logf.Log.WithName("workspacekind-resource") @@ -53,7 +56,7 @@ var _ webhook.Validator = &WorkspaceKind{} func (r *WorkspaceKind) ValidateCreate() (admission.Warnings, error) { workspacekindlog.Info("validate create", "name", r.Name) - // Reject cycles in image options + // Reject cycles in imageConfig options imageConfigIdMap := make(map[string]ImageConfigValue) for _, v := range r.Spec.PodTemplate.Options.ImageConfig.Values { // Ensure ports are unique @@ -87,7 +90,7 @@ func (r *WorkspaceKind) ValidateCreate() (admission.Warnings, error) { } } - // Reject cycles in pod options + // Reject cycles in podConfig options podConfigIdMap := make(map[string]PodConfigValue) for _, v := range r.Spec.PodTemplate.Options.PodConfig.Values { podConfigIdMap[v.Id] = v @@ -114,15 +117,23 @@ func (r *WorkspaceKind) ValidateCreate() (admission.Warnings, error) { // Ensure the default image config is present if _, ok := imageConfigIdMap[r.Spec.PodTemplate.Options.ImageConfig.Spawner.Default]; !ok { - return nil, fmt.Errorf("default image config with id '%s' is not present in spec.podTemplate.options.imageConfig.values", r.Spec.PodTemplate.Options.ImageConfig.Spawner.Default) + return nil, fmt.Errorf("default image config with id '%s' is not found in spec.podTemplate.options.imageConfig.values", r.Spec.PodTemplate.Options.ImageConfig.Spawner.Default) } // Ensure the default pod config is present if _, ok := podConfigIdMap[r.Spec.PodTemplate.Options.PodConfig.Spawner.Default]; !ok { - return nil, fmt.Errorf("default pod config with id '%s' is not present in spec.podTemplate.options.podConfig.values", r.Spec.PodTemplate.Options.PodConfig.Spawner.Default) + return nil, fmt.Errorf("default pod config with id '%s' is not found in spec.podTemplate.options.podConfig.values", r.Spec.PodTemplate.Options.PodConfig.Spawner.Default) } - // TODO: Ensure that `spec.podTemplate.extraEnv[].value` is a valid go template + // Validate the extraEnv values are valid go templates + for _, env := range r.Spec.PodTemplate.ExtraEnv { + rawValue := env.Value + _, err := template.New("value").Funcs(template.FuncMap{"httpPathPrefix": func(_ string) string { return "" }}).Parse(rawValue) + if err != nil { + err = fmt.Errorf("failed to parse value %q: %v", rawValue, err) + return nil, err + } + } return nil, nil } @@ -170,7 +181,6 @@ func (r *WorkspaceKind) ValidateUpdate(old runtime.Object) (admission.Warnings, } } - kbCacheWorkspaceKindField := ".spec.kind" workspaces := &WorkspaceList{} listOpts := &client.ListOptions{ FieldSelector: fields.OneTermEqualSelector(kbCacheWorkspaceKindField, r.Name), @@ -264,13 +274,24 @@ func (r *WorkspaceKind) ValidateUpdate(old runtime.Object) (admission.Warnings, // Ensure the default image config is present if _, ok := imageConfigIdMap[r.Spec.PodTemplate.Options.ImageConfig.Spawner.Default]; !ok { - return nil, fmt.Errorf("default image config with id '%s' is not present in spec.podTemplate.options.imageConfig.values", r.Spec.PodTemplate.Options.ImageConfig.Spawner.Default) + return nil, fmt.Errorf("default image config with id '%s' is not found in spec.podTemplate.options.imageConfig.values", r.Spec.PodTemplate.Options.ImageConfig.Spawner.Default) } // Ensure the default pod config is present if _, ok := podConfigIdMap[r.Spec.PodTemplate.Options.PodConfig.Spawner.Default]; !ok { - return nil, fmt.Errorf("default pod config with id '%s' is not present in spec.podTemplate.options.podConfig.values", r.Spec.PodTemplate.Options.PodConfig.Spawner.Default) + return nil, fmt.Errorf("default pod config with id '%s' is not found in spec.podTemplate.options.podConfig.values", r.Spec.PodTemplate.Options.PodConfig.Spawner.Default) } + + // Validate the extraEnv values are valid go templates + for _, env := range r.Spec.PodTemplate.ExtraEnv { + rawValue := env.Value + _, err := template.New("value").Funcs(template.FuncMap{"httpPathPrefix": func(_ string) string { return "" }}).Parse(rawValue) + if err != nil { + err = fmt.Errorf("failed to parse value %q: %v", rawValue, err) + return nil, err + } + } + return nil, nil } diff --git a/workspaces/controller/api/v1beta1/workspacekind_webhook_test.go b/workspaces/controller/api/v1beta1/workspacekind_webhook_test.go index c5780e37..d1cacc99 100644 --- a/workspaces/controller/api/v1beta1/workspacekind_webhook_test.go +++ b/workspaces/controller/api/v1beta1/workspacekind_webhook_test.go @@ -20,19 +20,84 @@ import ( "fmt" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + "time" ) var _ = Describe("WorkspaceKind Webhook", func() { + const ( + namespaceName = "default" + + // how long to wait in "Eventually" blocks + timeout = time.Second * 10 + + // how long to wait in "Consistently" blocks + duration = time.Second * 10 + + // how frequently to poll for conditions + interval = time.Millisecond * 250 + ) + + Context("When creating WorkspaceKind under Validating Webhook", Ordered, func() { + + testCases := []struct { + description string + workspaceKind *WorkspaceKind + }{ + { + description: "should reject WorkspaceKind creation with cycles in ImageConfig options", + workspaceKind: NewExampleWorkspaceKindWithImageConfigCycle("wsk-webhook-image-config-cycle-test"), + }, + { + description: "should reject WorkspaceKind creation with cycles in PodConfig options", + workspaceKind: NewExampleWorkspaceKindWithPodConfigCycle("wsk-webhook-pod-config-cycle-test"), + }, + { + description: "should reject WorkspaceKind creation with invalid redirects in ImageConfig options", + workspaceKind: NewExampleWorkspaceKindWithInvalidImageConfig("wsk-webhook-image-config-invalid-test"), + }, + { + description: "should reject WorkspaceKind creation with invalid redirects in PodConfig options", + workspaceKind: NewExampleWorkspaceKindWithInvalidPodConfig("wsk-webhook-pod-config-invalid-test"), + }, + { + description: "should reject WorkspaceKind creation if the default ImageConfig option is missing", + workspaceKind: NewExampleWorkspaceKindWithInvalidDefaultImageConfig("wsk-webhook-image-config-default-test"), + }, + { + description: "should reject WorkspaceKind creation if the default PodConfig option is missing", + workspaceKind: NewExampleWorkspaceKindWithInvalidDefaultPodConfig("wsk-webhook-pod-config-default-test"), + }, + { + description: "should reject WorkspaceKind creation with non-unique ports in PodConfig", + workspaceKind: NewExampleWorkspaceKindWithDuplicatePorts("wsk-webhook-ports-port-not-unique-test"), + }, + { + description: "should reject WorkspaceKind creation if extraEnv[].value is not a valid Go template", + workspaceKind: NewExampleWorkspaceKindWithInvalidExtraEnvValue("wsk-webhook-extra-env-value-invalid-test"), + }, + } + + for _, tc := range testCases { + tc := tc // Create a new instance of tc to avoid capturing the loop variable. + It(tc.description, func() { + By("creating the WorkspaceKind") + Expect(k8sClient.Create(ctx, tc.workspaceKind)).ToNot(Succeed()) + }) + } + + }) + Context("When updating WorkspaceKind under Validating Webhook", Ordered, func() { var ( workspaceKindName string workspaceKindKey types.NamespacedName + workspaceKind *WorkspaceKind ) BeforeAll(func() { @@ -41,37 +106,119 @@ var _ = Describe("WorkspaceKind Webhook", func() { workspaceKindKey = types.NamespacedName{Name: workspaceKindName} By("creating the WorkspaceKind") - workspaceKind := NewExampleWorkspaceKind1(workspaceKindName) - Expect(k8sClient.Create(ctx, workspaceKind)).To(Succeed()) + createdWorkspaceKind := NewExampleWorkspaceKind(workspaceKindName) + Expect(k8sClient.Create(ctx, createdWorkspaceKind)).To(Succeed()) + + By("getting the created WorkspaceKind") + workspaceKind = &WorkspaceKind{} + Eventually(func() error { + return k8sClient.Get(ctx, workspaceKindKey, workspaceKind) + }, timeout, interval).Should(Succeed()) }) AfterAll(func() { By("deleting the WorkspaceKind") - workspaceKind := &WorkspaceKind{ - ObjectMeta: metav1.ObjectMeta{ - Name: workspaceKindName, - }, - } Expect(k8sClient.Delete(ctx, workspaceKind)).To(Succeed()) }) - It("should not allow updating immutable fields", func() { - By("getting the WorkspaceKind") - workspaceKind := &WorkspaceKind{} - Expect(k8sClient.Get(ctx, workspaceKindKey, workspaceKind)).To(Succeed()) - patch := client.MergeFrom(workspaceKind.DeepCopy()) - - By("failing to update the `spec.podTemplate.options.imageConfig.values[0].spec` field") - newWorkspaceKind := workspaceKind.DeepCopy() - newWorkspaceKind.Spec.PodTemplate.Options.ImageConfig.Values[0].Spec.Image = "new-image:latest" - Expect(k8sClient.Patch(ctx, newWorkspaceKind, patch)).NotTo(Succeed()) - - By("failing to update the `spec.podTemplate.options.podConfig.values[0].spec` field") - newWorkspaceKind = workspaceKind.DeepCopy() - newWorkspaceKind.Spec.PodTemplate.Options.PodConfig.Values[0].Spec.Resources.Requests[v1.ResourceCPU] = resource.MustParse("99") - Expect(k8sClient.Patch(ctx, newWorkspaceKind, patch)).NotTo(Succeed()) - }) + testCases := []struct { + description string + modifyKindFn func(*WorkspaceKind) + workspaceName *string + }{ + { + description: "should reject updates to imageConfig spec", + modifyKindFn: func(wsk *WorkspaceKind) { + wsk.Spec.PodTemplate.Options.ImageConfig.Values[0].Spec.Image = "new-image:latest" + }, + }, + { + description: "should reject updates to podConfig spec", + modifyKindFn: func(wsk *WorkspaceKind) { + wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Spec.Resources = &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1.5"), + }, + } + }, + }, + { + description: "should reject WorkspaceKind update with cycles in imageConfig options", + modifyKindFn: func(wsk *WorkspaceKind) { + wsk.Spec.PodTemplate.Options.ImageConfig.Values[1].Redirect = &OptionRedirect{To: "jupyterlab_scipy_190"} + }, + }, + { + description: "should reject WorkspaceKind update with invalid redirects in ImageConfig options", + modifyKindFn: func(wsk *WorkspaceKind) { + wsk.Spec.PodTemplate.Options.ImageConfig.Values[1].Redirect = &OptionRedirect{To: "invalid-image-config"} + }, + }, + { + description: "should reject WorkspaceKind update with cycles in PodConfig options", + modifyKindFn: func(wsk *WorkspaceKind) { + wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Redirect = &OptionRedirect{To: "small_cpu"} + wsk.Spec.PodTemplate.Options.PodConfig.Values[1].Redirect = &OptionRedirect{To: "tiny_cpu"} + + }, + }, + { + description: "should reject WorkspaceKind creation with invalid redirects in PodConfig options", + modifyKindFn: func(wsk *WorkspaceKind) { + wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Redirect = &OptionRedirect{To: "invalid-pod-config"} + }, + }, + { + description: "should reject updates to WorkspaceKind with missing default imageConfig", + modifyKindFn: func(wsk *WorkspaceKind) { + wsk.Spec.PodTemplate.Options.ImageConfig.Spawner.Default = "invalid-image-config" + }, + }, + { + description: "should reject updates to WorkspaceKind with missing default podConfig", + modifyKindFn: func(wsk *WorkspaceKind) { + wsk.Spec.PodTemplate.Options.PodConfig.Spawner.Default = "invalid-pod-config" + }, + }, + { + description: "should reject updates to WorkspaceKind if extraEnv[].value is not a valid Go template", + modifyKindFn: func(wsk *WorkspaceKind) { + wsk.Spec.PodTemplate.ExtraEnv[0].Value = `{{ httpPathPrefix "jupyterlab" }` + }, + }, + { + description: "should reject updates that remove ImageConfig in use", + modifyKindFn: func(wsk *WorkspaceKind) { + wsk.Spec.PodTemplate.Options.ImageConfig.Values = wsk.Spec.PodTemplate.Options.ImageConfig.Values[1:] + }, + workspaceName: ptr.To("ws-webhook-update-image-config-test"), + }, + { + description: "should reject updates that remove podConfig in use", + modifyKindFn: func(wsk *WorkspaceKind) { + wsk.Spec.PodTemplate.Options.PodConfig.Values = wsk.Spec.PodTemplate.Options.PodConfig.Values[1:] + }, + workspaceName: ptr.To("ws-webhook-update-pod-config-test"), + }, + } + + for _, tc := range testCases { + tc := tc // Create a new instance of tc to avoid capturing the loop variable. + It(tc.description, func() { + if tc.workspaceName != nil { + + By("creating a Workspace with the WorkspaceKind") + workspace := NewExampleWorkspace(*tc.workspaceName, namespaceName, workspaceKind.Name) + Expect(k8sClient.Create(ctx, workspace)).To(Succeed()) + } + + patch := client.MergeFrom(workspaceKind.DeepCopy()) + modifiedWorkspaceKind := workspaceKind.DeepCopy() + tc.modifyKindFn(modifiedWorkspaceKind) + Expect(k8sClient.Patch(ctx, modifiedWorkspaceKind, patch)).NotTo(Succeed()) + }) + } }) }) From f09dcf401bb123611b2f4a0180d438c712ca3774 Mon Sep 17 00:00:00 2001 From: Adem Baccara <71262172+Adembc@users.noreply.github.com> Date: Thu, 15 Aug 2024 22:35:13 +0100 Subject: [PATCH 3/8] re-add cert-manager Signed-off-by: Adem Baccara <71262172+Adembc@users.noreply.github.com> --- workspaces/controller/test/e2e/e2e_test.go | 6 ++++ workspaces/controller/test/utils/utils.go | 41 +++++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/workspaces/controller/test/e2e/e2e_test.go b/workspaces/controller/test/e2e/e2e_test.go index 5a120523..9dfbf334 100644 --- a/workspaces/controller/test/e2e/e2e_test.go +++ b/workspaces/controller/test/e2e/e2e_test.go @@ -63,6 +63,9 @@ var ( var _ = Describe("controller", Ordered, func() { BeforeAll(func() { + By("installing the cert-manager") + Expect(utils.InstallCertManager()).To(Succeed()) + projectDir, _ = utils.GetProjectDir() By("creating the controller namespace") @@ -117,6 +120,9 @@ var _ = Describe("controller", Ordered, func() { By("deleting CRDs") cmd = exec.Command("make", "uninstall") _, _ = utils.Run(cmd) + + By("uninstalling the cert-manager bundle") + utils.UninstallCertManager() }) Context("Operator", func() { diff --git a/workspaces/controller/test/utils/utils.go b/workspaces/controller/test/utils/utils.go index 7992e4ad..6641e4e3 100644 --- a/workspaces/controller/test/utils/utils.go +++ b/workspaces/controller/test/utils/utils.go @@ -25,6 +25,17 @@ import ( . "github.com/onsi/ginkgo/v2" //nolint:golint,revive ) +const ( + // use LTS version of cert-manager + + certManagerVersion = "v1.12.13" + certManagerURLTmpl = "https://github.com/jetstack/cert-manager/releases/download/%s/cert-manager.yaml" +) + +func warnError(err error) { + fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) +} + // Run executes the provided command within this context func Run(cmd *exec.Cmd) ([]byte, error) { dir, _ := GetProjectDir() @@ -45,7 +56,35 @@ func Run(cmd *exec.Cmd) ([]byte, error) { return output, nil } -// LoadImageToKindCluster loads a local docker image to the kind cluster +// UninstallCertManager uninstalls the cert manager +func UninstallCertManager() { + url := fmt.Sprintf(certManagerURLTmpl, certManagerVersion) + cmd := exec.Command("kubectl", "delete", "-f", url) + if _, err := Run(cmd); err != nil { + warnError(err) + } +} + +// InstallCertManager installs the cert manager bundle. +func InstallCertManager() error { + url := fmt.Sprintf(certManagerURLTmpl, certManagerVersion) + cmd := exec.Command("kubectl", "apply", "-f", url) + if _, err := Run(cmd); err != nil { + return err + } + // Wait for cert-manager-webhook to be ready, which can take time if cert-manager + // was re-installed after uninstalling on a cluster. + cmd = exec.Command("kubectl", "wait", "deployment.apps/cert-manager-webhook", + "--for", "condition=Available", + "--namespace", "cert-manager", + "--timeout", "5m", + ) + + _, err := Run(cmd) + return err +} + +// LoadImageToKindClusterWithName loads a local docker image to the kind cluster func LoadImageToKindClusterWithName(name string) error { var cluster string if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { From ac43adc06faef29db8f14cd501db5123e804840a Mon Sep 17 00:00:00 2001 From: Adem Baccara <71262172+Adembc@users.noreply.github.com> Date: Tue, 20 Aug 2024 21:59:45 +0100 Subject: [PATCH 4/8] refactor ValidateCreate and ValidateUpdate functions in wokrspaceKind webhook Signed-off-by: Adem Baccara <71262172+Adembc@users.noreply.github.com> --- .../api/v1beta1/workspacekind_webhook.go | 335 +++++++++--------- 1 file changed, 167 insertions(+), 168 deletions(-) diff --git a/workspaces/controller/api/v1beta1/workspacekind_webhook.go b/workspaces/controller/api/v1beta1/workspacekind_webhook.go index 138f2ef5..f40e174f 100644 --- a/workspaces/controller/api/v1beta1/workspacekind_webhook.go +++ b/workspaces/controller/api/v1beta1/workspacekind_webhook.go @@ -36,7 +36,7 @@ import ( const kbCacheWorkspaceKindField = ".spec.kind" // log is for logging in this package. -var workspacekindlog = logf.Log.WithName("workspacekind-resource") +var workspaceKindLog = logf.Log.WithName("workspacekind-resource") // SetupWebhookWithManager will setup the manager to manage the webhooks func (r *WorkspaceKind) SetupWebhookWithManager(mgr ctrl.Manager) error { @@ -45,8 +45,6 @@ func (r *WorkspaceKind) SetupWebhookWithManager(mgr ctrl.Manager) error { Complete() } -// TODO(user): EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! - // TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation. //+kubebuilder:webhook:path=/validate-kubeflow-org-v1beta1-workspacekind,mutating=false,failurePolicy=fail,sideEffects=None,groups=kubeflow.org,resources=workspacekinds,verbs=create;update,versions=v1beta1,name=vworkspacekind.kb.io,admissionReviewVersions=v1 @@ -54,85 +52,30 @@ var _ webhook.Validator = &WorkspaceKind{} // ValidateCreate implements webhook.Validator so a webhook will be registered for the type func (r *WorkspaceKind) ValidateCreate() (admission.Warnings, error) { - workspacekindlog.Info("validate create", "name", r.Name) - - // Reject cycles in imageConfig options - imageConfigIdMap := make(map[string]ImageConfigValue) - for _, v := range r.Spec.PodTemplate.Options.ImageConfig.Values { - // Ensure ports are unique - ports := make(map[int32]bool) - for _, port := range v.Spec.Ports { - if _, exists := ports[port.Port]; exists { - return nil, fmt.Errorf("duplicate port %d in imageConfig with id '%s'", port.Port, v.Id) - } - ports[port.Port] = true - } + workspaceKindLog.Info("validate create", "name", r.Name) - imageConfigIdMap[v.Id] = v - } - for _, currentImageConfig := range imageConfigIdMap { - // follow any redirects to get the desired imageConfig - desiredImageConfig := currentImageConfig - visitedNodes := map[string]bool{currentImageConfig.Id: true} - for { - if desiredImageConfig.Redirect == nil { - break - } - if visitedNodes[desiredImageConfig.Redirect.To] { - return nil, fmt.Errorf("imageConfig with id '%s' has a circular redirect", desiredImageConfig.Id) - } - nextNode, ok := imageConfigIdMap[desiredImageConfig.Redirect.To] - if !ok { - return nil, fmt.Errorf("imageConfig with id '%s' not found, was redirected from '%s'", desiredImageConfig.Redirect.To, desiredImageConfig.Id) - } - desiredImageConfig = nextNode - visitedNodes[desiredImageConfig.Id] = true - } + imageConfigValueMap, err := generateImageConfigAndValidatePorts(r.Spec.PodTemplate.Options.ImageConfig) + if err != nil { + return nil, err } - - // Reject cycles in podConfig options - podConfigIdMap := make(map[string]PodConfigValue) + podConfigValueMap := make(map[string]PodConfigValue) for _, v := range r.Spec.PodTemplate.Options.PodConfig.Values { - podConfigIdMap[v.Id] = v - } - for _, currentPodConfig := range podConfigIdMap { - // follow any redirects to get the desired podConfig - desiredPodConfig := currentPodConfig - visitedNodes := map[string]bool{currentPodConfig.Id: true} - for { - if desiredPodConfig.Redirect == nil { - break - } - if visitedNodes[desiredPodConfig.Redirect.To] { - return nil, fmt.Errorf("podConfig with id '%s' has a circular redirect", desiredPodConfig.Id) - } - nextNode, ok := podConfigIdMap[desiredPodConfig.Redirect.To] - if !ok { - return nil, fmt.Errorf("podConfig with id '%s' not found, was redirected from '%s'", desiredPodConfig.Redirect.To, desiredPodConfig.Id) - } - desiredPodConfig = nextNode - visitedNodes[desiredPodConfig.Id] = true - } + podConfigValueMap[v.Id] = v } - // Ensure the default image config is present - if _, ok := imageConfigIdMap[r.Spec.PodTemplate.Options.ImageConfig.Spawner.Default]; !ok { - return nil, fmt.Errorf("default image config with id '%s' is not found in spec.podTemplate.options.imageConfig.values", r.Spec.PodTemplate.Options.ImageConfig.Spawner.Default) + if err := validateImageConfigCycles(imageConfigValueMap); err != nil { + return nil, err + } + if err := validatePodConfigCycle(podConfigValueMap); err != nil { + return nil, err } - // Ensure the default pod config is present - if _, ok := podConfigIdMap[r.Spec.PodTemplate.Options.PodConfig.Spawner.Default]; !ok { - return nil, fmt.Errorf("default pod config with id '%s' is not found in spec.podTemplate.options.podConfig.values", r.Spec.PodTemplate.Options.PodConfig.Spawner.Default) + if err := ensureDefaultOptions(imageConfigValueMap, podConfigValueMap, r.Spec.PodTemplate.Options); err != nil { + return nil, err } - // Validate the extraEnv values are valid go templates - for _, env := range r.Spec.PodTemplate.ExtraEnv { - rawValue := env.Value - _, err := template.New("value").Funcs(template.FuncMap{"httpPathPrefix": func(_ string) string { return "" }}).Parse(rawValue) - if err != nil { - err = fmt.Errorf("failed to parse value %q: %v", rawValue, err) - return nil, err - } + if err := validateExtraEnv(r.Spec.PodTemplate.ExtraEnv); err != nil { + return nil, err } return nil, nil @@ -140,7 +83,7 @@ func (r *WorkspaceKind) ValidateCreate() (admission.Warnings, error) { // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type func (r *WorkspaceKind) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { - workspacekindlog.Info("validate update", "name", r.Name) + workspaceKindLog.Info("validate update", "name", r.Name) // Type assertion to convert the old runtime.Object to WorkspaceKind oldWorkspaceKind, ok := old.(*WorkspaceKind) @@ -148,86 +91,75 @@ func (r *WorkspaceKind) ValidateUpdate(old runtime.Object) (admission.Warnings, return nil, errors.New("old object is not a WorkspaceKind") } - // Validate ImageConfig is immutable - imageConfigSpecMap := make(map[string]ImageConfigSpec) - for _, v := range oldWorkspaceKind.Spec.PodTemplate.Options.ImageConfig.Values { - imageConfigSpecMap[v.Id] = v.Spec + imageConfigUsageCount, podConfigUsageCount, err := getConfigUsageCount(r.Name) + + imageConfigValueMap, err := generateAndValidateImageConfig(r.Spec.PodTemplate.Options.ImageConfig, oldWorkspaceKind.Spec.PodTemplate.Options.ImageConfig, imageConfigUsageCount) + if err != nil { + return nil, err } - updatedImageConfigSpecMap := make(map[string]ImageConfigSpec) - for _, v := range r.Spec.PodTemplate.Options.ImageConfig.Values { - updatedImageConfigSpecMap[v.Id] = v.Spec - if oldSpec, exists := imageConfigSpecMap[v.Id]; exists { - if !reflect.DeepEqual(oldSpec, v.Spec) { - return nil, fmt.Errorf("spec.podTemplate.options.imageConfig.values with id '%s' is immutable", v.Id) - } - } + + podConfigValueMap, err := generateAndValidatePodConfig(r.Spec.PodTemplate.Options.PodConfig, oldWorkspaceKind.Spec.PodTemplate.Options.PodConfig, podConfigUsageCount) + if err != nil { + return nil, err } - // Validate PodConfig is immutable - podConfigSpecMap := make(map[string]PodConfigSpec) - for _, v := range oldWorkspaceKind.Spec.PodTemplate.Options.PodConfig.Values { - podConfigSpecMap[v.Id] = v.Spec + if err := validateImageConfigCycles(imageConfigValueMap); err != nil { + return nil, err } - updatedPodConfigSpecMap := make(map[string]PodConfigSpec) - for _, v := range r.Spec.PodTemplate.Options.PodConfig.Values { - updatedPodConfigSpecMap[v.Id] = v.Spec - if oldSpec, exists := podConfigSpecMap[v.Id]; exists { - normalizePodConfigSpec(&oldSpec) - normalizePodConfigSpec(&v.Spec) - if !reflect.DeepEqual(oldSpec, v.Spec) { - return nil, fmt.Errorf("spec.podTemplate.options.podConfig.values with id '%s' is immutable", v.Id) - } - } + if err := validatePodConfigCycle(podConfigValueMap); err != nil { + return nil, err } - workspaces := &WorkspaceList{} - listOpts := &client.ListOptions{ - FieldSelector: fields.OneTermEqualSelector(kbCacheWorkspaceKindField, r.Name), - Namespace: corev1.NamespaceAll, + if err := ensureDefaultOptions(imageConfigValueMap, podConfigValueMap, r.Spec.PodTemplate.Options); err != nil { + return nil, err } - if err := k8sClient.List(context.Background(), workspaces, listOpts); err != nil { + + if err := validateExtraEnv(r.Spec.PodTemplate.ExtraEnv); err != nil { return nil, err } - usedImageConfig := make(map[string]int) - usedPodConfig := make(map[string]int) - for _, ws := range workspaces.Items { - usedImageConfig[ws.Spec.PodTemplate.Options.ImageConfig]++ - usedPodConfig[ws.Spec.PodTemplate.Options.PodConfig]++ - } - // Only allow removing option ids which are not used - for id, _ := range imageConfigSpecMap { - if _, exists := updatedImageConfigSpecMap[id]; !exists { - // check if this option is used by any workspace - if usedImageConfig[id] > 0 { - errMsg := fmt.Sprintf("spec.podTemplate.options.imageConfig.values with id '%s' is used by %d workspace", id, usedImageConfig[id]) - if usedImageConfig[id] > 1 { - errMsg += "s" - } - return nil, fmt.Errorf(errMsg) + return nil, nil +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type +func (r *WorkspaceKind) ValidateDelete() (admission.Warnings, error) { + workspaceKindLog.Info("validate delete", "name", r.Name) + + // TODO(user): fill in your validation logic upon object deletion. + return nil, nil +} + +func generateImageConfigAndValidatePorts(imageConfig ImageConfig) (map[string]ImageConfigValue, error) { + imageConfigValueMap := make(map[string]ImageConfigValue) + for _, v := range imageConfig.Values { + + ports := make(map[int32]bool) + for _, port := range v.Spec.Ports { + if _, exists := ports[port.Port]; exists { + return nil, fmt.Errorf("duplicate port %d in imageConfig with id '%s'", port.Port, v.Id) } + ports[port.Port] = true } + + imageConfigValueMap[v.Id] = v } - for id, _ := range podConfigSpecMap { - if _, exists := updatedPodConfigSpecMap[id]; !exists { - // check if this option is used by any workspace - if usedPodConfig[id] > 0 { - errMsg := fmt.Sprintf("spec.podTemplate.options.podConfig.values with id '%s' is used by %d workspace", id, usedPodConfig[id]) - if usedPodConfig[id] > 1 { - errMsg += "s" - } - return nil, fmt.Errorf(errMsg) - } - } + return imageConfigValueMap, nil +} + +func ensureDefaultOptions(imageConfigValueMap map[string]ImageConfigValue, podConfigValueMap map[string]PodConfigValue, workspaceOptions WorkspaceKindPodOptions) error { + if _, ok := imageConfigValueMap[workspaceOptions.ImageConfig.Spawner.Default]; !ok { + return fmt.Errorf("default image config with id '%s' is not found in spec.podTemplate.options.imageConfig.values", workspaceOptions.ImageConfig.Spawner.Default) } - // Reject cycles in image options - imageConfigIdMap := make(map[string]ImageConfigValue) - for _, v := range r.Spec.PodTemplate.Options.ImageConfig.Values { - imageConfigIdMap[v.Id] = v + if _, ok := podConfigValueMap[workspaceOptions.PodConfig.Spawner.Default]; !ok { + return fmt.Errorf("default pod config with id '%s' is not found in spec.podTemplate.options.podConfig.values", workspaceOptions.PodConfig.Spawner.Default) } - for _, currentImageConfig := range imageConfigIdMap { + return nil +} + +func validateImageConfigCycles(imageConfigValueMap map[string]ImageConfigValue) error { + for _, currentImageConfig := range imageConfigValueMap { // follow any redirects to get the desired imageConfig desiredImageConfig := currentImageConfig visitedNodes := map[string]bool{currentImageConfig.Id: true} @@ -236,23 +168,21 @@ func (r *WorkspaceKind) ValidateUpdate(old runtime.Object) (admission.Warnings, break } if visitedNodes[desiredImageConfig.Redirect.To] { - return nil, fmt.Errorf("imageConfig with id '%s' has a circular redirect", desiredImageConfig.Id) + return fmt.Errorf("imageConfig with id '%s' has a circular redirect", desiredImageConfig.Id) } - nextNode, ok := imageConfigIdMap[desiredImageConfig.Redirect.To] + nextNode, ok := imageConfigValueMap[desiredImageConfig.Redirect.To] if !ok { - return nil, fmt.Errorf("imageConfig with id '%s' not found, was redirected from '%s'", desiredImageConfig.Redirect.To, desiredImageConfig.Id) + return fmt.Errorf("imageConfig with id '%s' not found, was redirected from '%s'", desiredImageConfig.Redirect.To, desiredImageConfig.Id) } desiredImageConfig = nextNode visitedNodes[desiredImageConfig.Id] = true } } + return nil +} - // Reject cycles in pod options - podConfigIdMap := make(map[string]PodConfigValue) - for _, v := range r.Spec.PodTemplate.Options.PodConfig.Values { - podConfigIdMap[v.Id] = v - } - for _, currentPodConfig := range podConfigIdMap { +func validatePodConfigCycle(podConfigValueMap map[string]PodConfigValue) error { + for _, currentPodConfig := range podConfigValueMap { // follow any redirects to get the desired podConfig desiredPodConfig := currentPodConfig visitedNodes := map[string]bool{currentPodConfig.Id: true} @@ -261,46 +191,115 @@ func (r *WorkspaceKind) ValidateUpdate(old runtime.Object) (admission.Warnings, break } if visitedNodes[desiredPodConfig.Redirect.To] { - return nil, fmt.Errorf("podConfig with id '%s' has a circular redirect", desiredPodConfig.Id) + return fmt.Errorf("podConfig with id '%s' has a circular redirect", desiredPodConfig.Id) } - nextNode, ok := podConfigIdMap[desiredPodConfig.Redirect.To] + nextNode, ok := podConfigValueMap[desiredPodConfig.Redirect.To] if !ok { - return nil, fmt.Errorf("podConfig with id '%s' not found, was redirected from '%s'", desiredPodConfig.Redirect.To, desiredPodConfig.Id) + return fmt.Errorf("podConfig with id '%s' not found, was redirected from '%s'", desiredPodConfig.Redirect.To, desiredPodConfig.Id) } desiredPodConfig = nextNode visitedNodes[desiredPodConfig.Id] = true } } + return nil +} - // Ensure the default image config is present - if _, ok := imageConfigIdMap[r.Spec.PodTemplate.Options.ImageConfig.Spawner.Default]; !ok { - return nil, fmt.Errorf("default image config with id '%s' is not found in spec.podTemplate.options.imageConfig.values", r.Spec.PodTemplate.Options.ImageConfig.Spawner.Default) - } - - // Ensure the default pod config is present - if _, ok := podConfigIdMap[r.Spec.PodTemplate.Options.PodConfig.Spawner.Default]; !ok { - return nil, fmt.Errorf("default pod config with id '%s' is not found in spec.podTemplate.options.podConfig.values", r.Spec.PodTemplate.Options.PodConfig.Spawner.Default) - } - - // Validate the extraEnv values are valid go templates - for _, env := range r.Spec.PodTemplate.ExtraEnv { +func validateExtraEnv(extraEnv []corev1.EnvVar) error { + for _, env := range extraEnv { rawValue := env.Value _, err := template.New("value").Funcs(template.FuncMap{"httpPathPrefix": func(_ string) string { return "" }}).Parse(rawValue) if err != nil { err = fmt.Errorf("failed to parse value %q: %v", rawValue, err) - return nil, err + return err } } + return nil +} - return nil, nil +func getConfigUsageCount(workspaceKindName string) (map[string]int, map[string]int, error) { + workspaces := &WorkspaceList{} + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(kbCacheWorkspaceKindField, workspaceKindName), + Namespace: corev1.NamespaceAll, + } + if err := k8sClient.List(context.Background(), workspaces, listOpts); err != nil { + return nil, nil, err + } + + imageConfigUsageCount := make(map[string]int) + podConfigUsageCount := make(map[string]int) + for _, ws := range workspaces.Items { + imageConfigUsageCount[ws.Spec.PodTemplate.Options.ImageConfig]++ + podConfigUsageCount[ws.Spec.PodTemplate.Options.PodConfig]++ + } + return imageConfigUsageCount, podConfigUsageCount, nil } -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (r *WorkspaceKind) ValidateDelete() (admission.Warnings, error) { - workspacekindlog.Info("validate delete", "name", r.Name) +func generateAndValidateImageConfig(imageConfig ImageConfig, oldImageConfig ImageConfig, imageConfigUsageCount map[string]int) (map[string]ImageConfigValue, error) { + oldImageConfigValueMap := make(map[string]ImageConfigValue) + imageConfigValueMap := make(map[string]ImageConfigValue) - // TODO(user): fill in your validation logic upon object deletion. - return nil, nil + for _, v := range oldImageConfig.Values { + oldImageConfigValueMap[v.Id] = v + } + + for _, v := range imageConfig.Values { + + if oldImageConfigValue, exists := oldImageConfigValueMap[v.Id]; exists { + if !reflect.DeepEqual(oldImageConfigValue.Spec, v.Spec) { + return nil, fmt.Errorf("spec.podTemplate.options.imageConfig.values with id '%s' is immutable", v.Id) + } + } + imageConfigValueMap[v.Id] = v + } + + for id, _ := range oldImageConfigValueMap { + if _, exists := imageConfigValueMap[id]; !exists { + if imageConfigUsageCount[id] > 0 { + errMsg := fmt.Sprintf("spec.podTemplate.options.imageConfig.values with id '%s' is used by %d workspace", id, imageConfigUsageCount[id]) + if imageConfigUsageCount[id] > 1 { + errMsg += "s" + } + return nil, fmt.Errorf(errMsg) + } + } + } + return imageConfigValueMap, nil +} + +func generateAndValidatePodConfig(podConfig PodConfig, oldPodConfig PodConfig, podConfigUsageCount map[string]int) (map[string]PodConfigValue, error) { + oldPodConfigValueMap := make(map[string]PodConfigValue) + podConfigValueMap := make(map[string]PodConfigValue) + + for _, v := range oldPodConfig.Values { + oldPodConfigValueMap[v.Id] = v + } + + for _, v := range podConfig.Values { + if oldPodConfigValue, exists := oldPodConfigValueMap[v.Id]; exists { + normalizePodConfigSpec(&oldPodConfigValue.Spec) + normalizePodConfigSpec(&v.Spec) + + if !reflect.DeepEqual(oldPodConfigValue.Spec, v.Spec) { + return nil, fmt.Errorf("spec.podTemplate.options.podConfig.values with id '%s' is immutable", v.Id) + } + } + podConfigValueMap[v.Id] = v + } + + for id, _ := range oldPodConfigValueMap { + if _, exists := podConfigValueMap[id]; !exists { + if podConfigUsageCount[id] > 0 { + errMsg := fmt.Sprintf("spec.podTemplate.options.podConfig.values with id '%s' is used by %d workspace", id, podConfigUsageCount[id]) + if podConfigUsageCount[id] > 1 { + errMsg += "s" + } + return nil, fmt.Errorf(errMsg) + } + } + } + + return podConfigValueMap, nil } func normalizePodConfigSpec(spec *PodConfigSpec) { From 3057797281ba2fe630af823dff45769f6b4bc959 Mon Sep 17 00:00:00 2001 From: Adem Baccara <71262172+Adembc@users.noreply.github.com> Date: Mon, 26 Aug 2024 16:30:56 +0100 Subject: [PATCH 5/8] add e2e test for webhooks Signed-off-by: Adem Baccara <71262172+Adembc@users.noreply.github.com> --- .../api/v1beta1/workspace_webhook.go | 68 ++++- .../api/v1beta1/workspacekind_webhook.go | 278 +++++++++++------- .../api/v1beta1/workspacekind_webhook_test.go | 6 +- .../api/v1beta1/zz_generated.deepcopy.go | 19 ++ .../controller/config/crd/kustomization.yaml | 4 +- .../controller/workspace_controller.go | 31 +- workspaces/controller/test/e2e/e2e_test.go | 9 + 7 files changed, 275 insertions(+), 140 deletions(-) diff --git a/workspaces/controller/api/v1beta1/workspace_webhook.go b/workspaces/controller/api/v1beta1/workspace_webhook.go index f2d34d19..979a1d1e 100644 --- a/workspaces/controller/api/v1beta1/workspace_webhook.go +++ b/workspaces/controller/api/v1beta1/workspace_webhook.go @@ -29,7 +29,7 @@ import ( // log is for logging in this package. var ( - workspacelog = logf.Log.WithName("workspace-resource") + workspaceLog = logf.Log.WithName("workspace-resource") k8sClient client.Client ) @@ -41,8 +41,6 @@ func (r *Workspace) SetupWebhookWithManager(mgr ctrl.Manager) error { Complete() } -// TODO(user): EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! - // TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation. //+kubebuilder:webhook:path=/validate-kubeflow-org-v1beta1-workspace,mutating=false,failurePolicy=fail,sideEffects=None,groups=kubeflow.org,resources=workspaces,verbs=create;update,versions=v1beta1,name=vworkspace.kb.io,admissionReviewVersions=v1 @@ -50,7 +48,7 @@ var _ webhook.Validator = &Workspace{} // ValidateCreate implements webhook.Validator so a webhook will be registered for the type func (r *Workspace) ValidateCreate() (admission.Warnings, error) { - workspacelog.Info("validate create", "name", r.Name) + workspaceLog.Info("validate create", "name", r.Name) workspaceKindName := r.Spec.Kind workspaceKind := &WorkspaceKind{} @@ -58,20 +56,76 @@ func (r *Workspace) ValidateCreate() (admission.Warnings, error) { return nil, fmt.Errorf("workspace kind %s not found", workspaceKindName) } + var errorList ErrorList + if err := validateImageConfig(workspaceKind, r.Spec.PodTemplate.Options.ImageConfig); err != nil { + errorList = append(errorList, err.Error()) + } + if err := validatePodConfig(workspaceKind, r.Spec.PodTemplate.Options.PodConfig); err != nil { + errorList = append(errorList, err.Error()) + } + + if len(errorList) > 0 { + return nil, errorList + } + return nil, nil } // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type func (r *Workspace) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { - workspacelog.Info("validate update", "name", r.Name) + workspaceLog.Info("validate update", "name", r.Name) - // TODO(user): fill in your validation logic upon object update. + oldWorkspace, ok := old.(*Workspace) + if !ok { + return nil, fmt.Errorf("old object is not a workspace") + } + workspaceKindName := r.Spec.Kind + workspaceKind := &WorkspaceKind{} + if err := k8sClient.Get(context.Background(), client.ObjectKey{Name: workspaceKindName}, workspaceKind); err != nil { + return nil, fmt.Errorf("workspace kind %s not found", workspaceKindName) + } + var errorList ErrorList + + if r.Spec.PodTemplate.Options.ImageConfig != oldWorkspace.Spec.PodTemplate.Options.ImageConfig { + if err := validateImageConfig(workspaceKind, r.Spec.PodTemplate.Options.ImageConfig); err != nil { + errorList = append(errorList, err.Error()) + } + } + if r.Spec.PodTemplate.Options.PodConfig != oldWorkspace.Spec.PodTemplate.Options.PodConfig { + if err := validatePodConfig(workspaceKind, r.Spec.PodTemplate.Options.PodConfig); err != nil { + errorList = append(errorList, err.Error()) + } + } + + if len(errorList) > 1 { + return nil, errorList + } return nil, nil } +// validateImageConfig checks if the selected imageConfig is valid +func validateImageConfig(workspaceKind *WorkspaceKind, imageConfigID string) error { + for _, imageConfig := range workspaceKind.Spec.PodTemplate.Options.ImageConfig.Values { + if imageConfig.Id == imageConfigID { + return nil + } + } + return fmt.Errorf("imageConfig %s not found in workspace kind %s", imageConfigID, workspaceKind.Name) +} + +// validatePodConfig checks if the selected podConfig is valid +func validatePodConfig(workspaceKind *WorkspaceKind, podConfigID string) error { + for _, podConfig := range workspaceKind.Spec.PodTemplate.Options.PodConfig.Values { + if podConfig.Id == podConfigID { + return nil + } + } + return fmt.Errorf("podConfig %s not found in workspace kind %s", podConfigID, workspaceKind.Name) +} + // ValidateDelete implements webhook.Validator so a webhook will be registered for the type func (r *Workspace) ValidateDelete() (admission.Warnings, error) { - workspacelog.Info("validate delete", "name", r.Name) + workspaceLog.Info("validate delete", "name", r.Name) // TODO(user): fill in your validation logic upon object deletion. return nil, nil diff --git a/workspaces/controller/api/v1beta1/workspacekind_webhook.go b/workspaces/controller/api/v1beta1/workspacekind_webhook.go index f40e174f..ab44c264 100644 --- a/workspaces/controller/api/v1beta1/workspacekind_webhook.go +++ b/workspaces/controller/api/v1beta1/workspacekind_webhook.go @@ -17,6 +17,7 @@ limitations under the License. package v1beta1 import ( + "bytes" "context" "errors" "fmt" @@ -30,9 +31,16 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + "strings" "text/template" ) +type ErrorList []string + +func (e ErrorList) Error() string { + return strings.Join(e, " , ") +} + const kbCacheWorkspaceKindField = ".spec.kind" // log is for logging in this package. @@ -59,8 +67,8 @@ func (r *WorkspaceKind) ValidateCreate() (admission.Warnings, error) { return nil, err } podConfigValueMap := make(map[string]PodConfigValue) - for _, v := range r.Spec.PodTemplate.Options.PodConfig.Values { - podConfigValueMap[v.Id] = v + for _, podConfigValue := range r.Spec.PodTemplate.Options.PodConfig.Values { + podConfigValueMap[podConfigValue.Id] = podConfigValue } if err := validateImageConfigCycles(imageConfigValueMap); err != nil { @@ -74,7 +82,7 @@ func (r *WorkspaceKind) ValidateCreate() (admission.Warnings, error) { return nil, err } - if err := validateExtraEnv(r.Spec.PodTemplate.ExtraEnv); err != nil { + if _, err := RenderAndValidateExtraEnv(r.Spec.PodTemplate.ExtraEnv, func(string) string { return "" }, false); err != nil { return nil, err } @@ -83,6 +91,87 @@ func (r *WorkspaceKind) ValidateCreate() (admission.Warnings, error) { // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type func (r *WorkspaceKind) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { + var ( + imageConfigUsageCount map[string]int + podConfigUsageCount map[string]int + isConfigUsageCountCalculated bool + err error + ) + + generateAndValidateImageConfig := func(imageConfig ImageConfig, oldImageConfig ImageConfig) (map[string]ImageConfigValue, map[string]ImageConfigValue, error) { + oldImageConfigValueMap := make(map[string]ImageConfigValue) + imageConfigValueMap := make(map[string]ImageConfigValue) + + for _, imageConfigValue := range oldImageConfig.Values { + oldImageConfigValueMap[imageConfigValue.Id] = imageConfigValue + } + + for _, imageConfigValue := range imageConfig.Values { + if oldImageConfigValue, exists := oldImageConfigValueMap[imageConfigValue.Id]; exists { + if !isConfigUsageCountCalculated { + imageConfigUsageCount, podConfigUsageCount, err = getConfigUsageCount(r.Name) + if err != nil { + return nil, nil, err + } + isConfigUsageCountCalculated = true + } + if imageConfigUsageCount[imageConfigValue.Id] > 0 && !reflect.DeepEqual(oldImageConfigValue.Spec, imageConfigValue.Spec) { + return nil, nil, fmt.Errorf("spec.podTemplate.options.imageConfig.values with id '%s' is immutable because it is used by %d workspace(s)", imageConfigValue.Id, imageConfigUsageCount[imageConfigValue.Id]) + } + } + imageConfigValueMap[imageConfigValue.Id] = imageConfigValue + } + + for id, _ := range oldImageConfigValueMap { + if _, exists := imageConfigValueMap[id]; !exists && imageConfigUsageCount[id] > 0 { + return nil, nil, fmt.Errorf("spec.podTemplate.options.imageConfig.values with id '%s' is used by %d workspace(s)", id, imageConfigUsageCount[id]) + } + } + return imageConfigValueMap, oldImageConfigValueMap, nil + } + generateAndValidatePodConfig := func(podConfig PodConfig, oldPodConfig PodConfig) (map[string]PodConfigValue, map[string]PodConfigValue, error) { + oldPodConfigValueMap := make(map[string]PodConfigValue) + podConfigValueMap := make(map[string]PodConfigValue) + + for _, podConfigValue := range oldPodConfig.Values { + oldPodConfigValueMap[podConfigValue.Id] = podConfigValue + } + + for _, podConfigValue := range podConfig.Values { + if oldPodConfigValue, exists := oldPodConfigValueMap[podConfigValue.Id]; exists { + err := normalizePodConfigSpec(&oldPodConfigValue.Spec) + if err != nil { + return nil, nil, err + } + err = normalizePodConfigSpec(&podConfigValue.Spec) + if err != nil { + return nil, nil, err + } + if !isConfigUsageCountCalculated { + _, podConfigUsageCount, err = getConfigUsageCount(r.Name) + if err != nil { + return nil, nil, err + } + isConfigUsageCountCalculated = true + } + if podConfigUsageCount[podConfigValue.Id] > 0 && !reflect.DeepEqual(oldPodConfigValue.Spec, podConfigValue.Spec) { + return nil, nil, fmt.Errorf("spec.podTemplate.options.podConfig.values with id '%s' is immutable because it is used by %d workspace(s)", podConfigValue.Id, podConfigUsageCount[podConfigValue.Id]) + } + } + podConfigValueMap[podConfigValue.Id] = podConfigValue + } + + for id, _ := range oldPodConfigValueMap { + if _, exists := podConfigValueMap[id]; !exists { + if podConfigUsageCount[id] > 0 { + return nil, nil, fmt.Errorf("spec.podTemplate.options.podConfig.values with id '%s' is used by %d workspace(s)", id, podConfigUsageCount[id]) + } + } + } + + return podConfigValueMap, oldPodConfigValueMap, nil + } + workspaceKindLog.Info("validate update", "name", r.Name) // Type assertion to convert the old runtime.Object to WorkspaceKind @@ -91,32 +180,46 @@ func (r *WorkspaceKind) ValidateUpdate(old runtime.Object) (admission.Warnings, return nil, errors.New("old object is not a WorkspaceKind") } - imageConfigUsageCount, podConfigUsageCount, err := getConfigUsageCount(r.Name) - - imageConfigValueMap, err := generateAndValidateImageConfig(r.Spec.PodTemplate.Options.ImageConfig, oldWorkspaceKind.Spec.PodTemplate.Options.ImageConfig, imageConfigUsageCount) + imageConfigValueMap, oldImageConfigValueMap, err := generateAndValidateImageConfig( + r.Spec.PodTemplate.Options.ImageConfig, + oldWorkspaceKind.Spec.PodTemplate.Options.ImageConfig, + ) if err != nil { return nil, err } - - podConfigValueMap, err := generateAndValidatePodConfig(r.Spec.PodTemplate.Options.PodConfig, oldWorkspaceKind.Spec.PodTemplate.Options.PodConfig, podConfigUsageCount) - if err != nil { - return nil, err + if !reflect.DeepEqual(imageConfigValueMap, oldImageConfigValueMap) { + if err := validateImageConfigCycles(imageConfigValueMap); err != nil { + return nil, err + } } - if err := validateImageConfigCycles(imageConfigValueMap); err != nil { + podConfigValueMap, oldPodConfigValueMap, err := generateAndValidatePodConfig( + r.Spec.PodTemplate.Options.PodConfig, + oldWorkspaceKind.Spec.PodTemplate.Options.PodConfig, + ) + if err != nil { return nil, err } - if err := validatePodConfigCycle(podConfigValueMap); err != nil { - return nil, err + if !reflect.DeepEqual(podConfigValueMap, oldPodConfigValueMap) { + if err := validatePodConfigCycle(podConfigValueMap); err != nil { + return nil, err + } } - if err := ensureDefaultOptions(imageConfigValueMap, podConfigValueMap, r.Spec.PodTemplate.Options); err != nil { - return nil, err + if !reflect.DeepEqual(imageConfigValueMap, oldImageConfigValueMap) || + !reflect.DeepEqual(podConfigValueMap, oldPodConfigValueMap) || + r.Spec.PodTemplate.Options.ImageConfig.Spawner.Default != oldWorkspaceKind.Spec.PodTemplate.Options.ImageConfig.Spawner.Default || + r.Spec.PodTemplate.Options.PodConfig.Spawner.Default != oldWorkspaceKind.Spec.PodTemplate.Options.PodConfig.Spawner.Default { + if err := ensureDefaultOptions(imageConfigValueMap, podConfigValueMap, r.Spec.PodTemplate.Options); err != nil { + return nil, err + } } - if err := validateExtraEnv(r.Spec.PodTemplate.ExtraEnv); err != nil { - return nil, err + if !reflect.DeepEqual(r.Spec.PodTemplate.ExtraEnv, oldWorkspaceKind.Spec.PodTemplate.ExtraEnv) { + if _, err := RenderAndValidateExtraEnv(r.Spec.PodTemplate.ExtraEnv, func(string) string { return "" }, false); err != nil { + return nil, err + } } return nil, nil @@ -125,35 +228,44 @@ func (r *WorkspaceKind) ValidateUpdate(old runtime.Object) (admission.Warnings, // ValidateDelete implements webhook.Validator so a webhook will be registered for the type func (r *WorkspaceKind) ValidateDelete() (admission.Warnings, error) { workspaceKindLog.Info("validate delete", "name", r.Name) - - // TODO(user): fill in your validation logic upon object deletion. + if r.Status.Workspaces > 0 { + return nil, fmt.Errorf("can not delete workspaceKind %s becuase it is used by %d workspace(s)", r.Name, r.Status.Workspaces) + } return nil, nil } func generateImageConfigAndValidatePorts(imageConfig ImageConfig) (map[string]ImageConfigValue, error) { + var errorList ErrorList imageConfigValueMap := make(map[string]ImageConfigValue) - for _, v := range imageConfig.Values { + for _, imageConfigValue := range imageConfig.Values { ports := make(map[int32]bool) - for _, port := range v.Spec.Ports { + for _, port := range imageConfigValue.Spec.Ports { if _, exists := ports[port.Port]; exists { - return nil, fmt.Errorf("duplicate port %d in imageConfig with id '%s'", port.Port, v.Id) + errorList = append(errorList, fmt.Sprintf("duplicate port %d in imageConfig with id '%s'", port.Port, imageConfigValue.Id)) } ports[port.Port] = true } - imageConfigValueMap[v.Id] = v + imageConfigValueMap[imageConfigValue.Id] = imageConfigValue + } + if len(errorList) > 0 { + return imageConfigValueMap, errorList } return imageConfigValueMap, nil } func ensureDefaultOptions(imageConfigValueMap map[string]ImageConfigValue, podConfigValueMap map[string]PodConfigValue, workspaceOptions WorkspaceKindPodOptions) error { + var errorList ErrorList if _, ok := imageConfigValueMap[workspaceOptions.ImageConfig.Spawner.Default]; !ok { - return fmt.Errorf("default image config with id '%s' is not found in spec.podTemplate.options.imageConfig.values", workspaceOptions.ImageConfig.Spawner.Default) + errorList = append(errorList, fmt.Sprintf("default image config with id '%s' is not found in spec.podTemplate.options.imageConfig.values", workspaceOptions.ImageConfig.Spawner.Default)) } if _, ok := podConfigValueMap[workspaceOptions.PodConfig.Spawner.Default]; !ok { - return fmt.Errorf("default pod config with id '%s' is not found in spec.podTemplate.options.podConfig.values", workspaceOptions.PodConfig.Spawner.Default) + errorList = append(errorList, fmt.Sprintf("default pod config with id '%s' is not found in spec.podTemplate.options.podConfig.values", workspaceOptions.PodConfig.Spawner.Default)) + } + if len(errorList) > 0 { + return errorList } return nil } @@ -204,16 +316,36 @@ func validatePodConfigCycle(podConfigValueMap map[string]PodConfigValue) error { return nil } -func validateExtraEnv(extraEnv []corev1.EnvVar) error { +func RenderAndValidateExtraEnv(extraEnv []corev1.EnvVar, templateFunc func(string) string, shouldExecTemplate bool) ([]corev1.EnvVar, error) { + var errorList ErrorList + containerEnv := make([]corev1.EnvVar, 0) + for _, env := range extraEnv { - rawValue := env.Value - _, err := template.New("value").Funcs(template.FuncMap{"httpPathPrefix": func(_ string) string { return "" }}).Parse(rawValue) - if err != nil { - err = fmt.Errorf("failed to parse value %q: %v", rawValue, err) - return err + if env.Value != "" { + rawValue := env.Value + tmpl, err := template.New("value").Funcs(template.FuncMap{"httpPathPrefix": templateFunc}).Parse(rawValue) + if err != nil { + errorList = append(errorList, fmt.Sprintf("failed to parse value %q: %v", rawValue, err)) + continue + } + if shouldExecTemplate { + var buf bytes.Buffer + err = tmpl.Execute(&buf, nil) + if err != nil { + errorList = append(errorList, fmt.Sprintf("failed to execute template for extraEnv '%s': %v", env.Name, err)) + continue + } + + env.Value = buf.String() + } } + containerEnv = append(containerEnv, env) } - return nil + if len(errorList) > 0 { + return nil, errorList + } + return containerEnv, nil + } func getConfigUsageCount(workspaceKindName string) (map[string]int, map[string]int, error) { @@ -235,74 +367,7 @@ func getConfigUsageCount(workspaceKindName string) (map[string]int, map[string]i return imageConfigUsageCount, podConfigUsageCount, nil } -func generateAndValidateImageConfig(imageConfig ImageConfig, oldImageConfig ImageConfig, imageConfigUsageCount map[string]int) (map[string]ImageConfigValue, error) { - oldImageConfigValueMap := make(map[string]ImageConfigValue) - imageConfigValueMap := make(map[string]ImageConfigValue) - - for _, v := range oldImageConfig.Values { - oldImageConfigValueMap[v.Id] = v - } - - for _, v := range imageConfig.Values { - - if oldImageConfigValue, exists := oldImageConfigValueMap[v.Id]; exists { - if !reflect.DeepEqual(oldImageConfigValue.Spec, v.Spec) { - return nil, fmt.Errorf("spec.podTemplate.options.imageConfig.values with id '%s' is immutable", v.Id) - } - } - imageConfigValueMap[v.Id] = v - } - - for id, _ := range oldImageConfigValueMap { - if _, exists := imageConfigValueMap[id]; !exists { - if imageConfigUsageCount[id] > 0 { - errMsg := fmt.Sprintf("spec.podTemplate.options.imageConfig.values with id '%s' is used by %d workspace", id, imageConfigUsageCount[id]) - if imageConfigUsageCount[id] > 1 { - errMsg += "s" - } - return nil, fmt.Errorf(errMsg) - } - } - } - return imageConfigValueMap, nil -} - -func generateAndValidatePodConfig(podConfig PodConfig, oldPodConfig PodConfig, podConfigUsageCount map[string]int) (map[string]PodConfigValue, error) { - oldPodConfigValueMap := make(map[string]PodConfigValue) - podConfigValueMap := make(map[string]PodConfigValue) - - for _, v := range oldPodConfig.Values { - oldPodConfigValueMap[v.Id] = v - } - - for _, v := range podConfig.Values { - if oldPodConfigValue, exists := oldPodConfigValueMap[v.Id]; exists { - normalizePodConfigSpec(&oldPodConfigValue.Spec) - normalizePodConfigSpec(&v.Spec) - - if !reflect.DeepEqual(oldPodConfigValue.Spec, v.Spec) { - return nil, fmt.Errorf("spec.podTemplate.options.podConfig.values with id '%s' is immutable", v.Id) - } - } - podConfigValueMap[v.Id] = v - } - - for id, _ := range oldPodConfigValueMap { - if _, exists := podConfigValueMap[id]; !exists { - if podConfigUsageCount[id] > 0 { - errMsg := fmt.Sprintf("spec.podTemplate.options.podConfig.values with id '%s' is used by %d workspace", id, podConfigUsageCount[id]) - if podConfigUsageCount[id] > 1 { - errMsg += "s" - } - return nil, fmt.Errorf(errMsg) - } - } - } - - return podConfigValueMap, nil -} - -func normalizePodConfigSpec(spec *PodConfigSpec) { +func normalizePodConfigSpec(spec *PodConfigSpec) (err error) { // Normalize NodeSelector if spec.NodeSelector != nil && len(spec.NodeSelector) == 0 { spec.NodeSelector = nil @@ -319,7 +384,11 @@ func normalizePodConfigSpec(spec *PodConfigSpec) { } if spec.Resources.Requests != nil { for key, value := range spec.Resources.Requests { - spec.Resources.Requests[key] = resource.MustParse(value.String()) + q, err := resource.ParseQuantity(value.String()) + if err != nil { + return err + } + spec.Resources.Requests[key] = q } } @@ -329,7 +398,12 @@ func normalizePodConfigSpec(spec *PodConfigSpec) { } if spec.Resources.Limits != nil { for key, value := range spec.Resources.Limits { - spec.Resources.Limits[key] = resource.MustParse(value.String()) + q, err := resource.ParseQuantity(value.String()) + if err != nil { + return err + } + spec.Resources.Limits[key] = q } } + return nil } diff --git a/workspaces/controller/api/v1beta1/workspacekind_webhook_test.go b/workspaces/controller/api/v1beta1/workspacekind_webhook_test.go index d1cacc99..6a120916 100644 --- a/workspaces/controller/api/v1beta1/workspacekind_webhook_test.go +++ b/workspaces/controller/api/v1beta1/workspacekind_webhook_test.go @@ -127,13 +127,14 @@ var _ = Describe("WorkspaceKind Webhook", func() { workspaceName *string }{ { - description: "should reject updates to imageConfig spec", + description: "should reject updates to used imageConfig spec", modifyKindFn: func(wsk *WorkspaceKind) { wsk.Spec.PodTemplate.Options.ImageConfig.Values[0].Spec.Image = "new-image:latest" }, + workspaceName: ptr.To("ws-webhook-update-image-config-spec-test"), }, { - description: "should reject updates to podConfig spec", + description: "should reject updates to used podConfig spec", modifyKindFn: func(wsk *WorkspaceKind) { wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Spec.Resources = &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ @@ -141,6 +142,7 @@ var _ = Describe("WorkspaceKind Webhook", func() { }, } }, + workspaceName: ptr.To("ws-webhook-update-pod-config-spec-test"), }, { description: "should reject WorkspaceKind update with cycles in imageConfig options", diff --git a/workspaces/controller/api/v1beta1/zz_generated.deepcopy.go b/workspaces/controller/api/v1beta1/zz_generated.deepcopy.go index 72655956..c1d8785a 100644 --- a/workspaces/controller/api/v1beta1/zz_generated.deepcopy.go +++ b/workspaces/controller/api/v1beta1/zz_generated.deepcopy.go @@ -85,6 +85,25 @@ func (in *ActivityProbeJupyter) DeepCopy() *ActivityProbeJupyter { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in ErrorList) DeepCopyInto(out *ErrorList) { + { + in := &in + *out = make(ErrorList, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ErrorList. +func (in ErrorList) DeepCopy() ErrorList { + if in == nil { + return nil + } + out := new(ErrorList) + in.DeepCopyInto(out) + return *out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPProxy) DeepCopyInto(out *HTTPProxy) { *out = *in diff --git a/workspaces/controller/config/crd/kustomization.yaml b/workspaces/controller/config/crd/kustomization.yaml index aac7d81b..642e7656 100644 --- a/workspaces/controller/config/crd/kustomization.yaml +++ b/workspaces/controller/config/crd/kustomization.yaml @@ -13,8 +13,8 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD -#- path: patches/cainjection_in_workspaces.yaml -#- path: patches/cainjection_in_workspacekinds.yaml +- path: patches/cainjection_in_workspaces.yaml +- path: patches/cainjection_in_workspacekinds.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # [WEBHOOK] To enable webhook, uncomment the following section diff --git a/workspaces/controller/internal/controller/workspace_controller.go b/workspaces/controller/internal/controller/workspace_controller.go index e34a374c..27aa5e41 100644 --- a/workspaces/controller/internal/controller/workspace_controller.go +++ b/workspaces/controller/internal/controller/workspace_controller.go @@ -17,14 +17,11 @@ limitations under the License. package controller import ( - "bytes" "context" "fmt" + "k8s.io/apimachinery/pkg/util/intstr" "reflect" "strings" - "text/template" - - "k8s.io/apimachinery/pkg/util/intstr" "github.com/kubeflow/notebooks/workspaces/controller/internal/helper" @@ -681,29 +678,9 @@ func generateStatefulSet(workspace *kubefloworgv1beta1.Workspace, workspaceKind } // generate container env - containerEnv := make([]corev1.EnvVar, len(workspaceKind.Spec.PodTemplate.ExtraEnv)) - for i, env := range workspaceKind.Spec.PodTemplate.ExtraEnv { - if env.Value != "" { - rawValue := env.Value - - tmpl, err := template.New("value"). - Funcs(template.FuncMap{"httpPathPrefix": httpPathPrefixFunc}). - Parse(rawValue) - if err != nil { - err = fmt.Errorf("failed to parse template for extraEnv '%s': %w", env.Name, err) - return nil, err - } - - var buf bytes.Buffer - err = tmpl.Execute(&buf, nil) - if err != nil { - err = fmt.Errorf("failed to execute template for extraEnv '%s': %w", env.Name, err) - return nil, err - } - - env.Value = buf.String() - } - containerEnv[i] = env + containerEnv, err := kubefloworgv1beta1.RenderAndValidateExtraEnv(workspaceKind.Spec.PodTemplate.ExtraEnv, httpPathPrefixFunc, true) + if err != nil { + return nil, err } // generate container resources diff --git a/workspaces/controller/test/e2e/e2e_test.go b/workspaces/controller/test/e2e/e2e_test.go index 9dfbf334..0e2d2f2a 100644 --- a/workspaces/controller/test/e2e/e2e_test.go +++ b/workspaces/controller/test/e2e/e2e_test.go @@ -310,6 +310,15 @@ var _ = Describe("controller", Ordered, func() { return err } Eventually(curlService, timeout, interval).Should(Succeed()) + + By("ensuring that an option in WorkspaceKind cannot be removed if it is currently in use") + EventuallyWithOffset(1, func() error { + // Attempt to remove an option from the WorkspaceKind that is currently in use + cmd := exec.Command("kubectl", "patch", "workspacekind", "jupyterlab", + "--type=json", "-p", `[{"op": "remove", "path": "/spec/podTemplate/options/imageConfig/values/1"}]`) + _, err := utils.Run(cmd) + return err + }, timeout, interval).ShouldNot(Succeed()) }) }) }) From ce5ea50940ef455ba8be5b5aa37707cf0a81c3c2 Mon Sep 17 00:00:00 2001 From: Mathew Wicks <5735406+thesuperzapper@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:07:13 -0700 Subject: [PATCH 6/8] mathew refactor 1 Signed-off-by: Mathew Wicks <5735406+thesuperzapper@users.noreply.github.com> --- .../api/v1beta1/workspace_webhook.go | 132 ---- .../api/v1beta1/workspacekind_webhook.go | 409 ------------ .../api/v1beta1/zz_generated.deepcopy.go | 21 +- workspaces/controller/cmd/main.go | 37 +- .../controller/config/webhook/manifests.yaml | 1 + .../internal/controller/suite_test.go | 13 +- .../controller/workspace_controller.go | 107 +--- .../controller/workspacekind_controller.go | 15 +- .../workspacekind_controller_test.go | 4 +- .../controller/internal/helper/graph.go | 55 ++ .../controller/internal/helper/graph_test.go | 110 ++++ .../controller/internal/helper/index.go | 77 +++ .../controller/internal/helper/suite_test.go | 39 ++ .../controller/internal/helper/template.go | 30 + .../webhook/suite_test.go} | 219 +++---- .../internal/webhook/workspace_webhook.go | 231 +++++++ .../webhook}/workspace_webhook_test.go | 20 +- .../internal/webhook/workspacekind_webhook.go | 595 ++++++++++++++++++ .../webhook}/workspacekind_webhook_test.go | 87 ++- 19 files changed, 1395 insertions(+), 807 deletions(-) delete mode 100644 workspaces/controller/api/v1beta1/workspace_webhook.go delete mode 100644 workspaces/controller/api/v1beta1/workspacekind_webhook.go create mode 100644 workspaces/controller/internal/helper/graph.go create mode 100644 workspaces/controller/internal/helper/graph_test.go create mode 100644 workspaces/controller/internal/helper/index.go create mode 100644 workspaces/controller/internal/helper/suite_test.go create mode 100644 workspaces/controller/internal/helper/template.go rename workspaces/controller/{api/v1beta1/webhook_suite_test.go => internal/webhook/suite_test.go} (69%) create mode 100644 workspaces/controller/internal/webhook/workspace_webhook.go rename workspaces/controller/{api/v1beta1 => internal/webhook}/workspace_webhook_test.go (86%) create mode 100644 workspaces/controller/internal/webhook/workspacekind_webhook.go rename workspaces/controller/{api/v1beta1 => internal/webhook}/workspacekind_webhook_test.go (76%) diff --git a/workspaces/controller/api/v1beta1/workspace_webhook.go b/workspaces/controller/api/v1beta1/workspace_webhook.go deleted file mode 100644 index 979a1d1e..00000000 --- a/workspaces/controller/api/v1beta1/workspace_webhook.go +++ /dev/null @@ -1,132 +0,0 @@ -/* -Copyright 2024. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "context" - "fmt" - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -// log is for logging in this package. -var ( - workspaceLog = logf.Log.WithName("workspace-resource") - k8sClient client.Client -) - -// SetupWebhookWithManager will setup the manager to manage the webhooks -func (r *Workspace) SetupWebhookWithManager(mgr ctrl.Manager) error { - k8sClient = mgr.GetClient() - return ctrl.NewWebhookManagedBy(mgr). - For(r). - Complete() -} - -// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation. -//+kubebuilder:webhook:path=/validate-kubeflow-org-v1beta1-workspace,mutating=false,failurePolicy=fail,sideEffects=None,groups=kubeflow.org,resources=workspaces,verbs=create;update,versions=v1beta1,name=vworkspace.kb.io,admissionReviewVersions=v1 - -var _ webhook.Validator = &Workspace{} - -// ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (r *Workspace) ValidateCreate() (admission.Warnings, error) { - workspaceLog.Info("validate create", "name", r.Name) - - workspaceKindName := r.Spec.Kind - workspaceKind := &WorkspaceKind{} - if err := k8sClient.Get(context.Background(), client.ObjectKey{Name: workspaceKindName}, workspaceKind); err != nil { - return nil, fmt.Errorf("workspace kind %s not found", workspaceKindName) - } - - var errorList ErrorList - if err := validateImageConfig(workspaceKind, r.Spec.PodTemplate.Options.ImageConfig); err != nil { - errorList = append(errorList, err.Error()) - } - if err := validatePodConfig(workspaceKind, r.Spec.PodTemplate.Options.PodConfig); err != nil { - errorList = append(errorList, err.Error()) - } - - if len(errorList) > 0 { - return nil, errorList - } - - return nil, nil -} - -// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (r *Workspace) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { - workspaceLog.Info("validate update", "name", r.Name) - - oldWorkspace, ok := old.(*Workspace) - if !ok { - return nil, fmt.Errorf("old object is not a workspace") - } - workspaceKindName := r.Spec.Kind - workspaceKind := &WorkspaceKind{} - if err := k8sClient.Get(context.Background(), client.ObjectKey{Name: workspaceKindName}, workspaceKind); err != nil { - return nil, fmt.Errorf("workspace kind %s not found", workspaceKindName) - } - var errorList ErrorList - - if r.Spec.PodTemplate.Options.ImageConfig != oldWorkspace.Spec.PodTemplate.Options.ImageConfig { - if err := validateImageConfig(workspaceKind, r.Spec.PodTemplate.Options.ImageConfig); err != nil { - errorList = append(errorList, err.Error()) - } - } - if r.Spec.PodTemplate.Options.PodConfig != oldWorkspace.Spec.PodTemplate.Options.PodConfig { - if err := validatePodConfig(workspaceKind, r.Spec.PodTemplate.Options.PodConfig); err != nil { - errorList = append(errorList, err.Error()) - } - } - - if len(errorList) > 1 { - return nil, errorList - } - return nil, nil -} - -// validateImageConfig checks if the selected imageConfig is valid -func validateImageConfig(workspaceKind *WorkspaceKind, imageConfigID string) error { - for _, imageConfig := range workspaceKind.Spec.PodTemplate.Options.ImageConfig.Values { - if imageConfig.Id == imageConfigID { - return nil - } - } - return fmt.Errorf("imageConfig %s not found in workspace kind %s", imageConfigID, workspaceKind.Name) -} - -// validatePodConfig checks if the selected podConfig is valid -func validatePodConfig(workspaceKind *WorkspaceKind, podConfigID string) error { - for _, podConfig := range workspaceKind.Spec.PodTemplate.Options.PodConfig.Values { - if podConfig.Id == podConfigID { - return nil - } - } - return fmt.Errorf("podConfig %s not found in workspace kind %s", podConfigID, workspaceKind.Name) -} - -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (r *Workspace) ValidateDelete() (admission.Warnings, error) { - workspaceLog.Info("validate delete", "name", r.Name) - - // TODO(user): fill in your validation logic upon object deletion. - return nil, nil -} diff --git a/workspaces/controller/api/v1beta1/workspacekind_webhook.go b/workspaces/controller/api/v1beta1/workspacekind_webhook.go deleted file mode 100644 index ab44c264..00000000 --- a/workspaces/controller/api/v1beta1/workspacekind_webhook.go +++ /dev/null @@ -1,409 +0,0 @@ -/* -Copyright 2024. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "bytes" - "context" - "errors" - "fmt" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/runtime" - "reflect" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - "strings" - "text/template" -) - -type ErrorList []string - -func (e ErrorList) Error() string { - return strings.Join(e, " , ") -} - -const kbCacheWorkspaceKindField = ".spec.kind" - -// log is for logging in this package. -var workspaceKindLog = logf.Log.WithName("workspacekind-resource") - -// SetupWebhookWithManager will setup the manager to manage the webhooks -func (r *WorkspaceKind) SetupWebhookWithManager(mgr ctrl.Manager) error { - return ctrl.NewWebhookManagedBy(mgr). - For(r). - Complete() -} - -// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation. -//+kubebuilder:webhook:path=/validate-kubeflow-org-v1beta1-workspacekind,mutating=false,failurePolicy=fail,sideEffects=None,groups=kubeflow.org,resources=workspacekinds,verbs=create;update,versions=v1beta1,name=vworkspacekind.kb.io,admissionReviewVersions=v1 - -var _ webhook.Validator = &WorkspaceKind{} - -// ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (r *WorkspaceKind) ValidateCreate() (admission.Warnings, error) { - workspaceKindLog.Info("validate create", "name", r.Name) - - imageConfigValueMap, err := generateImageConfigAndValidatePorts(r.Spec.PodTemplate.Options.ImageConfig) - if err != nil { - return nil, err - } - podConfigValueMap := make(map[string]PodConfigValue) - for _, podConfigValue := range r.Spec.PodTemplate.Options.PodConfig.Values { - podConfigValueMap[podConfigValue.Id] = podConfigValue - } - - if err := validateImageConfigCycles(imageConfigValueMap); err != nil { - return nil, err - } - if err := validatePodConfigCycle(podConfigValueMap); err != nil { - return nil, err - } - - if err := ensureDefaultOptions(imageConfigValueMap, podConfigValueMap, r.Spec.PodTemplate.Options); err != nil { - return nil, err - } - - if _, err := RenderAndValidateExtraEnv(r.Spec.PodTemplate.ExtraEnv, func(string) string { return "" }, false); err != nil { - return nil, err - } - - return nil, nil -} - -// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (r *WorkspaceKind) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { - var ( - imageConfigUsageCount map[string]int - podConfigUsageCount map[string]int - isConfigUsageCountCalculated bool - err error - ) - - generateAndValidateImageConfig := func(imageConfig ImageConfig, oldImageConfig ImageConfig) (map[string]ImageConfigValue, map[string]ImageConfigValue, error) { - oldImageConfigValueMap := make(map[string]ImageConfigValue) - imageConfigValueMap := make(map[string]ImageConfigValue) - - for _, imageConfigValue := range oldImageConfig.Values { - oldImageConfigValueMap[imageConfigValue.Id] = imageConfigValue - } - - for _, imageConfigValue := range imageConfig.Values { - if oldImageConfigValue, exists := oldImageConfigValueMap[imageConfigValue.Id]; exists { - if !isConfigUsageCountCalculated { - imageConfigUsageCount, podConfigUsageCount, err = getConfigUsageCount(r.Name) - if err != nil { - return nil, nil, err - } - isConfigUsageCountCalculated = true - } - if imageConfigUsageCount[imageConfigValue.Id] > 0 && !reflect.DeepEqual(oldImageConfigValue.Spec, imageConfigValue.Spec) { - return nil, nil, fmt.Errorf("spec.podTemplate.options.imageConfig.values with id '%s' is immutable because it is used by %d workspace(s)", imageConfigValue.Id, imageConfigUsageCount[imageConfigValue.Id]) - } - } - imageConfigValueMap[imageConfigValue.Id] = imageConfigValue - } - - for id, _ := range oldImageConfigValueMap { - if _, exists := imageConfigValueMap[id]; !exists && imageConfigUsageCount[id] > 0 { - return nil, nil, fmt.Errorf("spec.podTemplate.options.imageConfig.values with id '%s' is used by %d workspace(s)", id, imageConfigUsageCount[id]) - } - } - return imageConfigValueMap, oldImageConfigValueMap, nil - } - generateAndValidatePodConfig := func(podConfig PodConfig, oldPodConfig PodConfig) (map[string]PodConfigValue, map[string]PodConfigValue, error) { - oldPodConfigValueMap := make(map[string]PodConfigValue) - podConfigValueMap := make(map[string]PodConfigValue) - - for _, podConfigValue := range oldPodConfig.Values { - oldPodConfigValueMap[podConfigValue.Id] = podConfigValue - } - - for _, podConfigValue := range podConfig.Values { - if oldPodConfigValue, exists := oldPodConfigValueMap[podConfigValue.Id]; exists { - err := normalizePodConfigSpec(&oldPodConfigValue.Spec) - if err != nil { - return nil, nil, err - } - err = normalizePodConfigSpec(&podConfigValue.Spec) - if err != nil { - return nil, nil, err - } - if !isConfigUsageCountCalculated { - _, podConfigUsageCount, err = getConfigUsageCount(r.Name) - if err != nil { - return nil, nil, err - } - isConfigUsageCountCalculated = true - } - if podConfigUsageCount[podConfigValue.Id] > 0 && !reflect.DeepEqual(oldPodConfigValue.Spec, podConfigValue.Spec) { - return nil, nil, fmt.Errorf("spec.podTemplate.options.podConfig.values with id '%s' is immutable because it is used by %d workspace(s)", podConfigValue.Id, podConfigUsageCount[podConfigValue.Id]) - } - } - podConfigValueMap[podConfigValue.Id] = podConfigValue - } - - for id, _ := range oldPodConfigValueMap { - if _, exists := podConfigValueMap[id]; !exists { - if podConfigUsageCount[id] > 0 { - return nil, nil, fmt.Errorf("spec.podTemplate.options.podConfig.values with id '%s' is used by %d workspace(s)", id, podConfigUsageCount[id]) - } - } - } - - return podConfigValueMap, oldPodConfigValueMap, nil - } - - workspaceKindLog.Info("validate update", "name", r.Name) - - // Type assertion to convert the old runtime.Object to WorkspaceKind - oldWorkspaceKind, ok := old.(*WorkspaceKind) - if !ok { - return nil, errors.New("old object is not a WorkspaceKind") - } - - imageConfigValueMap, oldImageConfigValueMap, err := generateAndValidateImageConfig( - r.Spec.PodTemplate.Options.ImageConfig, - oldWorkspaceKind.Spec.PodTemplate.Options.ImageConfig, - ) - if err != nil { - return nil, err - } - if !reflect.DeepEqual(imageConfigValueMap, oldImageConfigValueMap) { - if err := validateImageConfigCycles(imageConfigValueMap); err != nil { - return nil, err - } - } - - podConfigValueMap, oldPodConfigValueMap, err := generateAndValidatePodConfig( - r.Spec.PodTemplate.Options.PodConfig, - oldWorkspaceKind.Spec.PodTemplate.Options.PodConfig, - ) - if err != nil { - return nil, err - } - - if !reflect.DeepEqual(podConfigValueMap, oldPodConfigValueMap) { - if err := validatePodConfigCycle(podConfigValueMap); err != nil { - return nil, err - } - } - - if !reflect.DeepEqual(imageConfigValueMap, oldImageConfigValueMap) || - !reflect.DeepEqual(podConfigValueMap, oldPodConfigValueMap) || - r.Spec.PodTemplate.Options.ImageConfig.Spawner.Default != oldWorkspaceKind.Spec.PodTemplate.Options.ImageConfig.Spawner.Default || - r.Spec.PodTemplate.Options.PodConfig.Spawner.Default != oldWorkspaceKind.Spec.PodTemplate.Options.PodConfig.Spawner.Default { - if err := ensureDefaultOptions(imageConfigValueMap, podConfigValueMap, r.Spec.PodTemplate.Options); err != nil { - return nil, err - } - } - - if !reflect.DeepEqual(r.Spec.PodTemplate.ExtraEnv, oldWorkspaceKind.Spec.PodTemplate.ExtraEnv) { - if _, err := RenderAndValidateExtraEnv(r.Spec.PodTemplate.ExtraEnv, func(string) string { return "" }, false); err != nil { - return nil, err - } - } - - return nil, nil -} - -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (r *WorkspaceKind) ValidateDelete() (admission.Warnings, error) { - workspaceKindLog.Info("validate delete", "name", r.Name) - if r.Status.Workspaces > 0 { - return nil, fmt.Errorf("can not delete workspaceKind %s becuase it is used by %d workspace(s)", r.Name, r.Status.Workspaces) - } - return nil, nil -} - -func generateImageConfigAndValidatePorts(imageConfig ImageConfig) (map[string]ImageConfigValue, error) { - var errorList ErrorList - imageConfigValueMap := make(map[string]ImageConfigValue) - for _, imageConfigValue := range imageConfig.Values { - - ports := make(map[int32]bool) - for _, port := range imageConfigValue.Spec.Ports { - if _, exists := ports[port.Port]; exists { - errorList = append(errorList, fmt.Sprintf("duplicate port %d in imageConfig with id '%s'", port.Port, imageConfigValue.Id)) - } - ports[port.Port] = true - } - - imageConfigValueMap[imageConfigValue.Id] = imageConfigValue - } - if len(errorList) > 0 { - return imageConfigValueMap, errorList - } - return imageConfigValueMap, nil -} - -func ensureDefaultOptions(imageConfigValueMap map[string]ImageConfigValue, podConfigValueMap map[string]PodConfigValue, workspaceOptions WorkspaceKindPodOptions) error { - var errorList ErrorList - if _, ok := imageConfigValueMap[workspaceOptions.ImageConfig.Spawner.Default]; !ok { - errorList = append(errorList, fmt.Sprintf("default image config with id '%s' is not found in spec.podTemplate.options.imageConfig.values", workspaceOptions.ImageConfig.Spawner.Default)) - } - - if _, ok := podConfigValueMap[workspaceOptions.PodConfig.Spawner.Default]; !ok { - errorList = append(errorList, fmt.Sprintf("default pod config with id '%s' is not found in spec.podTemplate.options.podConfig.values", workspaceOptions.PodConfig.Spawner.Default)) - } - if len(errorList) > 0 { - return errorList - } - return nil -} - -func validateImageConfigCycles(imageConfigValueMap map[string]ImageConfigValue) error { - for _, currentImageConfig := range imageConfigValueMap { - // follow any redirects to get the desired imageConfig - desiredImageConfig := currentImageConfig - visitedNodes := map[string]bool{currentImageConfig.Id: true} - for { - if desiredImageConfig.Redirect == nil { - break - } - if visitedNodes[desiredImageConfig.Redirect.To] { - return fmt.Errorf("imageConfig with id '%s' has a circular redirect", desiredImageConfig.Id) - } - nextNode, ok := imageConfigValueMap[desiredImageConfig.Redirect.To] - if !ok { - return fmt.Errorf("imageConfig with id '%s' not found, was redirected from '%s'", desiredImageConfig.Redirect.To, desiredImageConfig.Id) - } - desiredImageConfig = nextNode - visitedNodes[desiredImageConfig.Id] = true - } - } - return nil -} - -func validatePodConfigCycle(podConfigValueMap map[string]PodConfigValue) error { - for _, currentPodConfig := range podConfigValueMap { - // follow any redirects to get the desired podConfig - desiredPodConfig := currentPodConfig - visitedNodes := map[string]bool{currentPodConfig.Id: true} - for { - if desiredPodConfig.Redirect == nil { - break - } - if visitedNodes[desiredPodConfig.Redirect.To] { - return fmt.Errorf("podConfig with id '%s' has a circular redirect", desiredPodConfig.Id) - } - nextNode, ok := podConfigValueMap[desiredPodConfig.Redirect.To] - if !ok { - return fmt.Errorf("podConfig with id '%s' not found, was redirected from '%s'", desiredPodConfig.Redirect.To, desiredPodConfig.Id) - } - desiredPodConfig = nextNode - visitedNodes[desiredPodConfig.Id] = true - } - } - return nil -} - -func RenderAndValidateExtraEnv(extraEnv []corev1.EnvVar, templateFunc func(string) string, shouldExecTemplate bool) ([]corev1.EnvVar, error) { - var errorList ErrorList - containerEnv := make([]corev1.EnvVar, 0) - - for _, env := range extraEnv { - if env.Value != "" { - rawValue := env.Value - tmpl, err := template.New("value").Funcs(template.FuncMap{"httpPathPrefix": templateFunc}).Parse(rawValue) - if err != nil { - errorList = append(errorList, fmt.Sprintf("failed to parse value %q: %v", rawValue, err)) - continue - } - if shouldExecTemplate { - var buf bytes.Buffer - err = tmpl.Execute(&buf, nil) - if err != nil { - errorList = append(errorList, fmt.Sprintf("failed to execute template for extraEnv '%s': %v", env.Name, err)) - continue - } - - env.Value = buf.String() - } - } - containerEnv = append(containerEnv, env) - } - if len(errorList) > 0 { - return nil, errorList - } - return containerEnv, nil - -} - -func getConfigUsageCount(workspaceKindName string) (map[string]int, map[string]int, error) { - workspaces := &WorkspaceList{} - listOpts := &client.ListOptions{ - FieldSelector: fields.OneTermEqualSelector(kbCacheWorkspaceKindField, workspaceKindName), - Namespace: corev1.NamespaceAll, - } - if err := k8sClient.List(context.Background(), workspaces, listOpts); err != nil { - return nil, nil, err - } - - imageConfigUsageCount := make(map[string]int) - podConfigUsageCount := make(map[string]int) - for _, ws := range workspaces.Items { - imageConfigUsageCount[ws.Spec.PodTemplate.Options.ImageConfig]++ - podConfigUsageCount[ws.Spec.PodTemplate.Options.PodConfig]++ - } - return imageConfigUsageCount, podConfigUsageCount, nil -} - -func normalizePodConfigSpec(spec *PodConfigSpec) (err error) { - // Normalize NodeSelector - if spec.NodeSelector != nil && len(spec.NodeSelector) == 0 { - spec.NodeSelector = nil - } - - // Normalize Tolerations - if spec.Tolerations != nil && len(spec.Tolerations) == 0 { - spec.Tolerations = nil - } - - // Normalize ResourceRequests - if reflect.DeepEqual(spec.Resources.Requests, corev1.ResourceList{}) { - spec.Resources.Requests = nil - } - if spec.Resources.Requests != nil { - for key, value := range spec.Resources.Requests { - q, err := resource.ParseQuantity(value.String()) - if err != nil { - return err - } - spec.Resources.Requests[key] = q - } - } - - // Normalize ResourceLimits - if reflect.DeepEqual(spec.Resources.Limits, corev1.ResourceList{}) { - spec.Resources.Limits = nil - } - if spec.Resources.Limits != nil { - for key, value := range spec.Resources.Limits { - q, err := resource.ParseQuantity(value.String()) - if err != nil { - return err - } - spec.Resources.Limits[key] = q - } - } - return nil -} diff --git a/workspaces/controller/api/v1beta1/zz_generated.deepcopy.go b/workspaces/controller/api/v1beta1/zz_generated.deepcopy.go index c1d8785a..1beab4fd 100644 --- a/workspaces/controller/api/v1beta1/zz_generated.deepcopy.go +++ b/workspaces/controller/api/v1beta1/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ package v1beta1 import ( "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" + runtime "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -85,25 +85,6 @@ func (in *ActivityProbeJupyter) DeepCopy() *ActivityProbeJupyter { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in ErrorList) DeepCopyInto(out *ErrorList) { - { - in := &in - *out = make(ErrorList, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ErrorList. -func (in ErrorList) DeepCopy() ErrorList { - if in == nil { - return nil - } - out := new(ErrorList) - in.DeepCopyInto(out) - return *out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPProxy) DeepCopyInto(out *HTTPProxy) { *out = *in diff --git a/workspaces/controller/cmd/main.go b/workspaces/controller/cmd/main.go index e9f85fd2..82659c2f 100644 --- a/workspaces/controller/cmd/main.go +++ b/workspaces/controller/cmd/main.go @@ -21,6 +21,9 @@ import ( "flag" "os" + "github.com/kubeflow/notebooks/workspaces/controller/internal/helper" + webhookInternal "github.com/kubeflow/notebooks/workspaces/controller/internal/webhook" + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" @@ -35,7 +38,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" - "github.com/kubeflow/notebooks/workspaces/controller/internal/controller" + controllerInternal "github.com/kubeflow/notebooks/workspaces/controller/internal/controller" //+kubebuilder:scaffold:imports ) @@ -115,40 +118,56 @@ func main() { // the manager stops, so would be fine to enable this option. However, // if you are doing or is intended to do any operation such as perform cleanups // after the manager stops then its usage might be unsafe. - // LeaderElectionReleaseOnCancel: true, + // + // TODO: check if we are doing anything which would prevent us from using this option. + //LeaderElectionReleaseOnCancel: true, }) if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) } - if err = (&controller.WorkspaceReconciler{ + // setup field indexers on the manager cache. we use these indexes to efficiently + // query the cache for things like which Workspaces are using a particular WorkspaceKind + if err := helper.SetupManagerFieldIndexers(mgr); err != nil { + setupLog.Error(err, "unable to setup field indexers") + os.Exit(1) + } + + if err = (&controllerInternal.WorkspaceReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Workspace") os.Exit(1) } - if err = (&controller.WorkspaceKindReconciler{ + if err = (&controllerInternal.WorkspaceKindReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "WorkspaceKind") os.Exit(1) } + //+kubebuilder:scaffold:builder + if os.Getenv("ENABLE_WEBHOOKS") != "false" { - if err = (&kubefloworgv1beta1.WorkspaceKind{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "WorkspaceKind") + if err = (&webhookInternal.WorkspaceValidator{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "Workspace") os.Exit(1) } } if os.Getenv("ENABLE_WEBHOOKS") != "false" { - if err = (&kubefloworgv1beta1.Workspace{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "Workspace") + if err = (&webhookInternal.WorkspaceKindValidator{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "WorkspaceKind") os.Exit(1) } } - //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") diff --git a/workspaces/controller/config/webhook/manifests.yaml b/workspaces/controller/config/webhook/manifests.yaml index 04256eb3..d757a597 100644 --- a/workspaces/controller/config/webhook/manifests.yaml +++ b/workspaces/controller/config/webhook/manifests.yaml @@ -41,6 +41,7 @@ webhooks: operations: - CREATE - UPDATE + - DELETE resources: - workspacekinds sideEffects: None diff --git a/workspaces/controller/internal/controller/suite_test.go b/workspaces/controller/internal/controller/suite_test.go index f467c820..a4444fd6 100644 --- a/workspaces/controller/internal/controller/suite_test.go +++ b/workspaces/controller/internal/controller/suite_test.go @@ -41,6 +41,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" + "github.com/kubeflow/notebooks/workspaces/controller/internal/helper" //+kubebuilder:scaffold:imports ) @@ -101,26 +102,30 @@ var _ = BeforeSuite(func() { BindAddress: "0", // disable metrics serving }, }) - Expect(err).ToNot(HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) + + By("setting up the field indexers for the controller manager") + err = helper.SetupManagerFieldIndexers(k8sManager) + Expect(err).NotTo(HaveOccurred()) By("setting up the Workspace controller") err = (&WorkspaceReconciler{ Client: k8sManager.GetClient(), Scheme: k8sManager.GetScheme(), }).SetupWithManager(k8sManager) - Expect(err).ToNot(HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) By("setting up the WorkspaceKind controller") err = (&WorkspaceKindReconciler{ Client: k8sManager.GetClient(), Scheme: k8sManager.GetScheme(), }).SetupWithManager(k8sManager) - Expect(err).ToNot(HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) go func() { defer GinkgoRecover() err = k8sManager.Start(ctx) - Expect(err).ToNot(HaveOccurred(), "failed to run manager") + Expect(err).NotTo(HaveOccurred(), "failed to run manager") }() }) diff --git a/workspaces/controller/internal/controller/workspace_controller.go b/workspaces/controller/internal/controller/workspace_controller.go index 27aa5e41..8a7b0fcb 100644 --- a/workspaces/controller/internal/controller/workspace_controller.go +++ b/workspaces/controller/internal/controller/workspace_controller.go @@ -19,15 +19,13 @@ package controller import ( "context" "fmt" - "k8s.io/apimachinery/pkg/util/intstr" "reflect" "strings" - "github.com/kubeflow/notebooks/workspaces/controller/internal/helper" + "k8s.io/apimachinery/pkg/util/intstr" "github.com/go-logr/logr" - kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -44,6 +42,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + + kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" + "github.com/kubeflow/notebooks/workspaces/controller/internal/helper" ) const ( @@ -51,11 +52,6 @@ const ( workspaceNameLabel = "notebooks.kubeflow.org/workspace-name" workspaceSelectorLabel = "statefulset" - // KubeBuilder cache fields - kfCacheEventInvolvedObjectUidKey = ".involvedObject.uid" - kbCacheWorkspaceOwnerKey = ".metadata.controller" - kbCacheWorkspaceKindField = ".spec.kind" - // lengths for resource names generateNameSuffixLength = 6 maxServiceNameLength = 63 @@ -82,10 +78,6 @@ const ( stateMsgUnknown = "Workspace is in an unknown state" ) -var ( - apiGroupVersionStr = kubefloworgv1beta1.GroupVersion.String() -) - // WorkspaceReconciler reconciles a Workspace object type WorkspaceReconciler struct { client.Client @@ -145,8 +137,8 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // add finalizer to WorkspaceKind // NOTE: finalizers can only be added to non-deleted objects if workspaceKind.GetDeletionTimestamp().IsZero() { - if !controllerutil.ContainsFinalizer(workspaceKind, workspaceKindFinalizer) { - controllerutil.AddFinalizer(workspaceKind, workspaceKindFinalizer) + if !controllerutil.ContainsFinalizer(workspaceKind, WorkspaceKindFinalizer) { + controllerutil.AddFinalizer(workspaceKind, WorkspaceKindFinalizer) if err := r.Update(ctx, workspaceKind); err != nil { if apierrors.IsConflict(err) { log.V(2).Info("update conflict while adding finalizer to WorkspaceKind, will requeue") @@ -243,7 +235,7 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( var statefulSetName string ownedStatefulSets := &appsv1.StatefulSetList{} listOpts := &client.ListOptions{ - FieldSelector: fields.OneTermEqualSelector(kbCacheWorkspaceOwnerKey, workspace.Name), + FieldSelector: fields.OneTermEqualSelector(helper.IndexWorkspaceOwnerField, workspace.Name), Namespace: req.Namespace, } if err := r.List(ctx, ownedStatefulSets, listOpts); err != nil { @@ -307,7 +299,7 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( var serviceName string ownedServices := &corev1.ServiceList{} listOpts = &client.ListOptions{ - FieldSelector: fields.OneTermEqualSelector(kbCacheWorkspaceOwnerKey, workspace.Name), + FieldSelector: fields.OneTermEqualSelector(helper.IndexWorkspaceOwnerField, workspace.Name), Namespace: req.Namespace, } if err := r.List(ctx, ownedServices, listOpts); err != nil { @@ -390,57 +382,9 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // SetupWithManager sets up the controller with the Manager. func (r *WorkspaceReconciler) SetupWithManager(mgr ctrl.Manager) error { - // Index Event by `involvedObject.uid` - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &corev1.Event{}, kfCacheEventInvolvedObjectUidKey, func(rawObj client.Object) []string { - event := rawObj.(*corev1.Event) - if event.InvolvedObject.UID == "" { - return nil - } - return []string{string(event.InvolvedObject.UID)} - }); err != nil { - return err - } - // Index StatefulSet by owner - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &appsv1.StatefulSet{}, kbCacheWorkspaceOwnerKey, func(rawObj client.Object) []string { - statefulSet := rawObj.(*appsv1.StatefulSet) - owner := metav1.GetControllerOf(statefulSet) - if owner == nil { - return nil - } - if owner.APIVersion != apiGroupVersionStr || owner.Kind != "Workspace" { - return nil - } - return []string{owner.Name} - }); err != nil { - return err - } - - // Index Service by owner - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &corev1.Service{}, kbCacheWorkspaceOwnerKey, func(rawObj client.Object) []string { - service := rawObj.(*corev1.Service) - owner := metav1.GetControllerOf(service) - if owner == nil { - return nil - } - if owner.APIVersion != apiGroupVersionStr || owner.Kind != "Workspace" { - return nil - } - return []string{owner.Name} - }); err != nil { - return err - } - - // Index Workspace by WorkspaceKind - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &kubefloworgv1beta1.Workspace{}, kbCacheWorkspaceKindField, func(rawObj client.Object) []string { - ws := rawObj.(*kubefloworgv1beta1.Workspace) - if ws.Spec.Kind == "" { - return nil - } - return []string{ws.Spec.Kind} - }); err != nil { - return err - } + // NOTE: the SetupManagerFieldIndexers() helper in `helper/index.go` should have already been + // called on `mgr` by the time this function is called, so the indexes are already set up // function to convert pod events to reconcile requests for workspaces mapPodToRequest := func(ctx context.Context, object client.Object) []reconcile.Request { @@ -501,7 +445,7 @@ func (r *WorkspaceReconciler) updateWorkspaceState(ctx context.Context, log logr func (r *WorkspaceReconciler) mapWorkspaceKindToRequest(ctx context.Context, workspaceKind client.Object) []reconcile.Request { attachedWorkspaces := &kubefloworgv1beta1.WorkspaceList{} listOps := &client.ListOptions{ - FieldSelector: fields.OneTermEqualSelector(kbCacheWorkspaceKindField, workspaceKind.GetName()), + FieldSelector: fields.OneTermEqualSelector(helper.IndexWorkspaceKindField, workspaceKind.GetName()), Namespace: "", // fetch Workspaces in all namespaces } err := r.List(ctx, attachedWorkspaces, listOps) @@ -532,7 +476,7 @@ func getImageConfig(workspace *kubefloworgv1beta1.Workspace, workspaceKind *kube currentImageConfigKey := workspace.Spec.PodTemplate.Options.ImageConfig currentImageConfig, ok := imageConfigIdMap[currentImageConfigKey] if !ok { - return nil, nil, nil, fmt.Errorf("imageConfig with id '%s' not found", currentImageConfigKey) + return nil, nil, nil, fmt.Errorf("imageConfig with id %q not found", currentImageConfigKey) } // follow any redirects to get the desired imageConfig @@ -544,11 +488,11 @@ func getImageConfig(workspace *kubefloworgv1beta1.Workspace, workspaceKind *kube break } if visitedNodes[desiredImageConfig.Redirect.To] { - return nil, nil, nil, fmt.Errorf("imageConfig with id '%s' has a circular redirect", desiredImageConfig.Id) + return nil, nil, nil, fmt.Errorf("imageConfig with id %q has a circular redirect", desiredImageConfig.Id) } nextNode, ok := imageConfigIdMap[desiredImageConfig.Redirect.To] if !ok { - return nil, nil, nil, fmt.Errorf("imageConfig with id '%s' not found, was redirected from '%s'", desiredImageConfig.Redirect.To, desiredImageConfig.Id) + return nil, nil, nil, fmt.Errorf("imageConfig with id %q not found, was redirected from %q", desiredImageConfig.Redirect.To, desiredImageConfig.Id) } redirectChain = append(redirectChain, kubefloworgv1beta1.WorkspacePodOptionRedirectStep{ Source: desiredImageConfig.Id, @@ -577,7 +521,7 @@ func getPodConfig(workspace *kubefloworgv1beta1.Workspace, workspaceKind *kubefl currentPodConfigKey := workspace.Spec.PodTemplate.Options.PodConfig currentPodConfig, ok := podConfigIdMap[currentPodConfigKey] if !ok { - return nil, nil, nil, fmt.Errorf("podConfig with id '%s' not found", currentPodConfigKey) + return nil, nil, nil, fmt.Errorf("podConfig with id %q not found", currentPodConfigKey) } // follow any redirects to get the desired podConfig @@ -589,11 +533,11 @@ func getPodConfig(workspace *kubefloworgv1beta1.Workspace, workspaceKind *kubefl break } if visitedNodes[desiredPodConfig.Redirect.To] { - return nil, nil, nil, fmt.Errorf("podConfig with id '%s' has a circular redirect", desiredPodConfig.Id) + return nil, nil, nil, fmt.Errorf("podConfig with id %q has a circular redirect", desiredPodConfig.Id) } nextNode, ok := podConfigIdMap[desiredPodConfig.Redirect.To] if !ok { - return nil, nil, nil, fmt.Errorf("podConfig with id '%s' not found, was redirected from '%s'", desiredPodConfig.Redirect.To, desiredPodConfig.Id) + return nil, nil, nil, fmt.Errorf("podConfig with id %q not found, was redirected from %q", desiredPodConfig.Redirect.To, desiredPodConfig.Id) } redirectChain = append(redirectChain, kubefloworgv1beta1.WorkspacePodOptionRedirectStep{ Source: desiredPodConfig.Id, @@ -678,9 +622,18 @@ func generateStatefulSet(workspace *kubefloworgv1beta1.Workspace, workspaceKind } // generate container env - containerEnv, err := kubefloworgv1beta1.RenderAndValidateExtraEnv(workspaceKind.Spec.PodTemplate.ExtraEnv, httpPathPrefixFunc, true) - if err != nil { - return nil, err + containerEnv := make([]corev1.EnvVar, len(workspaceKind.Spec.PodTemplate.ExtraEnv)) + for i, env := range workspaceKind.Spec.PodTemplate.ExtraEnv { + env := env.DeepCopy() // copy to avoid modifying the original + if env.Value != "" { + rawValue := env.Value + outValue, err := helper.RenderExtraEnvValueTemplate(rawValue, httpPathPrefixFunc) + if err != nil { + return nil, fmt.Errorf("failed to render extraEnv %q: %w", env.Name, err) + } + env.Value = outValue + } + containerEnv[i] = *env } // generate container resources @@ -924,7 +877,7 @@ func (r *WorkspaceReconciler) generateWorkspaceStatus(ctx context.Context, log l // there might be StatefulSet events statefulSetEvents := &corev1.EventList{} listOpts := &client.ListOptions{ - FieldSelector: fields.OneTermEqualSelector(kfCacheEventInvolvedObjectUidKey, string(statefulSet.UID)), + FieldSelector: fields.OneTermEqualSelector(helper.IndexEventInvolvedObjectUidField, string(statefulSet.UID)), Namespace: statefulSet.Namespace, } if err := r.List(ctx, statefulSetEvents, listOpts); err != nil { diff --git a/workspaces/controller/internal/controller/workspacekind_controller.go b/workspaces/controller/internal/controller/workspacekind_controller.go index 996010b1..e6cd5c3b 100644 --- a/workspaces/controller/internal/controller/workspacekind_controller.go +++ b/workspaces/controller/internal/controller/workspacekind_controller.go @@ -35,10 +35,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" + "github.com/kubeflow/notebooks/workspaces/controller/internal/helper" ) const ( - workspaceKindFinalizer = "notebooks.kubeflow.org/workspacekind-protection" + WorkspaceKindFinalizer = "notebooks.kubeflow.org/workspacekind-protection" ) // WorkspaceKindReconciler reconciles a WorkspaceKind object @@ -73,7 +74,7 @@ func (r *WorkspaceKindReconciler) Reconcile(ctx context.Context, req ctrl.Reques // fetch all Workspaces that are using this WorkspaceKind workspaces := &kubefloworgv1beta1.WorkspaceList{} listOpts := &client.ListOptions{ - FieldSelector: fields.OneTermEqualSelector(kbCacheWorkspaceKindField, workspaceKind.Name), + FieldSelector: fields.OneTermEqualSelector(helper.IndexWorkspaceKindField, workspaceKind.Name), Namespace: "", // fetch Workspaces in all namespaces } if err := r.List(ctx, workspaces, listOpts); err != nil { @@ -84,8 +85,8 @@ func (r *WorkspaceKindReconciler) Reconcile(ctx context.Context, req ctrl.Reques // if no Workspaces are using this WorkspaceKind, remove the finalizer numWorkspace := len(workspaces.Items) if numWorkspace == 0 { - if controllerutil.ContainsFinalizer(workspaceKind, workspaceKindFinalizer) { - controllerutil.RemoveFinalizer(workspaceKind, workspaceKindFinalizer) + if controllerutil.ContainsFinalizer(workspaceKind, WorkspaceKindFinalizer) { + controllerutil.RemoveFinalizer(workspaceKind, WorkspaceKindFinalizer) if err := r.Update(ctx, workspaceKind); err != nil { if apierrors.IsConflict(err) { log.V(2).Info("update conflict while removing finalizer from WorkspaceKind, will requeue") @@ -146,10 +147,8 @@ func (r *WorkspaceKindReconciler) Reconcile(ctx context.Context, req ctrl.Reques // SetupWithManager sets up the controller with the Manager. func (r *WorkspaceKindReconciler) SetupWithManager(mgr ctrl.Manager) error { - // Index Workspace by WorkspaceKind - // NOTE: the Workspace index is defined in the SetupWithManager function of the WorkspaceReconciler. - // these controllers always share a manager (in both `main.go` and `suite_test.go`), - // so initializing the same index twice would result in a conflict. + // NOTE: the SetupManagerFieldIndexers() helper in `helper/index.go` should have already been + // called on `mgr` by the time this function is called, so the indexes are already set up // function to convert Workspace events to reconcile requests for WorkspaceKinds mapWorkspaceToRequest := func(ctx context.Context, object client.Object) []reconcile.Request { diff --git a/workspaces/controller/internal/controller/workspacekind_controller_test.go b/workspaces/controller/internal/controller/workspacekind_controller_test.go index dc062fd9..59e7dd15 100644 --- a/workspaces/controller/internal/controller/workspacekind_controller_test.go +++ b/workspaces/controller/internal/controller/workspacekind_controller_test.go @@ -203,7 +203,7 @@ var _ = Describe("WorkspaceKind Controller", func() { }, timeout, interval).Should(Equal(expectedStatus)) By("having a finalizer set on the WorkspaceKind") - Expect(workspaceKind.GetFinalizers()).To(ContainElement(workspaceKindFinalizer)) + Expect(workspaceKind.GetFinalizers()).To(ContainElement(WorkspaceKindFinalizer)) By("deleting the Workspace") Expect(k8sClient.Delete(ctx, workspace)).To(Succeed()) @@ -250,7 +250,7 @@ var _ = Describe("WorkspaceKind Controller", func() { By("deleting the WorkspaceKind") Expect(k8sClient.Delete(ctx, workspaceKind)).To(Succeed()) - Expect(k8sClient.Get(ctx, workspaceKindKey, workspaceKind)).ToNot(Succeed()) + Expect(k8sClient.Get(ctx, workspaceKindKey, workspaceKind)).NotTo(Succeed()) }) }) }) diff --git a/workspaces/controller/internal/helper/graph.go b/workspaces/controller/internal/helper/graph.go new file mode 100644 index 00000000..2cea2a5f --- /dev/null +++ b/workspaces/controller/internal/helper/graph.go @@ -0,0 +1,55 @@ +package helper + +// DetectGraphCycle checks if there is a cycle involving a given node in directed graph +// +// Assumptions: +// - all nodes have AT MOST one OUTGOING edge +// +// Parameters: +// - startNode: the node to start the cycle detection from +// - checkedNodes: a map of nodes which have already been checked for cycles (updated in-place if no cycle is detected) +// - edgeMap: a map representing the edges in the graph +// +// Returns: +// - returns nil, if no cycle is detected +// - returns a slice of nodes representing the cycle, if a cycle is detected +func DetectGraphCycle(startNode string, checkedNodes map[string]bool, edgeMap map[string]string) []string { + currentPath := make([]string, 0) + currentPathNodes := make(map[string]bool) + var currentNode = startNode + for { + // if the current node has already been checked, no cycle is detected + if checkedNodes[currentNode] { + break + } + + // if the current node is already in the current path, a cycle is detected + if currentPathNodes[currentNode] { + for i, node := range currentPath { + // the cycle starts from the first occurrence of the current node + if node == currentNode { + return currentPath[i:] + } + } + } + + // add the current node to the current path + currentPath = append(currentPath, currentNode) + currentPathNodes[currentNode] = true + + // get the next node + nextNode, exists := edgeMap[currentNode] + if !exists { + // if there is no outgoing edge, no cycle is detected + break + } + currentNode = nextNode + } + + // mark all nodes in the current path as checked + for node := range currentPathNodes { + checkedNodes[node] = true + } + + return nil +} diff --git a/workspaces/controller/internal/helper/graph_test.go b/workspaces/controller/internal/helper/graph_test.go new file mode 100644 index 00000000..2bfed2f2 --- /dev/null +++ b/workspaces/controller/internal/helper/graph_test.go @@ -0,0 +1,110 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helper + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("DetectGraphCycle", func() { + + It("should detect a simple cycle", func() { + startNode := "A" + checkedNodes := map[string]bool{} + edgeMap := map[string]string{"A": "B", "B": "C", "C": "A"} + + result := DetectGraphCycle(startNode, checkedNodes, edgeMap) + Expect(result).To(Equal([]string{"A", "B", "C"})) + }) + + It("should return nil for no cycle", func() { + startNode := "A" + checkedNodes := map[string]bool{} + edgeMap := map[string]string{"A": "B", "B": "C", "C": "D"} + + result := DetectGraphCycle(startNode, checkedNodes, edgeMap) + Expect(result).To(BeNil()) + Expect(checkedNodes).To(Equal(map[string]bool{"A": true, "B": true, "C": true, "D": true})) + }) + + It("should detect a self-loop cycle", func() { + startNode := "A" + checkedNodes := map[string]bool{} + edgeMap := map[string]string{"A": "A"} + + result := DetectGraphCycle(startNode, checkedNodes, edgeMap) + Expect(result).To(Equal([]string{"A"})) + }) + + It("should detect a cycle and ignore unconnected nodes", func() { + startNode := "A" + checkedNodes := map[string]bool{} + edgeMap := map[string]string{"A": "B", "B": "C", "C": "A", "D": "E"} + + result := DetectGraphCycle(startNode, checkedNodes, edgeMap) + Expect(result).To(Equal([]string{"A", "B", "C"})) + }) + + It("should detect cycles starting from different nodes in a complex graph", func() { + startNode := "A" + checkedNodes := map[string]bool{} + edgeMap := map[string]string{"A": "B", "B": "C", "C": "D", "D": "B", "E": "F"} + + result := DetectGraphCycle(startNode, checkedNodes, edgeMap) + Expect(result).To(Equal([]string{"B", "C", "D"})) + + startNode = "E" + result = DetectGraphCycle(startNode, checkedNodes, edgeMap) + Expect(result).To(BeNil()) + Expect(checkedNodes).To(Equal(map[string]bool{"E": true, "F": true})) + }) + + It("should detect cycles in a graph with multiple components", func() { + startNode := "X" + checkedNodes := map[string]bool{} + edgeMap := map[string]string{"A": "B", "B": "C", "C": "D", "D": "B", "X": "Y", "Y": "Z"} + + result := DetectGraphCycle(startNode, checkedNodes, edgeMap) + Expect(result).To(BeNil()) + Expect(checkedNodes).To(Equal(map[string]bool{"X": true, "Y": true, "Z": true})) + + startNode = "A" + result = DetectGraphCycle(startNode, checkedNodes, edgeMap) + Expect(result).To(Equal([]string{"B", "C", "D"})) + }) + + It("should return nil when starting from a node with no outgoing edge", func() { + startNode := "Z" + checkedNodes := map[string]bool{} + edgeMap := map[string]string{"A": "B", "B": "C", "C": "D", "D": "B", "X": "Y"} + + result := DetectGraphCycle(startNode, checkedNodes, edgeMap) + Expect(result).To(BeNil()) + Expect(checkedNodes).To(Equal(map[string]bool{"Z": true})) + }) + + It("should return nil when the start node has already been checked", func() { + startNode := "A" + checkedNodes := map[string]bool{"A": true, "B": true} + edgeMap := map[string]string{"A": "B", "B": "C", "C": "D", "D": "B"} + + result := DetectGraphCycle(startNode, checkedNodes, edgeMap) + Expect(result).To(BeNil()) + Expect(checkedNodes).To(Equal(map[string]bool{"A": true, "B": true})) + }) +}) diff --git a/workspaces/controller/internal/helper/index.go b/workspaces/controller/internal/helper/index.go new file mode 100644 index 00000000..22c24c00 --- /dev/null +++ b/workspaces/controller/internal/helper/index.go @@ -0,0 +1,77 @@ +package helper + +import ( + "context" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" +) + +const ( + IndexEventInvolvedObjectUidField = ".involvedObject.uid" + IndexWorkspaceOwnerField = ".metadata.controller" + IndexWorkspaceKindField = ".spec.kind" +) + +// SetupManagerFieldIndexers sets up field indexes on a controller-runtime manager +func SetupManagerFieldIndexers(mgr ctrl.Manager) error { + + // Index Event by `involvedObject.uid` + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &corev1.Event{}, IndexEventInvolvedObjectUidField, func(rawObj client.Object) []string { + event := rawObj.(*corev1.Event) + if event.InvolvedObject.UID == "" { + return nil + } + return []string{string(event.InvolvedObject.UID)} + }); err != nil { + return err + } + + // Index StatefulSet by its owner Workspace + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &appsv1.StatefulSet{}, IndexWorkspaceOwnerField, func(rawObj client.Object) []string { + statefulSet := rawObj.(*appsv1.StatefulSet) + owner := metav1.GetControllerOf(statefulSet) + if owner == nil { + return nil + } + if owner.APIVersion != kubefloworgv1beta1.GroupVersion.String() || owner.Kind != "Workspace" { + return nil + } + return []string{owner.Name} + }); err != nil { + return err + } + + // Index Service by its owner Workspace + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &corev1.Service{}, IndexWorkspaceOwnerField, func(rawObj client.Object) []string { + service := rawObj.(*corev1.Service) + owner := metav1.GetControllerOf(service) + if owner == nil { + return nil + } + if owner.APIVersion != kubefloworgv1beta1.GroupVersion.String() || owner.Kind != "Workspace" { + return nil + } + return []string{owner.Name} + }); err != nil { + return err + } + + // Index Workspace by WorkspaceKind + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &kubefloworgv1beta1.Workspace{}, IndexWorkspaceKindField, func(rawObj client.Object) []string { + ws := rawObj.(*kubefloworgv1beta1.Workspace) + if ws.Spec.Kind == "" { + return nil + } + return []string{ws.Spec.Kind} + }); err != nil { + return err + } + + return nil +} diff --git a/workspaces/controller/internal/helper/suite_test.go b/workspaces/controller/internal/helper/suite_test.go new file mode 100644 index 00000000..0aaabeee --- /dev/null +++ b/workspaces/controller/internal/helper/suite_test.go @@ -0,0 +1,39 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helper + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +func TestHelpers(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Helpers Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) +}) diff --git a/workspaces/controller/internal/helper/template.go b/workspaces/controller/internal/helper/template.go new file mode 100644 index 00000000..57d829a6 --- /dev/null +++ b/workspaces/controller/internal/helper/template.go @@ -0,0 +1,30 @@ +package helper + +import ( + "bytes" + "fmt" + "text/template" +) + +// RenderExtraEnvValueTemplate renders a single WorkspaceKind `spec.podTemplate.extraEnv[].value` string template +func RenderExtraEnvValueTemplate(rawValue string, httpPathPrefixFunc func(string) string) (string, error) { + + // Parse the raw value as a template + tmpl, err := template.New("value"). + Funcs(template.FuncMap{"httpPathPrefix": httpPathPrefixFunc}). + Parse(rawValue) + if err != nil { + err = fmt.Errorf("failed to parse template %q: %w", rawValue, err) + return "", err + } + + // Execute the template + var buf bytes.Buffer + err = tmpl.Execute(&buf, nil) + if err != nil { + err = fmt.Errorf("failed to execute template %q: %w", rawValue, err) + return "", err + } + + return buf.String(), nil +} diff --git a/workspaces/controller/api/v1beta1/webhook_suite_test.go b/workspaces/controller/internal/webhook/suite_test.go similarity index 69% rename from workspaces/controller/api/v1beta1/webhook_suite_test.go rename to workspaces/controller/internal/webhook/suite_test.go index 4885c0d5..1c480b13 100644 --- a/workspaces/controller/api/v1beta1/webhook_suite_test.go +++ b/workspaces/controller/internal/webhook/suite_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1beta1 +package webhook import ( "context" @@ -34,9 +34,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - admissionv1 "k8s.io/api/admission/v1" - //+kubebuilder:scaffold:imports - apimachineryruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -45,17 +43,23 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" + + kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" + "github.com/kubeflow/notebooks/workspaces/controller/internal/helper" + //+kubebuilder:scaffold:imports ) // These tests use Ginkgo (BDD-style Go testing framework). Refer to // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. var ( - cfg *rest.Config - k8sTestClient client.Client - testEnv *envtest.Environment - ctx context.Context - cancel context.CancelFunc + testEnv *envtest.Environment + cfg *rest.Config + + k8sClient client.Client + + ctx context.Context + cancel context.CancelFunc ) func TestAPIs(t *testing.T) { @@ -66,85 +70,82 @@ func TestAPIs(t *testing.T) { var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - - ctx, cancel = context.WithCancel(context.TODO()) + ctx, cancel = context.WithCancel(context.Background()) By("bootstrapping test environment") testEnv = &envtest.Environment{ CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, - ErrorIfCRDPathMissing: false, + ErrorIfCRDPathMissing: true, - // The BinaryAssetsDirectory is only required if you want to run the tests directly - // without call the makefile target test. If not informed it will look for the - // default path defined in controller-runtime which is /usr/local/kubebuilder/. - // Note that you must have the required binaries setup under the bin directory to perform - // the tests directly. When we run make test it will be setup and used automatically. - BinaryAssetsDirectory: filepath.Join("..", "..", "bin", "k8s", - fmt.Sprintf("1.29.0-%s-%s", runtime.GOOS, runtime.GOARCH)), + // The BinaryAssetsDirectory is only required if you want to run the tests directly without call the makefile target test. + // If not informed it will look for the default path defined in controller-runtime which is /usr/local/kubebuilder/. + // Note that you must have the required binaries setup under the bin directory to perform the tests directly. + // When we run make test it will be setup and used automatically. + BinaryAssetsDirectory: filepath.Join("..", "..", "bin", "k8s", fmt.Sprintf("1.29.0-%s-%s", runtime.GOOS, runtime.GOARCH)), WebhookInstallOptions: envtest.WebhookInstallOptions{ Paths: []string{filepath.Join("..", "..", "config", "webhook")}, }, } - var err error - // cfg is defined in this file globally. cfg, err = testEnv.Start() Expect(err).NotTo(HaveOccurred()) Expect(cfg).NotTo(BeNil()) - scheme := apimachineryruntime.NewScheme() - err = AddToScheme(scheme) - Expect(err).NotTo(HaveOccurred()) - - err = admissionv1.AddToScheme(scheme) + By("setting up the scheme") + err = kubefloworgv1beta1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) //+kubebuilder:scaffold:scheme - k8sTestClient, err = client.New(cfg, client.Options{Scheme: scheme}) + By("creating the k8s client") + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(err).NotTo(HaveOccurred()) - Expect(k8sTestClient).NotTo(BeNil()) + Expect(k8sClient).NotTo(BeNil()) - // start webhook server using Manager + By("setting up the controller manager") webhookInstallOptions := &testEnv.WebhookInstallOptions - mgr, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme, + k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + Metrics: metricsserver.Options{ + BindAddress: "0", // disable metrics serving + }, WebhookServer: webhook.NewServer(webhook.Options{ Host: webhookInstallOptions.LocalServingHost, Port: webhookInstallOptions.LocalServingPort, CertDir: webhookInstallOptions.LocalServingCertDir, }), LeaderElection: false, - Metrics: metricsserver.Options{BindAddress: "0"}, }) Expect(err).NotTo(HaveOccurred()) - err = (&WorkspaceKind{}).SetupWebhookWithManager(mgr) + By("setting up the field indexers for the controller manager") + err = helper.SetupManagerFieldIndexers(k8sManager) Expect(err).NotTo(HaveOccurred()) - // Indexing `.spec.kind` here, not in SetupWebhookWithManager, to avoid conflicts with existing indexing. - // This indexing is specifically for testing purposes to index `Workspace` by `WorkspaceKind`. - err = mgr.GetFieldIndexer().IndexField(context.Background(), &Workspace{}, kbCacheWorkspaceKindField, func(rawObj client.Object) []string { - ws := rawObj.(*Workspace) - if ws.Spec.Kind == "" { - return nil - } - return []string{ws.Spec.Kind} - }) + By("setting up the Workspace webhook") + err = (&WorkspaceValidator{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + }).SetupWebhookWithManager(k8sManager) Expect(err).NotTo(HaveOccurred()) - err = (&Workspace{}).SetupWebhookWithManager(mgr) + + By("setting up the WorkspaceKind webhook") + err = (&WorkspaceKindValidator{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + }).SetupWebhookWithManager(k8sManager) Expect(err).NotTo(HaveOccurred()) //+kubebuilder:scaffold:webhook go func() { defer GinkgoRecover() - err = mgr.Start(ctx) - Expect(err).NotTo(HaveOccurred()) + err = k8sManager.Start(ctx) + Expect(err).NotTo(HaveOccurred(), "failed to run manager") }() - // wait for the webhook server to get ready + // wait for the webhook server to become ready dialer := &net.Dialer{Timeout: time.Second} addrPort := fmt.Sprintf("%s:%d", webhookInstallOptions.LocalServingHost, webhookInstallOptions.LocalServingPort) Eventually(func() error { @@ -158,56 +159,58 @@ var _ = BeforeSuite(func() { }) var _ = AfterSuite(func() { + By("stopping the manager") cancel() + By("tearing down the test environment") err := testEnv.Stop() Expect(err).NotTo(HaveOccurred()) }) // NewExampleWorkspaceKind returns the common "WorkspaceKind" object used in tests. -func NewExampleWorkspaceKind(name string) *WorkspaceKind { - return &WorkspaceKind{ +func NewExampleWorkspaceKind(name string) *kubefloworgv1beta1.WorkspaceKind { + return &kubefloworgv1beta1.WorkspaceKind{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, - Spec: WorkspaceKindSpec{ - Spawner: WorkspaceKindSpawner{ + Spec: kubefloworgv1beta1.WorkspaceKindSpec{ + Spawner: kubefloworgv1beta1.WorkspaceKindSpawner{ DisplayName: "JupyterLab Notebook", Description: "A Workspace which runs JupyterLab in a Pod", Hidden: ptr.To(false), Deprecated: ptr.To(false), DeprecationMessage: ptr.To("This WorkspaceKind will be removed on 20XX-XX-XX, please use another WorkspaceKind."), - Icon: WorkspaceKindIcon{ + Icon: kubefloworgv1beta1.WorkspaceKindIcon{ Url: ptr.To("https://jupyter.org/assets/favicons/apple-touch-icon-152x152.png"), }, - Logo: WorkspaceKindIcon{ - ConfigMap: &WorkspaceKindConfigMap{ + Logo: kubefloworgv1beta1.WorkspaceKindIcon{ + ConfigMap: &kubefloworgv1beta1.WorkspaceKindConfigMap{ Name: "my-logos", Key: "apple-touch-icon-152x152.png", }, }, }, - PodTemplate: WorkspaceKindPodTemplate{ - PodMetadata: &WorkspaceKindPodMetadata{}, - ServiceAccount: WorkspaceKindServiceAccount{ + PodTemplate: kubefloworgv1beta1.WorkspaceKindPodTemplate{ + PodMetadata: &kubefloworgv1beta1.WorkspaceKindPodMetadata{}, + ServiceAccount: kubefloworgv1beta1.WorkspaceKindServiceAccount{ Name: "default-editor", }, - Culling: &WorkspaceKindCullingConfig{ + Culling: &kubefloworgv1beta1.WorkspaceKindCullingConfig{ Enabled: ptr.To(true), MaxInactiveSeconds: ptr.To(int32(86400)), - ActivityProbe: ActivityProbe{ - Jupyter: &ActivityProbeJupyter{ + ActivityProbe: kubefloworgv1beta1.ActivityProbe{ + Jupyter: &kubefloworgv1beta1.ActivityProbeJupyter{ LastActivity: true, }, }, }, - Probes: &WorkspaceKindProbes{}, - VolumeMounts: WorkspaceKindVolumeMounts{ + Probes: &kubefloworgv1beta1.WorkspaceKindProbes{}, + VolumeMounts: kubefloworgv1beta1.WorkspaceKindVolumeMounts{ Home: "/home/jovyan", }, - HTTPProxy: &HTTPProxy{ + HTTPProxy: &kubefloworgv1beta1.HTTPProxy{ RemovePathPrefix: ptr.To(false), - RequestHeaders: &IstioHeaderOperations{ + RequestHeaders: &kubefloworgv1beta1.IstioHeaderOperations{ Set: map[string]string{"X-RStudio-Root-Path": "{{ .PathPrefix }}"}, Add: map[string]string{}, Remove: []string{}, @@ -245,18 +248,18 @@ func NewExampleWorkspaceKind(name string) *WorkspaceKind { }, RunAsNonRoot: ptr.To(true), }, - Options: WorkspaceKindPodOptions{ - ImageConfig: ImageConfig{ - Spawner: OptionsSpawnerConfig{ + Options: kubefloworgv1beta1.WorkspaceKindPodOptions{ + ImageConfig: kubefloworgv1beta1.ImageConfig{ + Spawner: kubefloworgv1beta1.OptionsSpawnerConfig{ Default: "jupyterlab_scipy_190", }, - Values: []ImageConfigValue{ + Values: []kubefloworgv1beta1.ImageConfigValue{ { Id: "jupyterlab_scipy_180", - Spawner: OptionSpawnerInfo{ + Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ DisplayName: "jupyter-scipy:v1.8.0", Description: ptr.To("JupyterLab, with SciPy Packages"), - Labels: []OptionSpawnerLabel{ + Labels: []kubefloworgv1beta1.OptionSpawnerLabel{ { Key: "python_version", Value: "3.11", @@ -264,16 +267,16 @@ func NewExampleWorkspaceKind(name string) *WorkspaceKind { }, Hidden: ptr.To(true), }, - Redirect: &OptionRedirect{ + Redirect: &kubefloworgv1beta1.OptionRedirect{ To: "jupyterlab_scipy_190", - Message: &RedirectMessage{ + Message: &kubefloworgv1beta1.RedirectMessage{ Level: "Info", Text: "This update will change...", }, }, - Spec: ImageConfigSpec{ + Spec: kubefloworgv1beta1.ImageConfigSpec{ Image: "docker.io/kubeflownotebookswg/jupyter-scipy:v1.8.0", - Ports: []ImagePort{ + Ports: []kubefloworgv1beta1.ImagePort{ { Id: "jupyterlab", DisplayName: "JupyterLab", @@ -285,19 +288,19 @@ func NewExampleWorkspaceKind(name string) *WorkspaceKind { }, { Id: "jupyterlab_scipy_190", - Spawner: OptionSpawnerInfo{ + Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ DisplayName: "jupyter-scipy:v1.9.0", Description: ptr.To("JupyterLab, with SciPy Packages"), - Labels: []OptionSpawnerLabel{ + Labels: []kubefloworgv1beta1.OptionSpawnerLabel{ { Key: "python_version", Value: "3.11", }, }, }, - Spec: ImageConfigSpec{ + Spec: kubefloworgv1beta1.ImageConfigSpec{ Image: "docker.io/kubeflownotebookswg/jupyter-scipy:v1.9.0", - Ports: []ImagePort{ + Ports: []kubefloworgv1beta1.ImagePort{ { Id: "jupyterlab", DisplayName: "JupyterLab", @@ -309,17 +312,17 @@ func NewExampleWorkspaceKind(name string) *WorkspaceKind { }, }, }, - PodConfig: PodConfig{ - Spawner: OptionsSpawnerConfig{ + PodConfig: kubefloworgv1beta1.PodConfig{ + Spawner: kubefloworgv1beta1.OptionsSpawnerConfig{ Default: "tiny_cpu", }, - Values: []PodConfigValue{ + Values: []kubefloworgv1beta1.PodConfigValue{ { Id: "tiny_cpu", - Spawner: OptionSpawnerInfo{ + Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ DisplayName: "Tiny CPU", Description: ptr.To("Pod with 0.1 CPU, 128 MB RAM"), - Labels: []OptionSpawnerLabel{ + Labels: []kubefloworgv1beta1.OptionSpawnerLabel{ { Key: "cpu", Value: "100m", @@ -330,7 +333,7 @@ func NewExampleWorkspaceKind(name string) *WorkspaceKind { }, }, }, - Spec: PodConfigSpec{ + Spec: kubefloworgv1beta1.PodConfigSpec{ Resources: &v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ v1.ResourceCPU: resource.MustParse("100m"), @@ -341,10 +344,10 @@ func NewExampleWorkspaceKind(name string) *WorkspaceKind { }, { Id: "small_cpu", - Spawner: OptionSpawnerInfo{ + Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ DisplayName: "Small CPU", Description: ptr.To("Pod with 1 CPU, 2 GB RAM"), - Labels: []OptionSpawnerLabel{ + Labels: []kubefloworgv1beta1.OptionSpawnerLabel{ { Key: "cpu", Value: "1000m", @@ -355,7 +358,7 @@ func NewExampleWorkspaceKind(name string) *WorkspaceKind { }, }, }, - Spec: PodConfigSpec{ + Spec: kubefloworgv1beta1.PodConfigSpec{ Resources: &v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ v1.ResourceCPU: resource.MustParse("1000m"), @@ -366,10 +369,10 @@ func NewExampleWorkspaceKind(name string) *WorkspaceKind { }, { Id: "big_gpu", - Spawner: OptionSpawnerInfo{ + Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ DisplayName: "Big GPU", Description: ptr.To("Pod with 4 CPU, 16 GB RAM, and 1 GPU"), - Labels: []OptionSpawnerLabel{ + Labels: []kubefloworgv1beta1.OptionSpawnerLabel{ { Key: "cpu", Value: "4000m", @@ -384,7 +387,7 @@ func NewExampleWorkspaceKind(name string) *WorkspaceKind { }, }, }, - Spec: PodConfigSpec{ + Spec: kubefloworgv1beta1.PodConfigSpec{ Affinity: nil, NodeSelector: nil, Tolerations: []v1.Toleration{ @@ -414,25 +417,25 @@ func NewExampleWorkspaceKind(name string) *WorkspaceKind { } // NewExampleWorkspaceKindWithImageConfigCycle returns a WorkspaceKind with a cycle in the ImageConfig options. -func NewExampleWorkspaceKindWithImageConfigCycle(name string) *WorkspaceKind { +func NewExampleWorkspaceKindWithImageConfigCycle(name string) *kubefloworgv1beta1.WorkspaceKind { workspaceKind := NewExampleWorkspaceKind(name) - workspaceKind.Spec.PodTemplate.Options.ImageConfig.Values[1].Redirect = &OptionRedirect{ + workspaceKind.Spec.PodTemplate.Options.ImageConfig.Values[1].Redirect = &kubefloworgv1beta1.OptionRedirect{ To: "jupyterlab_scipy_180", } return workspaceKind } // NewExampleWorkspaceKindWithPodConfigCycle returns a WorkspaceKind with a cycle in the PodConfig options. -func NewExampleWorkspaceKindWithPodConfigCycle(name string) *WorkspaceKind { +func NewExampleWorkspaceKindWithPodConfigCycle(name string) *kubefloworgv1beta1.WorkspaceKind { workspaceKind := NewExampleWorkspaceKind(name) - workspaceKind.Spec.PodTemplate.Options.PodConfig.Values[0].Redirect = &OptionRedirect{ + workspaceKind.Spec.PodTemplate.Options.PodConfig.Values[0].Redirect = &kubefloworgv1beta1.OptionRedirect{ To: "small_cpu", - Message: &RedirectMessage{ + Message: &kubefloworgv1beta1.RedirectMessage{ Level: "Info", Text: "This update will change...", }, } - workspaceKind.Spec.PodTemplate.Options.PodConfig.Values[1].Redirect = &OptionRedirect{ + workspaceKind.Spec.PodTemplate.Options.PodConfig.Values[1].Redirect = &kubefloworgv1beta1.OptionRedirect{ To: "tiny_cpu", } @@ -440,9 +443,9 @@ func NewExampleWorkspaceKindWithPodConfigCycle(name string) *WorkspaceKind { } // NewExampleWorkspaceKindWithInvalidImageConfig returns a WorkspaceKind with an invalid redirect in the ImageConfig options. -func NewExampleWorkspaceKindWithInvalidImageConfig(name string) *WorkspaceKind { +func NewExampleWorkspaceKindWithInvalidImageConfig(name string) *kubefloworgv1beta1.WorkspaceKind { workspaceKind := NewExampleWorkspaceKind(name) - workspaceKind.Spec.PodTemplate.Options.ImageConfig.Values[1].Redirect = &OptionRedirect{ + workspaceKind.Spec.PodTemplate.Options.ImageConfig.Values[1].Redirect = &kubefloworgv1beta1.OptionRedirect{ To: "invalid_image_config", } @@ -450,9 +453,9 @@ func NewExampleWorkspaceKindWithInvalidImageConfig(name string) *WorkspaceKind { } // NewExampleWorkspaceKindWithInvalidPodConfig returns a WorkspaceKind with an invalid redirect in the PodConfig options. -func NewExampleWorkspaceKindWithInvalidPodConfig(name string) *WorkspaceKind { +func NewExampleWorkspaceKindWithInvalidPodConfig(name string) *kubefloworgv1beta1.WorkspaceKind { workspaceKind := NewExampleWorkspaceKind(name) - workspaceKind.Spec.PodTemplate.Options.PodConfig.Values[0].Redirect = &OptionRedirect{ + workspaceKind.Spec.PodTemplate.Options.PodConfig.Values[0].Redirect = &kubefloworgv1beta1.OptionRedirect{ To: "invalid_pod_config", } @@ -460,23 +463,23 @@ func NewExampleWorkspaceKindWithInvalidPodConfig(name string) *WorkspaceKind { } // NewExampleWorkspaceKindWithMissingDefaultImageConfig returns a WorkspaceKind with missing default image config. -func NewExampleWorkspaceKindWithInvalidDefaultImageConfig(name string) *WorkspaceKind { +func NewExampleWorkspaceKindWithInvalidDefaultImageConfig(name string) *kubefloworgv1beta1.WorkspaceKind { workspaceKind := NewExampleWorkspaceKind(name) workspaceKind.Spec.PodTemplate.Options.ImageConfig.Spawner.Default = "invalid_image_config" return workspaceKind } // NewExampleWorkspaceKindWithMissingDefaultPodConfig returns a WorkspaceKind with missing default pod config. -func NewExampleWorkspaceKindWithInvalidDefaultPodConfig(name string) *WorkspaceKind { +func NewExampleWorkspaceKindWithInvalidDefaultPodConfig(name string) *kubefloworgv1beta1.WorkspaceKind { workspaceKind := NewExampleWorkspaceKind(name) workspaceKind.Spec.PodTemplate.Options.PodConfig.Spawner.Default = "invalid_pod_config" return workspaceKind } // NewExampleWorkspaceKindWithInvalidExtraEnvValue returns a WorkspaceKind with an invalid extraEnv value. -func NewExampleWorkspaceKindWithDuplicatePorts(name string) *WorkspaceKind { +func NewExampleWorkspaceKindWithDuplicatePorts(name string) *kubefloworgv1beta1.WorkspaceKind { workspaceKind := NewExampleWorkspaceKind(name) - workspaceKind.Spec.PodTemplate.Options.ImageConfig.Values[0].Spec.Ports = []ImagePort{ + workspaceKind.Spec.PodTemplate.Options.ImageConfig.Values[0].Spec.Ports = []kubefloworgv1beta1.ImagePort{ { Id: "jupyterlab", DisplayName: "JupyterLab", @@ -494,7 +497,7 @@ func NewExampleWorkspaceKindWithDuplicatePorts(name string) *WorkspaceKind { } // NewExampleWorkspaceKindWithInvalidExtraEnvValue returns a WorkspaceKind with an invalid extraEnv value. -func NewExampleWorkspaceKindWithInvalidExtraEnvValue(name string) *WorkspaceKind { +func NewExampleWorkspaceKindWithInvalidExtraEnvValue(name string) *kubefloworgv1beta1.WorkspaceKind { workspaceKind := NewExampleWorkspaceKind(name) workspaceKind.Spec.PodTemplate.ExtraEnv = []v1.EnvVar{ { @@ -506,15 +509,15 @@ func NewExampleWorkspaceKindWithInvalidExtraEnvValue(name string) *WorkspaceKind } // NewExampleWorkspace returns the common "Workspace" object used in tests. -func NewExampleWorkspace(name, namespace, workspaceKindName string) *Workspace { - return &Workspace{ +func NewExampleWorkspace(name, namespace, workspaceKindName string) *kubefloworgv1beta1.Workspace { + return &kubefloworgv1beta1.Workspace{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, }, - Spec: WorkspaceSpec{ + Spec: kubefloworgv1beta1.WorkspaceSpec{ Kind: workspaceKindName, - PodTemplate: WorkspacePodTemplate{Options: WorkspacePodOptions{ + PodTemplate: kubefloworgv1beta1.WorkspacePodTemplate{Options: kubefloworgv1beta1.WorkspacePodOptions{ ImageConfig: "jupyterlab_scipy_180", PodConfig: "tiny_cpu", }, diff --git a/workspaces/controller/internal/webhook/workspace_webhook.go b/workspaces/controller/internal/webhook/workspace_webhook.go new file mode 100644 index 00000000..7e8cfe16 --- /dev/null +++ b/workspaces/controller/internal/webhook/workspace_webhook.go @@ -0,0 +1,231 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "context" + "fmt" + + kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation/field" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// WorkspaceValidator validates a Workspace object +type WorkspaceValidator struct { + client.Client + Scheme *runtime.Scheme +} + +//+kubebuilder:webhook:path=/validate-kubeflow-org-v1beta1-workspace,mutating=false,failurePolicy=fail,sideEffects=None,groups=kubeflow.org,resources=workspaces,verbs=create;update,versions=v1beta1,name=vworkspace.kb.io,admissionReviewVersions=v1 + +// SetupWebhookWithManager sets up the webhook with the manager +func (v *WorkspaceValidator) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(&kubefloworgv1beta1.Workspace{}). + WithValidator(v). + Complete() +} + +// ValidateCreate validates the Workspace on creation. +// The optional warnings will be added to the response as warning messages. +// Return an error if the object is invalid. +func (v *WorkspaceValidator) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + log := log.FromContext(ctx) + log.V(1).Info("validating Workspace create") + + var allErrs field.ErrorList + + workspace, ok := obj.(*kubefloworgv1beta1.Workspace) + if !ok { + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Workspace object but got %T", obj)) + } + + // fetch the WorkspaceKind + workspaceKind, err := v.validateWorkspaceKind(ctx, workspace) + if err != nil { + allErrs = append(allErrs, err) + + // if the WorkspaceKind is not found, we cannot validate the Workspace further + return nil, apierrors.NewInvalid( + schema.GroupKind{Group: kubefloworgv1beta1.GroupVersion.Group, Kind: "Workspace"}, + workspace.Name, + allErrs, + ) + } + + // validate the Workspace + if err := v.validateImageConfig(workspace, workspaceKind); err != nil { + allErrs = append(allErrs, err) + } + if err := v.validatePodConfig(workspace, workspaceKind); err != nil { + allErrs = append(allErrs, err) + } + if len(allErrs) == 0 { + return nil, nil + } + + return nil, apierrors.NewInvalid( + schema.GroupKind{Group: kubefloworgv1beta1.GroupVersion.Group, Kind: "Workspace"}, + workspace.Name, + allErrs, + ) +} + +// ValidateUpdate validates the Workspace on update. +// The optional warnings will be added to the response as warning messages. +// Return an error if the object is invalid. +func (v *WorkspaceValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + log := log.FromContext(ctx) + log.V(1).Info("validating Workspace update") + + var allErrs field.ErrorList + + newWorkspace, ok := newObj.(*kubefloworgv1beta1.Workspace) + if !ok { + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Workspace object but got %T", newObj)) + } + oldWorkspace, ok := oldObj.(*kubefloworgv1beta1.Workspace) + if !ok { + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected old object to be a Workspace but got %T", oldObj)) + } + + // check if workspace kind related fields have changed + var workspaceKindChange = false + var imageConfigChange = false + var podConfigChange = false + if newWorkspace.Spec.Kind != oldWorkspace.Spec.Kind { + workspaceKindChange = true + } + if newWorkspace.Spec.PodTemplate.Options.ImageConfig != oldWorkspace.Spec.PodTemplate.Options.ImageConfig { + imageConfigChange = true + } + if newWorkspace.Spec.PodTemplate.Options.PodConfig != oldWorkspace.Spec.PodTemplate.Options.PodConfig { + podConfigChange = true + } + + // if any of the workspace kind related fields have changed, revalidate the workspace + if workspaceKindChange || imageConfigChange || podConfigChange { + // fetch the WorkspaceKind + workspaceKind, err := v.validateWorkspaceKind(ctx, newWorkspace) + if err != nil { + allErrs = append(allErrs, err) + + // if the WorkspaceKind is not found, we cannot validate the Workspace further + return nil, apierrors.NewInvalid( + schema.GroupKind{Group: kubefloworgv1beta1.GroupVersion.Group, Kind: "Workspace"}, + newWorkspace.Name, + allErrs, + ) + } + + // validate the new imageConfig + if imageConfigChange { + if err := v.validateImageConfig(newWorkspace, workspaceKind); err != nil { + allErrs = append(allErrs, err) + } + } + + // validate the new podConfig + if podConfigChange { + if err := v.validatePodConfig(newWorkspace, workspaceKind); err != nil { + allErrs = append(allErrs, err) + } + } + } + + if len(allErrs) == 0 { + return nil, nil + } + + return nil, apierrors.NewInvalid( + schema.GroupKind{Group: kubefloworgv1beta1.GroupVersion.Group, Kind: "Workspace"}, + newWorkspace.Name, + allErrs, + ) +} + +// ValidateDelete validates the Workspace on deletion. +// The optional warnings will be added to the response as warning messages. +// Return an error if the object is invalid. +func (v *WorkspaceValidator) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + // no validation needed for deletion + // NOTE: add "delete" to the webhook configuration (+kubebuilder:webhook) if you want to enable deletion validation + return nil, nil +} + +// validateWorkspaceKind fetches the WorkspaceKind for a Workspace and returns an error if it does not exist +func (v *WorkspaceValidator) validateWorkspaceKind(ctx context.Context, workspace *kubefloworgv1beta1.Workspace) (*kubefloworgv1beta1.WorkspaceKind, *field.Error) { + workspaceKindName := workspace.Spec.Kind + workspaceKind := &kubefloworgv1beta1.WorkspaceKind{} + if err := v.Get(ctx, client.ObjectKey{Name: workspaceKindName}, workspaceKind); err != nil { + workspaceKindNamePath := field.NewPath("spec", "kind") + if apierrors.IsNotFound(err) { + return nil, field.Invalid( + workspaceKindNamePath, + workspaceKindName, + fmt.Sprintf("workspace kind %q not found", workspaceKindName), + ) + } else { + return nil, field.InternalError( + workspaceKindNamePath, + err, + ) + } + } + return workspaceKind, nil +} + +// validateImageConfig checks if the imageConfig selected by a Workspace exists a WorkspaceKind +func (v *WorkspaceValidator) validateImageConfig(workspace *kubefloworgv1beta1.Workspace, workspaceKind *kubefloworgv1beta1.WorkspaceKind) *field.Error { + imageConfig := workspace.Spec.PodTemplate.Options.ImageConfig + for _, value := range workspaceKind.Spec.PodTemplate.Options.ImageConfig.Values { + if imageConfig == value.Id { + // imageConfig found + return nil + } + } + imageConfigPath := field.NewPath("spec", "podTemplate", "options", "imageConfig") + return field.Invalid( + imageConfigPath, + imageConfig, + fmt.Sprintf("imageConfig with id %q not found in workspace kind %q", imageConfig, workspaceKind.Name), + ) +} + +// validatePodConfig checks if the podConfig selected by a Workspace exists a WorkspaceKind +func (v *WorkspaceValidator) validatePodConfig(workspace *kubefloworgv1beta1.Workspace, workspaceKind *kubefloworgv1beta1.WorkspaceKind) *field.Error { + podConfig := workspace.Spec.PodTemplate.Options.PodConfig + for _, value := range workspaceKind.Spec.PodTemplate.Options.PodConfig.Values { + if podConfig == value.Id { + // podConfig found + return nil + } + } + podConfigPath := field.NewPath("spec", "podTemplate", "options", "podConfig") + return field.Invalid( + podConfigPath, + podConfig, + fmt.Sprintf("podConfig with id %q not found in workspace kind %q", podConfig, workspaceKind.Name), + ) +} diff --git a/workspaces/controller/api/v1beta1/workspace_webhook_test.go b/workspaces/controller/internal/webhook/workspace_webhook_test.go similarity index 86% rename from workspaces/controller/api/v1beta1/workspace_webhook_test.go rename to workspaces/controller/internal/webhook/workspace_webhook_test.go index 729bc421..9136c3ea 100644 --- a/workspaces/controller/api/v1beta1/workspace_webhook_test.go +++ b/workspaces/controller/internal/webhook/workspace_webhook_test.go @@ -14,10 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1beta1 +package webhook import ( "fmt" + + kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -41,41 +43,35 @@ var _ = Describe("Workspace Webhook", func() { By("creating the WorkspaceKind") workspaceKind := NewExampleWorkspaceKind(workspaceKindName) Expect(k8sClient.Create(ctx, workspaceKind)).To(Succeed()) - }) AfterAll(func() { - By("deleting the WorkspaceKind") - workspaceKind := &WorkspaceKind{ + workspaceKind := &kubefloworgv1beta1.WorkspaceKind{ ObjectMeta: metav1.ObjectMeta{ Name: workspaceKindName, }, } Expect(k8sClient.Delete(ctx, workspaceKind)).To(Succeed()) - }) It("should reject workspace creation with an invalid WorkspaceKind", func() { - workspaceKindName := "invalid-workspace-kind" + invalidWorkspaceKindName := "invalid-workspace-kind" By("creating the Workspace") - workspace := NewExampleWorkspace(workspaceName, namespaceName, workspaceKindName) + workspace := NewExampleWorkspace(workspaceName, namespaceName, invalidWorkspaceKindName) err := k8sClient.Create(ctx, workspace) - Expect(err).ToNot(Succeed()) - Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("workspace kind %s not found", workspaceKindName))) - + Expect(err).NotTo(Succeed()) + Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("workspace kind %q not found", invalidWorkspaceKindName))) }) It("should successfully create workspace with a valid WorkspaceKind", func() { - By("creating the Workspace") workspace := NewExampleWorkspace(workspaceName, namespaceName, workspaceKindName) Expect(k8sClient.Create(ctx, workspace)).To(Succeed()) By("deleting the Workspace") Expect(k8sClient.Delete(ctx, workspace)).To(Succeed()) - }) }) diff --git a/workspaces/controller/internal/webhook/workspacekind_webhook.go b/workspaces/controller/internal/webhook/workspacekind_webhook.go new file mode 100644 index 00000000..bee30a61 --- /dev/null +++ b/workspaces/controller/internal/webhook/workspacekind_webhook.go @@ -0,0 +1,595 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "context" + "errors" + "fmt" + "reflect" + + kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" + "github.com/kubeflow/notebooks/workspaces/controller/internal/controller" + "github.com/kubeflow/notebooks/workspaces/controller/internal/helper" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation/field" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// WorkspaceKindValidator validates a Workspace object +type WorkspaceKindValidator struct { + client.Client + Scheme *runtime.Scheme +} + +//+kubebuilder:webhook:path=/validate-kubeflow-org-v1beta1-workspacekind,mutating=false,failurePolicy=fail,sideEffects=None,groups=kubeflow.org,resources=workspacekinds,verbs=create;update;delete,versions=v1beta1,name=vworkspacekind.kb.io,admissionReviewVersions=v1 + +// SetupWebhookWithManager sets up the webhook with the manager +func (v *WorkspaceKindValidator) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(&kubefloworgv1beta1.WorkspaceKind{}). + WithValidator(v). + Complete() +} + +// ValidateCreate validates the WorkspaceKind on creation. +// The optional warnings will be added to the response as warning messages. +// Return an error if the object is invalid. +func (v *WorkspaceKindValidator) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + log := log.FromContext(ctx) + log.V(1).Info("validating WorkspaceKind create") + + var allErrs field.ErrorList + + workspaceKind, ok := obj.(*kubefloworgv1beta1.WorkspaceKind) + if !ok { + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a WorkspaceKind object but got %T", obj)) + } + + // validate the extra environment variables + allErrs = append(allErrs, validateExtraEnv(workspaceKind)...) + + // generate helper maps for imageConfig values + imageConfigIdMap := make(map[string]kubefloworgv1beta1.ImageConfigValue) + imageConfigRedirectMap := make(map[string]string) + for _, imageConfigValue := range workspaceKind.Spec.PodTemplate.Options.ImageConfig.Values { + imageConfigIdMap[imageConfigValue.Id] = imageConfigValue + if imageConfigValue.Redirect != nil { + imageConfigRedirectMap[imageConfigValue.Id] = imageConfigValue.Redirect.To + } + } + + // generate helper maps for podConfig values + podConfigIdMap := make(map[string]kubefloworgv1beta1.PodConfigValue) + podConfigRedirectMap := make(map[string]string) + for _, podConfigValue := range workspaceKind.Spec.PodTemplate.Options.PodConfig.Values { + podConfigIdMap[podConfigValue.Id] = podConfigValue + if podConfigValue.Redirect != nil { + podConfigRedirectMap[podConfigValue.Id] = podConfigValue.Redirect.To + } + } + + // validate default options + allErrs = append(allErrs, validateDefaultImageConfig(workspaceKind, imageConfigIdMap)...) + allErrs = append(allErrs, validateDefaultPodConfig(workspaceKind, podConfigIdMap)...) + + // validate imageConfig values + for _, imageConfigValue := range imageConfigIdMap { + imageConfigValueId := imageConfigValue.Id + imageConfigValuePath := field.NewPath("spec", "podTemplate", "options", "imageConfig", "values").Key(imageConfigValueId) + allErrs = append(allErrs, validateImageConfigValue(&imageConfigValue, imageConfigValuePath)...) + } + + // validate redirects + allErrs = append(allErrs, validateImageConfigRedirects(imageConfigIdMap, imageConfigRedirectMap)...) + allErrs = append(allErrs, validatePodConfigRedirects(podConfigIdMap, podConfigRedirectMap)...) + + if len(allErrs) == 0 { + return nil, nil + } + + return nil, apierrors.NewInvalid( + schema.GroupKind{Group: kubefloworgv1beta1.GroupVersion.Group, Kind: "WorkspaceKind"}, + workspaceKind.Name, + allErrs, + ) +} + +// ValidateUpdate validates the WorkspaceKind on update. +// The optional warnings will be added to the response as warning messages. +// Return an error if the object is invalid. +func (v *WorkspaceKindValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { // nolint:gocyclo + log := log.FromContext(ctx) + log.V(1).Info("validating WorkspaceKind update") + + var allErrs field.ErrorList + + newWorkspaceKind, ok := newObj.(*kubefloworgv1beta1.WorkspaceKind) + if !ok { + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a WorkspaceKind object but got %T", newObj)) + } + oldWorkspaceKind, ok := oldObj.(*kubefloworgv1beta1.WorkspaceKind) + if !ok { + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected old object to be a WorkspaceKind but got %T", oldObj)) + } + + // get usage count for imageConfig and podConfig values + imageConfigUsageCount, podConfigUsageCount, err := v.getOptionsUsageCounts(ctx, oldWorkspaceKind) + if err != nil { + return nil, err + } + + // validate the extra environment variables + if !reflect.DeepEqual(newWorkspaceKind.Spec.PodTemplate.ExtraEnv, oldWorkspaceKind.Spec.PodTemplate.ExtraEnv) { + allErrs = append(allErrs, validateExtraEnv(newWorkspaceKind)...) + } + + // calculate changes to imageConfig values + var shouldValidateImageConfigRedirects = false + toValidateImageConfigIds := make(map[string]bool) + badChangedImageConfigIds := make(map[string]bool) + badRemovedImageConfigIds := make(map[string]bool) + oldImageConfigIdMap := make(map[string]kubefloworgv1beta1.ImageConfigValue) + newImageConfigIdMap := make(map[string]kubefloworgv1beta1.ImageConfigValue) + newImageConfigRedirectMap := make(map[string]string) + for _, imageConfigValue := range oldWorkspaceKind.Spec.PodTemplate.Options.ImageConfig.Values { + oldImageConfigIdMap[imageConfigValue.Id] = imageConfigValue + } + for _, imageConfigValue := range newWorkspaceKind.Spec.PodTemplate.Options.ImageConfig.Values { + newImageConfigIdMap[imageConfigValue.Id] = imageConfigValue + if imageConfigValue.Redirect != nil { + newImageConfigRedirectMap[imageConfigValue.Id] = imageConfigValue.Redirect.To + } + + // check if the imageConfig value is new + if _, exists := oldImageConfigIdMap[imageConfigValue.Id]; !exists { + // we need to validate this imageConfig value since it is new + toValidateImageConfigIds[imageConfigValue.Id] = true + + // we always need to validate the imageConfig redirects if an imageConfig value was added + shouldValidateImageConfigRedirects = true + } else { + // check if this imageConfig value is used by any workspaces + var usageCount int32 + if usageCount, exists = imageConfigUsageCount[imageConfigValue.Id]; !exists { + return nil, apierrors.NewInternalError(fmt.Errorf("usage count not found for imageConfig value %q", imageConfigValue.Id)) + } + + // check if the spec has changed + if !reflect.DeepEqual(oldImageConfigIdMap[imageConfigValue.Id].Spec, imageConfigValue.Spec) { + // we need to validate this imageConfig value since it has changed + toValidateImageConfigIds[imageConfigValue.Id] = true + + // if this imageConfig is used by any workspaces, mark this imageConfig as bad, + // (the spec is immutable while in use) + if usageCount > 0 { + badChangedImageConfigIds[imageConfigValue.Id] = true + } + } + + // if we haven't already decided to validate the imageConfig redirects, + // check if the redirect has changed + if !shouldValidateImageConfigRedirects && !reflect.DeepEqual(oldImageConfigIdMap[imageConfigValue.Id].Redirect, imageConfigValue.Redirect) { + shouldValidateImageConfigRedirects = true + } + } + } + for id := range oldImageConfigIdMap { + + // check if this imageConfig value was removed + if _, exists := newImageConfigIdMap[id]; !exists { + // check if this imageConfig value is used by any workspaces + var usageCount int32 + if usageCount, exists = imageConfigUsageCount[id]; !exists { + return nil, apierrors.NewInternalError(fmt.Errorf("usage count not found for imageConfig value %q", id)) + } + + // if this imageConfig is used by any workspaces, mark this imageConfig as bad, + // it is not safe to remove an imageConfig value that is in use + if usageCount > 0 { + badRemovedImageConfigIds[id] = true + } + + // we always need to validate the imageConfig redirects if an imageConfig was removed + shouldValidateImageConfigRedirects = true + } + } + + // calculate changes to podConfig values + var shouldValidatePodConfigRedirects = false + badChangedPodConfigIds := make(map[string]bool) + badRemovedPodConfigIds := make(map[string]bool) + newPodConfigIdMap := make(map[string]kubefloworgv1beta1.PodConfigValue) + newPodConfigRedirectMap := make(map[string]string) + oldPodConfigIdMap := make(map[string]kubefloworgv1beta1.PodConfigValue) + for _, podConfigValue := range oldWorkspaceKind.Spec.PodTemplate.Options.PodConfig.Values { + oldPodConfigIdMap[podConfigValue.Id] = podConfigValue + } + for _, podConfigValue := range newWorkspaceKind.Spec.PodTemplate.Options.PodConfig.Values { + newPodConfigIdMap[podConfigValue.Id] = podConfigValue + if podConfigValue.Redirect != nil { + newPodConfigRedirectMap[podConfigValue.Id] = podConfigValue.Redirect.To + } + + // check if the podConfig value is new + if _, exists := oldPodConfigIdMap[podConfigValue.Id]; !exists { + // we always need to validate the podConfig redirects if a podConfig was added + shouldValidatePodConfigRedirects = true + } else { + // check if this podConfig value is used by any workspaces + var usageCount int32 + if usageCount, exists = podConfigUsageCount[podConfigValue.Id]; !exists { + return nil, apierrors.NewInternalError(fmt.Errorf("usage count not found for podConfig value %q", podConfigValue.Id)) + } + + // normalize the podConfig specs + oldPodConfigSpec := oldPodConfigIdMap[podConfigValue.Id].Spec + err := normalizePodConfigSpec(oldPodConfigSpec) + if err != nil { + return nil, apierrors.NewInternalError(fmt.Errorf("failed to normalize podConfig spec: %w", err)) + } + newPodConfigSpec := podConfigValue.Spec + err = normalizePodConfigSpec(newPodConfigSpec) + if err != nil { + return nil, apierrors.NewInternalError(fmt.Errorf("failed to normalize podConfig spec: %w", err)) + } + + // if this podConfig is used by any workspaces, check if the spec has changed + // if the spec has changed, mark this podConfig as bad (the spec is immutable while in use) + if usageCount > 0 && !reflect.DeepEqual(oldPodConfigSpec, newPodConfigSpec) { + badChangedPodConfigIds[podConfigValue.Id] = true + } + + // if we haven't already decided to validate the podConfig redirects, + // check if the redirect has changed + if !shouldValidatePodConfigRedirects && !reflect.DeepEqual(oldPodConfigIdMap[podConfigValue.Id].Redirect, podConfigValue.Redirect) { + shouldValidatePodConfigRedirects = true + } + } + } + for id := range oldPodConfigIdMap { + + // check if this podConfig value was removed + if _, exists := newPodConfigIdMap[id]; !exists { + // check if this podConfig value is used by any workspaces + var usageCount int32 + if usageCount, exists = podConfigUsageCount[id]; !exists { + return nil, apierrors.NewInternalError(fmt.Errorf("usage count not found for podConfig value %q", id)) + } + + // if this podConfig is used by any workspaces, mark this podConfig as bad, + // it is not safe to remove a podConfig value that is in use + if usageCount > 0 { + badRemovedPodConfigIds[id] = true + } + + // we always need to validate the podConfig redirects if a podConfig was removed + shouldValidatePodConfigRedirects = true + } + } + + // validate default options + if oldWorkspaceKind.Spec.PodTemplate.Options.ImageConfig.Spawner.Default != newWorkspaceKind.Spec.PodTemplate.Options.ImageConfig.Spawner.Default { + allErrs = append(allErrs, validateDefaultImageConfig(newWorkspaceKind, newImageConfigIdMap)...) + } + if oldWorkspaceKind.Spec.PodTemplate.Options.PodConfig.Spawner.Default != newWorkspaceKind.Spec.PodTemplate.Options.PodConfig.Spawner.Default { + allErrs = append(allErrs, validateDefaultPodConfig(newWorkspaceKind, newPodConfigIdMap)...) + } + + // validate imageConfig values + // NOTE: we only need to validate new or changed imageConfig values + for imageConfigValueId := range toValidateImageConfigIds { + imageConfigValue := newImageConfigIdMap[imageConfigValueId] + imageConfigValuePath := field.NewPath("spec", "podTemplate", "options", "imageConfig", "values").Key(imageConfigValueId) + allErrs = append(allErrs, validateImageConfigValue(&imageConfigValue, imageConfigValuePath)...) + } + + // validate bad imageConfig values + for id := range badChangedImageConfigIds { + imageConfigValuePath := field.NewPath("spec", "podTemplate", "options", "imageConfig", "values").Key(id) + allErrs = append(allErrs, field.Invalid(imageConfigValuePath, id, fmt.Sprintf("imageConfig value %q is in use and cannot be changed", id))) + } + for id := range badRemovedImageConfigIds { + imageConfigValuePath := field.NewPath("spec", "podTemplate", "options", "imageConfig", "values").Key(id) + allErrs = append(allErrs, field.Invalid(imageConfigValuePath, id, fmt.Sprintf("imageConfig value %q is in use and cannot be removed", id))) + } + + // validate bad podConfig values + for id := range badChangedPodConfigIds { + podConfigValuePath := field.NewPath("spec", "podTemplate", "options", "podConfig", "values").Key(id) + allErrs = append(allErrs, field.Invalid(podConfigValuePath, id, fmt.Sprintf("podConfig value %q is in use and cannot be changed", id))) + } + for id := range badRemovedPodConfigIds { + podConfigValuePath := field.NewPath("spec", "podTemplate", "options", "podConfig", "values").Key(id) + allErrs = append(allErrs, field.Invalid(podConfigValuePath, id, fmt.Sprintf("podConfig value %q is in use and cannot be removed", id))) + } + + // validate redirects + if shouldValidateImageConfigRedirects { + allErrs = append(allErrs, validateImageConfigRedirects(newImageConfigIdMap, newImageConfigRedirectMap)...) + } + if shouldValidatePodConfigRedirects { + allErrs = append(allErrs, validatePodConfigRedirects(newPodConfigIdMap, newPodConfigRedirectMap)...) + } + + if len(allErrs) == 0 { + return nil, nil + } + + return nil, apierrors.NewInvalid( + schema.GroupKind{Group: kubefloworgv1beta1.GroupVersion.Group, Kind: "WorkspaceKind"}, + newWorkspaceKind.Name, + allErrs, + ) +} + +// ValidateDelete validates the WorkspaceKind on deletion. +// The optional warnings will be added to the response as warning messages. +// Return an error if the object is invalid. +func (v *WorkspaceKindValidator) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + log := log.FromContext(ctx) + log.V(1).Info("validating WorkspaceKind delete") + + workspaceKind, ok := obj.(*kubefloworgv1beta1.WorkspaceKind) + if !ok { + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a WorkspaceKind object but got %T", obj)) + } + + // don't allow deletion of WorkspaceKind if it is used by any workspaces + if workspaceKind.Status.Workspaces > 0 { + return nil, apierrors.NewConflict( + schema.GroupResource{Group: kubefloworgv1beta1.GroupVersion.Group, Resource: "WorkspaceKind"}, + workspaceKind.Name, + fmt.Errorf("WorkspaceKind is used by %d workspace(s)", workspaceKind.Status.Workspaces), + ) + } + + // don't allow deletion of WorkspaceKind if it has the protection finalizer + // NOTE: while the finalizer also protects the WorkspaceKind from deletion, + // it is impossible to "un-delete" a resource once it has started terminating + // and this is a bad user experience, so we prevent deletion in the first place + if controllerutil.ContainsFinalizer(workspaceKind, controller.WorkspaceKindFinalizer) { + return nil, apierrors.NewConflict( + schema.GroupResource{Group: kubefloworgv1beta1.GroupVersion.Group, Resource: "WorkspaceKind"}, + workspaceKind.Name, + errors.New("WorkspaceKind has protection finalizer, indicating one or more workspaces are still using it"), + ) + } + + return nil, nil +} + +// getOptionsUsageCounts returns the usage counts for each imageConfig and podConfig value +func (v *WorkspaceKindValidator) getOptionsUsageCounts(ctx context.Context, workspaceKind *kubefloworgv1beta1.WorkspaceKind) (map[string]int32, map[string]int32, *apierrors.StatusError) { + podConfigUsageCount := make(map[string]int32) + imageConfigUsageCount := make(map[string]int32) + + // if possible, we get the counts from the status of the WorkspaceKind. these counts are updated by the + // controller so could be stale if the controller is not running or a workspace was very recently added or removed. + // however, since the controller gracefully handles cases of a Workspace referencing a non-existent imageConfig + // or podConfig value, we can safely use these counts to validate the WorkspaceKind, Workspaces will simply be + // put into an error state and the user can correct the issue. these counts will NOT be set in the WorkspaceKind + // unit tests, so we implement a fallback method to count the Workspaces that are using each option. + if len(workspaceKind.Status.PodTemplateOptions.ImageConfig) > 0 && len(workspaceKind.Status.PodTemplateOptions.PodConfig) > 0 { + for _, imageConfigMetrics := range workspaceKind.Status.PodTemplateOptions.ImageConfig { + imageConfigUsageCount[imageConfigMetrics.Id] = imageConfigMetrics.Workspaces + } + for _, podConfigMetrics := range workspaceKind.Status.PodTemplateOptions.PodConfig { + podConfigUsageCount[podConfigMetrics.Id] = podConfigMetrics.Workspaces + } + } + + // fetch all Workspaces that are using this WorkspaceKind + workspaces := &kubefloworgv1beta1.WorkspaceList{} + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(helper.IndexWorkspaceKindField, workspaceKind.Name), + Namespace: "", // fetch Workspaces in all namespaces + } + if err := v.List(ctx, workspaces, listOpts); err != nil { + return nil, nil, apierrors.NewInternalError(err) + } + + // count the number of Workspaces using each option + for _, imageConfig := range workspaceKind.Spec.PodTemplate.Options.ImageConfig.Values { + imageConfigUsageCount[imageConfig.Id] = 0 + } + for _, podConfig := range workspaceKind.Spec.PodTemplate.Options.PodConfig.Values { + podConfigUsageCount[podConfig.Id] = 0 + } + for _, ws := range workspaces.Items { + imageConfigUsageCount[ws.Spec.PodTemplate.Options.ImageConfig]++ + podConfigUsageCount[ws.Spec.PodTemplate.Options.PodConfig]++ + } + + return imageConfigUsageCount, podConfigUsageCount, nil +} + +// validateExtraEnv validates the extra environment variables in a WorkspaceKind +func validateExtraEnv(workspaceKind *kubefloworgv1beta1.WorkspaceKind) []*field.Error { + var errs []*field.Error + + // the real httpPathPrefix can't fail, so we return a dummy value + httpPathPrefixFunc := func(portId string) string { + return "DUMMY_HTTP_PATH_PREFIX" + } + + // validate that each value template can be rendered successfully + for _, env := range workspaceKind.Spec.PodTemplate.ExtraEnv { + if env.Value != "" { + rawValue := env.Value + _, err := helper.RenderExtraEnvValueTemplate(rawValue, httpPathPrefixFunc) + if err != nil { + extraEnvPath := field.NewPath("spec", "podTemplate", "extraEnv").Key(env.Name).Child("value") + errs = append(errs, field.Invalid(extraEnvPath, rawValue, err.Error())) + } + } + } + + return errs +} + +// validateDefaultImageConfig validates the default imageConfig in a WorkspaceKind +func validateDefaultImageConfig(workspaceKind *kubefloworgv1beta1.WorkspaceKind, imageConfigValueMap map[string]kubefloworgv1beta1.ImageConfigValue) []*field.Error { + var errs []*field.Error + + // validate the default imageConfig + defaultImageConfig := workspaceKind.Spec.PodTemplate.Options.ImageConfig.Spawner.Default + if _, exists := imageConfigValueMap[defaultImageConfig]; !exists { + defaultImageConfigPath := field.NewPath("spec", "podTemplate", "options", "imageConfig", "spawner", "default") + errs = append(errs, field.Invalid(defaultImageConfigPath, defaultImageConfig, fmt.Sprintf("default imageConfig %q not found", defaultImageConfig))) + } + + return errs +} + +// validateDefaultPodConfig validates the default podConfig in a WorkspaceKind +func validateDefaultPodConfig(workspaceKind *kubefloworgv1beta1.WorkspaceKind, podConfigValueMap map[string]kubefloworgv1beta1.PodConfigValue) []*field.Error { + var errs []*field.Error + + // validate the default podConfig + defaultPodConfig := workspaceKind.Spec.PodTemplate.Options.PodConfig.Spawner.Default + if _, exists := podConfigValueMap[defaultPodConfig]; !exists { + defaultPodConfigPath := field.NewPath("spec", "podTemplate", "options", "podConfig", "spawner", "default") + errs = append(errs, field.Invalid(defaultPodConfigPath, defaultPodConfig, fmt.Sprintf("default podConfig %q not found", defaultPodConfig))) + } + + return errs +} + +// validateImageConfigValue validates an imageConfig value +func validateImageConfigValue(imageConfigValue *kubefloworgv1beta1.ImageConfigValue, imageConfigValuePath *field.Path) []*field.Error { + var errs []*field.Error + + // validate the ports + seenPorts := make(map[int32]bool) + for _, port := range imageConfigValue.Spec.Ports { + portId := port.Id + portNumber := port.Port + if _, exists := seenPorts[portNumber]; exists { + portPath := imageConfigValuePath.Child("spec", "ports").Key(portId).Child("port") + errs = append(errs, field.Invalid(portPath, portNumber, fmt.Sprintf("port %d is defined more than once", portNumber))) + } + seenPorts[portNumber] = true + } + + return errs +} + +// validateImageConfigRedirects validates redirects in the imageConfig values +func validateImageConfigRedirects(imageConfigIdMap map[string]kubefloworgv1beta1.ImageConfigValue, imageConfigRedirectMap map[string]string) []*field.Error { + var errs []*field.Error + + // validate imageConfig redirects + checkedNodes := make(map[string]bool) + for id, redirectTo := range imageConfigRedirectMap { + // check if there is a cycle involving the current node + if cycle := helper.DetectGraphCycle(id, checkedNodes, imageConfigRedirectMap); cycle != nil { + redirectToPath := field.NewPath("spec", "podTemplate", "options", "imageConfig", "values").Key(id).Child("redirect", "to") + errs = append(errs, field.Invalid(redirectToPath, redirectTo, fmt.Sprintf("cycle detected: %v", cycle))) + break // stop checking redirects if a cycle is detected + } + + // ensure the target of the redirect exists + if _, exists := imageConfigIdMap[redirectTo]; !exists { + redirectToPath := field.NewPath("spec", "podTemplate", "options", "imageConfig", "values").Key(id).Child("redirect", "to") + errs = append(errs, field.Invalid(redirectToPath, redirectTo, fmt.Sprintf("invalid redirect target %q", redirectTo))) + } + } + + return errs +} + +// validatePodConfigRedirects validates redirects in the podConfig values +func validatePodConfigRedirects(podConfigIdMap map[string]kubefloworgv1beta1.PodConfigValue, podConfigRedirectMap map[string]string) []*field.Error { + var errs []*field.Error + + // validate podConfig redirects + checkedNodes := make(map[string]bool) + for id, redirectTo := range podConfigRedirectMap { + // check if there is a cycle involving the current node + if cycle := helper.DetectGraphCycle(id, checkedNodes, podConfigRedirectMap); cycle != nil { + redirectToPath := field.NewPath("spec", "podTemplate", "options", "podConfig", "values").Key(id).Child("redirect", "to") + errs = append(errs, field.Invalid(redirectToPath, redirectTo, fmt.Sprintf("cycle detected: %v", cycle))) + break // stop checking redirects if a cycle is detected + } + + // ensure the target of the redirect exists + if _, exists := podConfigIdMap[redirectTo]; !exists { + redirectToPath := field.NewPath("spec", "podTemplate", "options", "podConfig", "values").Key(id).Child("redirect", "to") + errs = append(errs, field.Invalid(redirectToPath, redirectTo, fmt.Sprintf("invalid redirect target %q", redirectTo))) + } + } + + return errs +} + +// normalizePodConfigSpec normalizes a PodConfigSpec so that it can be compared with reflect.DeepEqual +func normalizePodConfigSpec(spec kubefloworgv1beta1.PodConfigSpec) (err error) { + // Normalize Affinity + if spec.Affinity != nil && reflect.DeepEqual(spec.Affinity, corev1.Affinity{}) { + spec.Affinity = nil + } + + // Normalize NodeSelector + if spec.NodeSelector != nil && len(spec.NodeSelector) == 0 { + spec.NodeSelector = nil + } + + // Normalize Tolerations + if spec.Tolerations != nil && len(spec.Tolerations) == 0 { + spec.Tolerations = nil + } + + // Normalize Resources.Requests + if reflect.DeepEqual(spec.Resources.Requests, corev1.ResourceList{}) { + spec.Resources.Requests = nil + } + if spec.Resources.Requests != nil { + for key, value := range spec.Resources.Requests { + q, err := resource.ParseQuantity(value.String()) + if err != nil { + return err + } + spec.Resources.Requests[key] = q + } + } + + // Normalize Resources.Limits + if reflect.DeepEqual(spec.Resources.Limits, corev1.ResourceList{}) { + spec.Resources.Limits = nil + } + if spec.Resources.Limits != nil { + for key, value := range spec.Resources.Limits { + q, err := resource.ParseQuantity(value.String()) + if err != nil { + return err + } + spec.Resources.Limits[key] = q + } + } + + return nil +} diff --git a/workspaces/controller/api/v1beta1/workspacekind_webhook_test.go b/workspaces/controller/internal/webhook/workspacekind_webhook_test.go similarity index 76% rename from workspaces/controller/api/v1beta1/workspacekind_webhook_test.go rename to workspaces/controller/internal/webhook/workspacekind_webhook_test.go index 6a120916..9ef7f968 100644 --- a/workspaces/controller/api/v1beta1/workspacekind_webhook_test.go +++ b/workspaces/controller/internal/webhook/workspacekind_webhook_test.go @@ -14,10 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1beta1 +package webhook import ( "fmt" + "time" + + "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -25,7 +28,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - "time" ) var _ = Describe("WorkspaceKind Webhook", func() { @@ -47,39 +49,48 @@ var _ = Describe("WorkspaceKind Webhook", func() { testCases := []struct { description string - workspaceKind *WorkspaceKind + workspaceKind *v1beta1.WorkspaceKind + shouldSucceed bool }{ { description: "should reject WorkspaceKind creation with cycles in ImageConfig options", workspaceKind: NewExampleWorkspaceKindWithImageConfigCycle("wsk-webhook-image-config-cycle-test"), + shouldSucceed: false, }, { description: "should reject WorkspaceKind creation with cycles in PodConfig options", workspaceKind: NewExampleWorkspaceKindWithPodConfigCycle("wsk-webhook-pod-config-cycle-test"), + shouldSucceed: false, }, { description: "should reject WorkspaceKind creation with invalid redirects in ImageConfig options", workspaceKind: NewExampleWorkspaceKindWithInvalidImageConfig("wsk-webhook-image-config-invalid-test"), + shouldSucceed: false, }, { description: "should reject WorkspaceKind creation with invalid redirects in PodConfig options", workspaceKind: NewExampleWorkspaceKindWithInvalidPodConfig("wsk-webhook-pod-config-invalid-test"), + shouldSucceed: false, }, { description: "should reject WorkspaceKind creation if the default ImageConfig option is missing", workspaceKind: NewExampleWorkspaceKindWithInvalidDefaultImageConfig("wsk-webhook-image-config-default-test"), + shouldSucceed: false, }, { description: "should reject WorkspaceKind creation if the default PodConfig option is missing", workspaceKind: NewExampleWorkspaceKindWithInvalidDefaultPodConfig("wsk-webhook-pod-config-default-test"), + shouldSucceed: false, }, { description: "should reject WorkspaceKind creation with non-unique ports in PodConfig", workspaceKind: NewExampleWorkspaceKindWithDuplicatePorts("wsk-webhook-ports-port-not-unique-test"), + shouldSucceed: false, }, { description: "should reject WorkspaceKind creation if extraEnv[].value is not a valid Go template", workspaceKind: NewExampleWorkspaceKindWithInvalidExtraEnvValue("wsk-webhook-extra-env-value-invalid-test"), + shouldSucceed: false, }, } @@ -87,7 +98,11 @@ var _ = Describe("WorkspaceKind Webhook", func() { tc := tc // Create a new instance of tc to avoid capturing the loop variable. It(tc.description, func() { By("creating the WorkspaceKind") - Expect(k8sClient.Create(ctx, tc.workspaceKind)).ToNot(Succeed()) + if tc.shouldSucceed { + Expect(k8sClient.Create(ctx, tc.workspaceKind)).To(Succeed()) + } else { + Expect(k8sClient.Create(ctx, tc.workspaceKind)).NotTo(Succeed()) + } }) } @@ -97,7 +112,7 @@ var _ = Describe("WorkspaceKind Webhook", func() { var ( workspaceKindName string workspaceKindKey types.NamespacedName - workspaceKind *WorkspaceKind + workspaceKind *v1beta1.WorkspaceKind ) BeforeAll(func() { @@ -110,7 +125,7 @@ var _ = Describe("WorkspaceKind Webhook", func() { Expect(k8sClient.Create(ctx, createdWorkspaceKind)).To(Succeed()) By("getting the created WorkspaceKind") - workspaceKind = &WorkspaceKind{} + workspaceKind = &v1beta1.WorkspaceKind{} Eventually(func() error { return k8sClient.Get(ctx, workspaceKindKey, workspaceKind) }, timeout, interval).Should(Succeed()) @@ -123,19 +138,21 @@ var _ = Describe("WorkspaceKind Webhook", func() { testCases := []struct { description string - modifyKindFn func(*WorkspaceKind) + modifyKindFn func(*v1beta1.WorkspaceKind) workspaceName *string + shouldSucceed bool }{ { description: "should reject updates to used imageConfig spec", - modifyKindFn: func(wsk *WorkspaceKind) { + modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { wsk.Spec.PodTemplate.Options.ImageConfig.Values[0].Spec.Image = "new-image:latest" }, workspaceName: ptr.To("ws-webhook-update-image-config-spec-test"), + shouldSucceed: false, }, { description: "should reject updates to used podConfig spec", - modifyKindFn: func(wsk *WorkspaceKind) { + modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Spec.Resources = &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1.5"), @@ -143,64 +160,79 @@ var _ = Describe("WorkspaceKind Webhook", func() { } }, workspaceName: ptr.To("ws-webhook-update-pod-config-spec-test"), + shouldSucceed: false, }, { description: "should reject WorkspaceKind update with cycles in imageConfig options", - modifyKindFn: func(wsk *WorkspaceKind) { - wsk.Spec.PodTemplate.Options.ImageConfig.Values[1].Redirect = &OptionRedirect{To: "jupyterlab_scipy_190"} + modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { + wsk.Spec.PodTemplate.Options.ImageConfig.Values[1].Redirect = &v1beta1.OptionRedirect{To: "jupyterlab_scipy_190"} }, + shouldSucceed: false, }, { description: "should reject WorkspaceKind update with invalid redirects in ImageConfig options", - modifyKindFn: func(wsk *WorkspaceKind) { - wsk.Spec.PodTemplate.Options.ImageConfig.Values[1].Redirect = &OptionRedirect{To: "invalid-image-config"} + modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { + wsk.Spec.PodTemplate.Options.ImageConfig.Values[1].Redirect = &v1beta1.OptionRedirect{To: "invalid-image-config"} }, + shouldSucceed: false, }, { description: "should reject WorkspaceKind update with cycles in PodConfig options", - modifyKindFn: func(wsk *WorkspaceKind) { - wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Redirect = &OptionRedirect{To: "small_cpu"} - wsk.Spec.PodTemplate.Options.PodConfig.Values[1].Redirect = &OptionRedirect{To: "tiny_cpu"} - + modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { + wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Redirect = &v1beta1.OptionRedirect{To: "small_cpu"} + wsk.Spec.PodTemplate.Options.PodConfig.Values[1].Redirect = &v1beta1.OptionRedirect{To: "tiny_cpu"} }, + shouldSucceed: false, }, { description: "should reject WorkspaceKind creation with invalid redirects in PodConfig options", - modifyKindFn: func(wsk *WorkspaceKind) { - wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Redirect = &OptionRedirect{To: "invalid-pod-config"} + modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { + wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Redirect = &v1beta1.OptionRedirect{To: "invalid-pod-config"} }, + shouldSucceed: false, }, { description: "should reject updates to WorkspaceKind with missing default imageConfig", - modifyKindFn: func(wsk *WorkspaceKind) { + modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { wsk.Spec.PodTemplate.Options.ImageConfig.Spawner.Default = "invalid-image-config" }, + shouldSucceed: false, }, { description: "should reject updates to WorkspaceKind with missing default podConfig", - modifyKindFn: func(wsk *WorkspaceKind) { + modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { wsk.Spec.PodTemplate.Options.PodConfig.Spawner.Default = "invalid-pod-config" }, }, { description: "should reject updates to WorkspaceKind if extraEnv[].value is not a valid Go template", - modifyKindFn: func(wsk *WorkspaceKind) { + modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { wsk.Spec.PodTemplate.ExtraEnv[0].Value = `{{ httpPathPrefix "jupyterlab" }` }, + shouldSucceed: false, + }, + { + description: "should accept updates to WorkspaceKind with valid extraEnv[].value Go template", + modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { + wsk.Spec.PodTemplate.ExtraEnv[0].Value = `{{ httpPathPrefix "jupyterlab" }}` + }, + shouldSucceed: true, }, { description: "should reject updates that remove ImageConfig in use", - modifyKindFn: func(wsk *WorkspaceKind) { + modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { wsk.Spec.PodTemplate.Options.ImageConfig.Values = wsk.Spec.PodTemplate.Options.ImageConfig.Values[1:] }, workspaceName: ptr.To("ws-webhook-update-image-config-test"), + shouldSucceed: false, }, { description: "should reject updates that remove podConfig in use", - modifyKindFn: func(wsk *WorkspaceKind) { + modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { wsk.Spec.PodTemplate.Options.PodConfig.Values = wsk.Spec.PodTemplate.Options.PodConfig.Values[1:] }, workspaceName: ptr.To("ws-webhook-update-pod-config-test"), + shouldSucceed: false, }, } @@ -208,7 +240,6 @@ var _ = Describe("WorkspaceKind Webhook", func() { tc := tc // Create a new instance of tc to avoid capturing the loop variable. It(tc.description, func() { if tc.workspaceName != nil { - By("creating a Workspace with the WorkspaceKind") workspace := NewExampleWorkspace(*tc.workspaceName, namespaceName, workspaceKind.Name) Expect(k8sClient.Create(ctx, workspace)).To(Succeed()) @@ -218,7 +249,11 @@ var _ = Describe("WorkspaceKind Webhook", func() { modifiedWorkspaceKind := workspaceKind.DeepCopy() tc.modifyKindFn(modifiedWorkspaceKind) - Expect(k8sClient.Patch(ctx, modifiedWorkspaceKind, patch)).NotTo(Succeed()) + if tc.shouldSucceed { + Expect(k8sClient.Patch(ctx, modifiedWorkspaceKind, patch)).To(Succeed()) + } else { + Expect(k8sClient.Patch(ctx, modifiedWorkspaceKind, patch)).NotTo(Succeed()) + } }) } }) From f6ca24bdd2b0e8595fc143d0b8928fb4b834ba92 Mon Sep 17 00:00:00 2001 From: Mathew Wicks <5735406+thesuperzapper@users.noreply.github.com> Date: Tue, 27 Aug 2024 17:18:35 -0700 Subject: [PATCH 7/8] mathew refactor 2 Signed-off-by: Mathew Wicks <5735406+thesuperzapper@users.noreply.github.com> --- workspaces/controller/cmd/main.go | 5 +- .../controller/workspace_controller_test.go | 2 +- .../workspacekind_controller_test.go | 2 +- .../controller/internal/helper/helper.go | 51 ++++ .../controller/internal/webhook/suite_test.go | 5 +- .../internal/webhook/workspace_webhook.go | 11 +- .../webhook/workspace_webhook_test.go | 137 ++++++++- .../internal/webhook/workspacekind_webhook.go | 265 +++++++++--------- .../webhook/workspacekind_webhook_test.go | 239 ++++++++++------ workspaces/controller/test/e2e/e2e_test.go | 55 +++- 10 files changed, 536 insertions(+), 236 deletions(-) diff --git a/workspaces/controller/cmd/main.go b/workspaces/controller/cmd/main.go index 82659c2f..eb2a26c7 100644 --- a/workspaces/controller/cmd/main.go +++ b/workspaces/controller/cmd/main.go @@ -21,9 +21,6 @@ import ( "flag" "os" - "github.com/kubeflow/notebooks/workspaces/controller/internal/helper" - webhookInternal "github.com/kubeflow/notebooks/workspaces/controller/internal/webhook" - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" @@ -39,6 +36,8 @@ import ( kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" controllerInternal "github.com/kubeflow/notebooks/workspaces/controller/internal/controller" + "github.com/kubeflow/notebooks/workspaces/controller/internal/helper" + webhookInternal "github.com/kubeflow/notebooks/workspaces/controller/internal/webhook" //+kubebuilder:scaffold:imports ) diff --git a/workspaces/controller/internal/controller/workspace_controller_test.go b/workspaces/controller/internal/controller/workspace_controller_test.go index e28749b0..868c645e 100644 --- a/workspaces/controller/internal/controller/workspace_controller_test.go +++ b/workspaces/controller/internal/controller/workspace_controller_test.go @@ -44,7 +44,7 @@ var _ = Describe("Workspace Controller", func() { timeout = time.Second * 10 // how long to wait in "Consistently" blocks - duration = time.Second * 10 + duration = time.Second * 10 // nolint:unused // how frequently to poll for conditions interval = time.Millisecond * 250 diff --git a/workspaces/controller/internal/controller/workspacekind_controller_test.go b/workspaces/controller/internal/controller/workspacekind_controller_test.go index 59e7dd15..505a9dc8 100644 --- a/workspaces/controller/internal/controller/workspacekind_controller_test.go +++ b/workspaces/controller/internal/controller/workspacekind_controller_test.go @@ -40,7 +40,7 @@ var _ = Describe("WorkspaceKind Controller", func() { timeout = time.Second * 10 // how long to wait in "Consistently" blocks - duration = time.Second * 10 + duration = time.Second * 10 // nolint:unused // how frequently to poll for conditions interval = time.Millisecond * 250 diff --git a/workspaces/controller/internal/helper/helper.go b/workspaces/controller/internal/helper/helper.go index 486b1424..ca68176f 100644 --- a/workspaces/controller/internal/helper/helper.go +++ b/workspaces/controller/internal/helper/helper.go @@ -3,6 +3,9 @@ package helper import ( "reflect" + kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" + "k8s.io/apimachinery/pkg/api/resource" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" ) @@ -98,3 +101,51 @@ func CopyServiceFields(desired *corev1.Service, target *corev1.Service) bool { return requireUpdate } + +// NormalizePodConfigSpec normalizes a PodConfigSpec so that it can be compared with reflect.DeepEqual +func NormalizePodConfigSpec(spec kubefloworgv1beta1.PodConfigSpec) error { + // Normalize Affinity + if spec.Affinity != nil && reflect.DeepEqual(spec.Affinity, corev1.Affinity{}) { + spec.Affinity = nil + } + + // Normalize NodeSelector + if spec.NodeSelector != nil && len(spec.NodeSelector) == 0 { + spec.NodeSelector = nil + } + + // Normalize Tolerations + if spec.Tolerations != nil && len(spec.Tolerations) == 0 { + spec.Tolerations = nil + } + + // Normalize Resources.Requests + if reflect.DeepEqual(spec.Resources.Requests, corev1.ResourceList{}) { + spec.Resources.Requests = nil + } + if spec.Resources.Requests != nil { + for key, value := range spec.Resources.Requests { + q, err := resource.ParseQuantity(value.String()) + if err != nil { + return err + } + spec.Resources.Requests[key] = q + } + } + + // Normalize Resources.Limits + if reflect.DeepEqual(spec.Resources.Limits, corev1.ResourceList{}) { + spec.Resources.Limits = nil + } + if spec.Resources.Limits != nil { + for key, value := range spec.Resources.Limits { + q, err := resource.ParseQuantity(value.String()) + if err != nil { + return err + } + spec.Resources.Limits[key] = q + } + } + + return nil +} diff --git a/workspaces/controller/internal/webhook/suite_test.go b/workspaces/controller/internal/webhook/suite_test.go index 1c480b13..579123c7 100644 --- a/workspaces/controller/internal/webhook/suite_test.go +++ b/workspaces/controller/internal/webhook/suite_test.go @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +// nolint:goconst package webhook import ( @@ -443,7 +444,7 @@ func NewExampleWorkspaceKindWithPodConfigCycle(name string) *kubefloworgv1beta1. } // NewExampleWorkspaceKindWithInvalidImageConfig returns a WorkspaceKind with an invalid redirect in the ImageConfig options. -func NewExampleWorkspaceKindWithInvalidImageConfig(name string) *kubefloworgv1beta1.WorkspaceKind { +func NewExampleWorkspaceKindWithInvalidImageConfigRedirect(name string) *kubefloworgv1beta1.WorkspaceKind { workspaceKind := NewExampleWorkspaceKind(name) workspaceKind.Spec.PodTemplate.Options.ImageConfig.Values[1].Redirect = &kubefloworgv1beta1.OptionRedirect{ To: "invalid_image_config", @@ -453,7 +454,7 @@ func NewExampleWorkspaceKindWithInvalidImageConfig(name string) *kubefloworgv1be } // NewExampleWorkspaceKindWithInvalidPodConfig returns a WorkspaceKind with an invalid redirect in the PodConfig options. -func NewExampleWorkspaceKindWithInvalidPodConfig(name string) *kubefloworgv1beta1.WorkspaceKind { +func NewExampleWorkspaceKindWithInvalidPodConfigRedirect(name string) *kubefloworgv1beta1.WorkspaceKind { workspaceKind := NewExampleWorkspaceKind(name) workspaceKind.Spec.PodTemplate.Options.PodConfig.Values[0].Redirect = &kubefloworgv1beta1.OptionRedirect{ To: "invalid_pod_config", diff --git a/workspaces/controller/internal/webhook/workspace_webhook.go b/workspaces/controller/internal/webhook/workspace_webhook.go index 7e8cfe16..412d6ebd 100644 --- a/workspaces/controller/internal/webhook/workspace_webhook.go +++ b/workspaces/controller/internal/webhook/workspace_webhook.go @@ -20,7 +20,6 @@ import ( "context" "fmt" - kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -29,6 +28,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" ) // WorkspaceValidator validates a Workspace object @@ -54,13 +55,13 @@ func (v *WorkspaceValidator) ValidateCreate(ctx context.Context, obj runtime.Obj log := log.FromContext(ctx) log.V(1).Info("validating Workspace create") - var allErrs field.ErrorList - workspace, ok := obj.(*kubefloworgv1beta1.Workspace) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Workspace object but got %T", obj)) } + var allErrs field.ErrorList + // fetch the WorkspaceKind workspaceKind, err := v.validateWorkspaceKind(ctx, workspace) if err != nil { @@ -99,8 +100,6 @@ func (v *WorkspaceValidator) ValidateUpdate(ctx context.Context, oldObj, newObj log := log.FromContext(ctx) log.V(1).Info("validating Workspace update") - var allErrs field.ErrorList - newWorkspace, ok := newObj.(*kubefloworgv1beta1.Workspace) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Workspace object but got %T", newObj)) @@ -110,6 +109,8 @@ func (v *WorkspaceValidator) ValidateUpdate(ctx context.Context, oldObj, newObj return nil, apierrors.NewBadRequest(fmt.Sprintf("expected old object to be a Workspace but got %T", oldObj)) } + var allErrs field.ErrorList + // check if workspace kind related fields have changed var workspaceKindChange = false var imageConfigChange = false diff --git a/workspaces/controller/internal/webhook/workspace_webhook_test.go b/workspaces/controller/internal/webhook/workspace_webhook_test.go index 9136c3ea..e6efc213 100644 --- a/workspaces/controller/internal/webhook/workspace_webhook_test.go +++ b/workspaces/controller/internal/webhook/workspace_webhook_test.go @@ -19,25 +19,31 @@ package webhook import ( "fmt" - kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" + "k8s.io/apimachinery/pkg/types" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" ) var _ = Describe("Workspace Webhook", func() { - Context("When creating Workspace under Validating Webhook", Ordered, func() { + const ( + namespaceName = "default" + ) + + Context("When creating a Workspace", Ordered, func() { var ( workspaceName string workspaceKindName string - namespaceName string ) BeforeAll(func() { - uniqueName := "ws-create-test" + uniqueName := "ws-webhook-create-test" workspaceName = fmt.Sprintf("workspace-%s", uniqueName) - namespaceName = "default" workspaceKindName = fmt.Sprintf("workspacekind-%s", uniqueName) By("creating the WorkspaceKind") @@ -55,7 +61,7 @@ var _ = Describe("Workspace Webhook", func() { Expect(k8sClient.Delete(ctx, workspaceKind)).To(Succeed()) }) - It("should reject workspace creation with an invalid WorkspaceKind", func() { + It("should reject an invalid workspace kind", func() { invalidWorkspaceKindName := "invalid-workspace-kind" By("creating the Workspace") @@ -65,7 +71,29 @@ var _ = Describe("Workspace Webhook", func() { Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("workspace kind %q not found", invalidWorkspaceKindName))) }) - It("should successfully create workspace with a valid WorkspaceKind", func() { + It("should reject an invalid imageConfig", func() { + invalidImageConfig := "invalid_image_config" + + By("creating the Workspace") + workspace := NewExampleWorkspace(workspaceName, namespaceName, workspaceKindName) + workspace.Spec.PodTemplate.Options.ImageConfig = invalidImageConfig + err := k8sClient.Create(ctx, workspace) + Expect(err).NotTo(Succeed()) + Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("imageConfig with id %q not found in workspace kind %q", invalidImageConfig, workspaceKindName))) + }) + + It("should reject an invalid podConfig", func() { + invalidPodConfig := "invalid_pod_config" + + By("creating the Workspace") + workspace := NewExampleWorkspace(workspaceName, namespaceName, workspaceKindName) + workspace.Spec.PodTemplate.Options.PodConfig = invalidPodConfig + err := k8sClient.Create(ctx, workspace) + Expect(err).NotTo(Succeed()) + Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("podConfig with id %q not found in workspace kind %q", invalidPodConfig, workspaceKindName))) + }) + + It("should accept a valid workspace", func() { By("creating the Workspace") workspace := NewExampleWorkspace(workspaceName, namespaceName, workspaceKindName) Expect(k8sClient.Create(ctx, workspace)).To(Succeed()) @@ -75,4 +103,99 @@ var _ = Describe("Workspace Webhook", func() { }) }) + Context("When updating a Workspace", Ordered, func() { + var ( + workspaceName string + workspaceKindName string + workspaceKey types.NamespacedName + ) + + BeforeAll(func() { + uniqueName := "ws-webhook-update-test" + workspaceName = fmt.Sprintf("workspace-%s", uniqueName) + workspaceKindName = fmt.Sprintf("workspacekind-%s", uniqueName) + workspaceKey = types.NamespacedName{Name: workspaceName, Namespace: namespaceName} + + By("creating the WorkspaceKind") + workspaceKind := NewExampleWorkspaceKind(workspaceKindName) + Expect(k8sClient.Create(ctx, workspaceKind)).To(Succeed()) + + By("creating the Workspace") + workspace := NewExampleWorkspace(workspaceName, namespaceName, workspaceKindName) + Expect(k8sClient.Create(ctx, workspace)).To(Succeed()) + }) + + AfterAll(func() { + By("deleting the WorkspaceKind") + workspaceKind := &kubefloworgv1beta1.WorkspaceKind{ + ObjectMeta: metav1.ObjectMeta{ + Name: workspaceKindName, + }, + } + Expect(k8sClient.Delete(ctx, workspaceKind)).To(Succeed()) + + By("deleting the Workspace") + workspace := &kubefloworgv1beta1.Workspace{ + ObjectMeta: metav1.ObjectMeta{ + Name: workspaceName, + Namespace: namespaceName, + }, + } + Expect(k8sClient.Delete(ctx, workspace)).To(Succeed()) + }) + + It("should not allow updating immutable fields", func() { + By("getting the Workspace") + workspace := &kubefloworgv1beta1.Workspace{} + Expect(k8sClient.Get(ctx, workspaceKey, workspace)).To(Succeed()) + patch := client.MergeFrom(workspace.DeepCopy()) + + By("failing to update the `spec.kind` field") + newWorkspace := workspace.DeepCopy() + newWorkspace.Spec.Kind = "new_kind" + Expect(k8sClient.Patch(ctx, newWorkspace, patch)).NotTo(Succeed()) + }) + + It("should handle imageConfig updates", func() { + By("getting the Workspace") + workspace := &kubefloworgv1beta1.Workspace{} + Expect(k8sClient.Get(ctx, workspaceKey, workspace)).To(Succeed()) + patch := client.MergeFrom(workspace.DeepCopy()) + + By("failing to update the `spec.podTemplate.options.imageConfig` field to an invalid value") + invalidPodConfig := "invalid_image_config" + newWorkspace := workspace.DeepCopy() + newWorkspace.Spec.PodTemplate.Options.ImageConfig = invalidPodConfig + err := k8sClient.Patch(ctx, newWorkspace, patch) + Expect(err).NotTo(Succeed()) + Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("imageConfig with id %q not found in workspace kind %q", invalidPodConfig, workspace.Spec.Kind))) + + By("updating the `spec.podTemplate.options.imageConfig` field to a valid value") + validImageConfig := "jupyterlab_scipy_190" + newWorkspace = workspace.DeepCopy() + newWorkspace.Spec.PodTemplate.Options.ImageConfig = validImageConfig + Expect(k8sClient.Patch(ctx, newWorkspace, patch)).To(Succeed()) + }) + + It("should handle podConfig updates", func() { + By("getting the Workspace") + workspace := &kubefloworgv1beta1.Workspace{} + Expect(k8sClient.Get(ctx, workspaceKey, workspace)).To(Succeed()) + patch := client.MergeFrom(workspace.DeepCopy()) + + By("failing to update the `spec.podTemplate.options.podConfig` field to an invalid value") + invalidPodConfig := "invalid_pod_config" + newWorkspace := workspace.DeepCopy() + newWorkspace.Spec.PodTemplate.Options.PodConfig = invalidPodConfig + err := k8sClient.Patch(ctx, newWorkspace, patch) + Expect(err).NotTo(Succeed()) + Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("podConfig with id %q not found in workspace kind %q", invalidPodConfig, workspace.Spec.Kind))) + + By("updating the `spec.podTemplate.options.podConfig` field to a valid value") + validPodConfig := "small_cpu" + newWorkspace = workspace.DeepCopy() + newWorkspace.Spec.PodTemplate.Options.PodConfig = validPodConfig + Expect(k8sClient.Patch(ctx, newWorkspace, patch)).To(Succeed()) + }) + }) }) diff --git a/workspaces/controller/internal/webhook/workspacekind_webhook.go b/workspaces/controller/internal/webhook/workspacekind_webhook.go index bee30a61..e84fc1b6 100644 --- a/workspaces/controller/internal/webhook/workspacekind_webhook.go +++ b/workspaces/controller/internal/webhook/workspacekind_webhook.go @@ -21,13 +21,9 @@ import ( "errors" "fmt" "reflect" + "sync" - kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" - "github.com/kubeflow/notebooks/workspaces/controller/internal/controller" - "github.com/kubeflow/notebooks/workspaces/controller/internal/helper" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -37,6 +33,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" + "github.com/kubeflow/notebooks/workspaces/controller/internal/controller" + "github.com/kubeflow/notebooks/workspaces/controller/internal/helper" ) // WorkspaceKindValidator validates a Workspace object @@ -62,13 +62,13 @@ func (v *WorkspaceKindValidator) ValidateCreate(ctx context.Context, obj runtime log := log.FromContext(ctx) log.V(1).Info("validating WorkspaceKind create") - var allErrs field.ErrorList - workspaceKind, ok := obj.(*kubefloworgv1beta1.WorkspaceKind) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a WorkspaceKind object but got %T", obj)) } + var allErrs field.ErrorList + // validate the extra environment variables allErrs = append(allErrs, validateExtraEnv(workspaceKind)...) @@ -125,22 +125,20 @@ func (v *WorkspaceKindValidator) ValidateUpdate(ctx context.Context, oldObj, new log := log.FromContext(ctx) log.V(1).Info("validating WorkspaceKind update") - var allErrs field.ErrorList - newWorkspaceKind, ok := newObj.(*kubefloworgv1beta1.WorkspaceKind) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a WorkspaceKind object but got %T", newObj)) } oldWorkspaceKind, ok := oldObj.(*kubefloworgv1beta1.WorkspaceKind) if !ok { - return nil, apierrors.NewBadRequest(fmt.Sprintf("expected old object to be a WorkspaceKind but got %T", oldObj)) + return nil, apierrors.NewInternalError(fmt.Errorf("old object is not a WorkspaceKind, but a %T", oldObj)) } - // get usage count for imageConfig and podConfig values - imageConfigUsageCount, podConfigUsageCount, err := v.getOptionsUsageCounts(ctx, oldWorkspaceKind) - if err != nil { - return nil, err - } + var allErrs field.ErrorList + + // get functions to lazily fetch usage counts for imageConfig and podConfig values + // NOTE: the cluster is only queried when either function is called for the first time + getImageConfigUsageCount, getPodConfigUsageCount := v.getLazyOptionUsageCountFuncs(ctx, oldWorkspaceKind) // validate the extra environment variables if !reflect.DeepEqual(newWorkspaceKind.Spec.PodTemplate.ExtraEnv, oldWorkspaceKind.Spec.PodTemplate.ExtraEnv) { @@ -170,12 +168,13 @@ func (v *WorkspaceKindValidator) ValidateUpdate(ctx context.Context, oldObj, new toValidateImageConfigIds[imageConfigValue.Id] = true // we always need to validate the imageConfig redirects if an imageConfig value was added + // because the new imageConfig value could be used by a redirect or cause a cycle shouldValidateImageConfigRedirects = true } else { - // check if this imageConfig value is used by any workspaces - var usageCount int32 - if usageCount, exists = imageConfigUsageCount[imageConfigValue.Id]; !exists { - return nil, apierrors.NewInternalError(fmt.Errorf("usage count not found for imageConfig value %q", imageConfigValue.Id)) + // if we haven't already decided to validate the imageConfig redirects, + // check if the redirect has changed + if !shouldValidateImageConfigRedirects && !reflect.DeepEqual(oldImageConfigIdMap[imageConfigValue.Id].Redirect, imageConfigValue.Redirect) { + shouldValidateImageConfigRedirects = true } // check if the spec has changed @@ -183,28 +182,30 @@ func (v *WorkspaceKindValidator) ValidateUpdate(ctx context.Context, oldObj, new // we need to validate this imageConfig value since it has changed toValidateImageConfigIds[imageConfigValue.Id] = true + // check how many workspaces are using this imageConfig value + usageCount, err := getImageConfigUsageCount(imageConfigValue.Id) + if err != nil { + // if the usage count is not found, we cannot validate the WorkspaceKind further + return nil, apierrors.NewInternalError(fmt.Errorf("failed to get usage count for imageConfig with id %q: %w", imageConfigValue.Id, err)) + } + // if this imageConfig is used by any workspaces, mark this imageConfig as bad, - // (the spec is immutable while in use) + // the spec is immutable while in use if usageCount > 0 { badChangedImageConfigIds[imageConfigValue.Id] = true } } - - // if we haven't already decided to validate the imageConfig redirects, - // check if the redirect has changed - if !shouldValidateImageConfigRedirects && !reflect.DeepEqual(oldImageConfigIdMap[imageConfigValue.Id].Redirect, imageConfigValue.Redirect) { - shouldValidateImageConfigRedirects = true - } } } for id := range oldImageConfigIdMap { // check if this imageConfig value was removed if _, exists := newImageConfigIdMap[id]; !exists { - // check if this imageConfig value is used by any workspaces - var usageCount int32 - if usageCount, exists = imageConfigUsageCount[id]; !exists { - return nil, apierrors.NewInternalError(fmt.Errorf("usage count not found for imageConfig value %q", id)) + // check how many workspaces are using this imageConfig value + usageCount, err := getImageConfigUsageCount(id) + if err != nil { + // if the usage count is not found, we cannot validate the WorkspaceKind further + return nil, apierrors.NewInternalError(fmt.Errorf("failed to get usage count for imageConfig with id %q: %w", id, err)) } // if this imageConfig is used by any workspaces, mark this imageConfig as bad, @@ -214,6 +215,7 @@ func (v *WorkspaceKindValidator) ValidateUpdate(ctx context.Context, oldObj, new } // we always need to validate the imageConfig redirects if an imageConfig was removed + // because an existing redirect could be pointing to the removed imageConfig value shouldValidateImageConfigRedirects = true } } @@ -237,36 +239,50 @@ func (v *WorkspaceKindValidator) ValidateUpdate(ctx context.Context, oldObj, new // check if the podConfig value is new if _, exists := oldPodConfigIdMap[podConfigValue.Id]; !exists { // we always need to validate the podConfig redirects if a podConfig was added + // because the new podConfig value could be used by a redirect or cause a cycle shouldValidatePodConfigRedirects = true } else { - // check if this podConfig value is used by any workspaces - var usageCount int32 - if usageCount, exists = podConfigUsageCount[podConfigValue.Id]; !exists { - return nil, apierrors.NewInternalError(fmt.Errorf("usage count not found for podConfig value %q", podConfigValue.Id)) + // if we haven't already decided to validate the podConfig redirects, + // check if the redirect has changed + if !shouldValidatePodConfigRedirects && !reflect.DeepEqual(oldPodConfigIdMap[podConfigValue.Id].Redirect, podConfigValue.Redirect) { + shouldValidatePodConfigRedirects = true } - // normalize the podConfig specs - oldPodConfigSpec := oldPodConfigIdMap[podConfigValue.Id].Spec - err := normalizePodConfigSpec(oldPodConfigSpec) + // we must normalize the podConfig specs so that we can compare them + newPodConfigSpec := podConfigValue.Spec + err := helper.NormalizePodConfigSpec(newPodConfigSpec) if err != nil { - return nil, apierrors.NewInternalError(fmt.Errorf("failed to normalize podConfig spec: %w", err)) + podConfigValueSpecPath := field.NewPath("spec", "podTemplate", "options", "podConfig", "values").Key(podConfigValue.Id).Child("spec") + allErrs = append(allErrs, field.InternalError(podConfigValueSpecPath, fmt.Errorf("failed to normalize podConfig spec: %w", err))) + + // if the spec could not be normalized, we cannot validate the WorkspaceKind further + return nil, apierrors.NewInvalid( + schema.GroupKind{Group: kubefloworgv1beta1.GroupVersion.Group, Kind: "WorkspaceKind"}, + newWorkspaceKind.Name, + allErrs, + ) } - newPodConfigSpec := podConfigValue.Spec - err = normalizePodConfigSpec(newPodConfigSpec) + oldPodConfigSpec := oldPodConfigIdMap[podConfigValue.Id].Spec + err = helper.NormalizePodConfigSpec(oldPodConfigSpec) if err != nil { - return nil, apierrors.NewInternalError(fmt.Errorf("failed to normalize podConfig spec: %w", err)) + // this should never happen, as it would indicate that the old podConfig spec is invalid + return nil, apierrors.NewInternalError(fmt.Errorf("old podConfig spec of %q could not be normalized: %w", podConfigValue.Id, err)) } - // if this podConfig is used by any workspaces, check if the spec has changed - // if the spec has changed, mark this podConfig as bad (the spec is immutable while in use) - if usageCount > 0 && !reflect.DeepEqual(oldPodConfigSpec, newPodConfigSpec) { - badChangedPodConfigIds[podConfigValue.Id] = true - } + // check if the spec has changed + if !reflect.DeepEqual(oldPodConfigSpec, newPodConfigSpec) { + // check how many workspaces are using this podConfig value + usageCount, err := getPodConfigUsageCount(podConfigValue.Id) + if err != nil { + // if the usage count is not found, we cannot validate the WorkspaceKind further + return nil, apierrors.NewInternalError(fmt.Errorf("failed to get usage count for podConfig with id %q: %w", podConfigValue.Id, err)) + } - // if we haven't already decided to validate the podConfig redirects, - // check if the redirect has changed - if !shouldValidatePodConfigRedirects && !reflect.DeepEqual(oldPodConfigIdMap[podConfigValue.Id].Redirect, podConfigValue.Redirect) { - shouldValidatePodConfigRedirects = true + // if this podConfig is used by any workspaces, mark this podConfig as bad, + // the spec is immutable while in use + if usageCount > 0 { + badChangedPodConfigIds[podConfigValue.Id] = true + } } } } @@ -274,10 +290,11 @@ func (v *WorkspaceKindValidator) ValidateUpdate(ctx context.Context, oldObj, new // check if this podConfig value was removed if _, exists := newPodConfigIdMap[id]; !exists { - // check if this podConfig value is used by any workspaces - var usageCount int32 - if usageCount, exists = podConfigUsageCount[id]; !exists { - return nil, apierrors.NewInternalError(fmt.Errorf("usage count not found for podConfig value %q", id)) + // check how many workspaces are using this podConfig value + usageCount, err := getPodConfigUsageCount(id) + if err != nil { + // if the usage count is not found, we cannot validate the WorkspaceKind further + return nil, apierrors.NewInternalError(fmt.Errorf("failed to get usage count for podConfig with id %q: %w", id, err)) } // if this podConfig is used by any workspaces, mark this podConfig as bad, @@ -287,17 +304,16 @@ func (v *WorkspaceKindValidator) ValidateUpdate(ctx context.Context, oldObj, new } // we always need to validate the podConfig redirects if a podConfig was removed + // because an existing redirect could be pointing to the removed imageConfig value shouldValidatePodConfigRedirects = true } } // validate default options - if oldWorkspaceKind.Spec.PodTemplate.Options.ImageConfig.Spawner.Default != newWorkspaceKind.Spec.PodTemplate.Options.ImageConfig.Spawner.Default { - allErrs = append(allErrs, validateDefaultImageConfig(newWorkspaceKind, newImageConfigIdMap)...) - } - if oldWorkspaceKind.Spec.PodTemplate.Options.PodConfig.Spawner.Default != newWorkspaceKind.Spec.PodTemplate.Options.PodConfig.Spawner.Default { - allErrs = append(allErrs, validateDefaultPodConfig(newWorkspaceKind, newPodConfigIdMap)...) - } + // NOTE: we always check this because it's cheap, and otherwise we would need to keep track of if + // any options were changed or removed + allErrs = append(allErrs, validateDefaultImageConfig(newWorkspaceKind, newImageConfigIdMap)...) + allErrs = append(allErrs, validateDefaultPodConfig(newWorkspaceKind, newPodConfigIdMap)...) // validate imageConfig values // NOTE: we only need to validate new or changed imageConfig values @@ -307,7 +323,7 @@ func (v *WorkspaceKindValidator) ValidateUpdate(ctx context.Context, oldObj, new allErrs = append(allErrs, validateImageConfigValue(&imageConfigValue, imageConfigValuePath)...) } - // validate bad imageConfig values + // process bad imageConfig values for id := range badChangedImageConfigIds { imageConfigValuePath := field.NewPath("spec", "podTemplate", "options", "imageConfig", "values").Key(id) allErrs = append(allErrs, field.Invalid(imageConfigValuePath, id, fmt.Sprintf("imageConfig value %q is in use and cannot be changed", id))) @@ -317,7 +333,7 @@ func (v *WorkspaceKindValidator) ValidateUpdate(ctx context.Context, oldObj, new allErrs = append(allErrs, field.Invalid(imageConfigValuePath, id, fmt.Sprintf("imageConfig value %q is in use and cannot be removed", id))) } - // validate bad podConfig values + // process bad podConfig values for id := range badChangedPodConfigIds { podConfigValuePath := field.NewPath("spec", "podTemplate", "options", "podConfig", "values").Key(id) allErrs = append(allErrs, field.Invalid(podConfigValuePath, id, fmt.Sprintf("podConfig value %q is in use and cannot be changed", id))) @@ -382,26 +398,63 @@ func (v *WorkspaceKindValidator) ValidateDelete(ctx context.Context, obj runtime return nil, nil } -// getOptionsUsageCounts returns the usage counts for each imageConfig and podConfig value -func (v *WorkspaceKindValidator) getOptionsUsageCounts(ctx context.Context, workspaceKind *kubefloworgv1beta1.WorkspaceKind) (map[string]int32, map[string]int32, *apierrors.StatusError) { - podConfigUsageCount := make(map[string]int32) - imageConfigUsageCount := make(map[string]int32) +// getLazyOptionUsageCountFuncs returns functions that get usage counts for imageConfig and podConfig values in a WorkspaceKind +// the cluster is only queried when either function is called for the first time +func (v *WorkspaceKindValidator) getLazyOptionUsageCountFuncs(ctx context.Context, workspaceKind *kubefloworgv1beta1.WorkspaceKind) (func(string) (int32, error), func(string) (int32, error)) { + + // usageCountWrapper is a wrapper for the usage count maps + // NOTE: this is needed because sync.OnceValues can only return 2 values + type usageCountWrapper struct { + imageConfigUsageCounts map[string]int32 + podConfigUsageCounts map[string]int32 + } + + // this function will lazily fetch the usage counts for each option + lazyGetOptionsUsageCounts := sync.OnceValues(func() (usageCountWrapper, error) { + imageConfigUsageCounts, podConfigUsageCounts, err := v.getOptionsUsageCounts(ctx, workspaceKind) + if err != nil { + return usageCountWrapper{}, err + } + return usageCountWrapper{ + imageConfigUsageCounts: imageConfigUsageCounts, + podConfigUsageCounts: podConfigUsageCounts, + }, nil + }) + + // getImageConfigUsageCount returns the usage count for an imageConfig value + getImageConfigUsageCount := func(id string) (int32, error) { + wrapper, err := lazyGetOptionsUsageCounts() + if err != nil { + return 0, err + } + if count, exists := wrapper.imageConfigUsageCounts[id]; !exists { + return 0, errors.New("unknown imageConfig id") + } else { + return count, nil + } + } - // if possible, we get the counts from the status of the WorkspaceKind. these counts are updated by the - // controller so could be stale if the controller is not running or a workspace was very recently added or removed. - // however, since the controller gracefully handles cases of a Workspace referencing a non-existent imageConfig - // or podConfig value, we can safely use these counts to validate the WorkspaceKind, Workspaces will simply be - // put into an error state and the user can correct the issue. these counts will NOT be set in the WorkspaceKind - // unit tests, so we implement a fallback method to count the Workspaces that are using each option. - if len(workspaceKind.Status.PodTemplateOptions.ImageConfig) > 0 && len(workspaceKind.Status.PodTemplateOptions.PodConfig) > 0 { - for _, imageConfigMetrics := range workspaceKind.Status.PodTemplateOptions.ImageConfig { - imageConfigUsageCount[imageConfigMetrics.Id] = imageConfigMetrics.Workspaces + // getPodConfigUsageCount returns the usage count for a podConfig value + getPodConfigUsageCount := func(id string) (int32, error) { + wrapper, err := lazyGetOptionsUsageCounts() + if err != nil { + return 0, err } - for _, podConfigMetrics := range workspaceKind.Status.PodTemplateOptions.PodConfig { - podConfigUsageCount[podConfigMetrics.Id] = podConfigMetrics.Workspaces + if count, exists := wrapper.podConfigUsageCounts[id]; !exists { + return 0, errors.New("unknown podConfig id") + } else { + return count, nil } } + return getImageConfigUsageCount, getPodConfigUsageCount +} + +// getOptionsUsageCounts returns the usage counts for each imageConfig and podConfig value +func (v *WorkspaceKindValidator) getOptionsUsageCounts(ctx context.Context, workspaceKind *kubefloworgv1beta1.WorkspaceKind) (map[string]int32, map[string]int32, error) { + imageConfigUsageCount := make(map[string]int32) + podConfigUsageCount := make(map[string]int32) + // fetch all Workspaces that are using this WorkspaceKind workspaces := &kubefloworgv1beta1.WorkspaceList{} listOpts := &client.ListOptions{ @@ -409,7 +462,7 @@ func (v *WorkspaceKindValidator) getOptionsUsageCounts(ctx context.Context, work Namespace: "", // fetch Workspaces in all namespaces } if err := v.List(ctx, workspaces, listOpts); err != nil { - return nil, nil, apierrors.NewInternalError(err) + return nil, nil, err } // count the number of Workspaces using each option @@ -508,14 +561,14 @@ func validateImageConfigRedirects(imageConfigIdMap map[string]kubefloworgv1beta1 // check if there is a cycle involving the current node if cycle := helper.DetectGraphCycle(id, checkedNodes, imageConfigRedirectMap); cycle != nil { redirectToPath := field.NewPath("spec", "podTemplate", "options", "imageConfig", "values").Key(id).Child("redirect", "to") - errs = append(errs, field.Invalid(redirectToPath, redirectTo, fmt.Sprintf("cycle detected: %v", cycle))) + errs = append(errs, field.Invalid(redirectToPath, redirectTo, fmt.Sprintf("imageConfig redirect cycle detected: %v", cycle))) break // stop checking redirects if a cycle is detected } // ensure the target of the redirect exists if _, exists := imageConfigIdMap[redirectTo]; !exists { redirectToPath := field.NewPath("spec", "podTemplate", "options", "imageConfig", "values").Key(id).Child("redirect", "to") - errs = append(errs, field.Invalid(redirectToPath, redirectTo, fmt.Sprintf("invalid redirect target %q", redirectTo))) + errs = append(errs, field.Invalid(redirectToPath, redirectTo, fmt.Sprintf("target imageConfig %q does not exist", redirectTo))) } } @@ -532,64 +585,16 @@ func validatePodConfigRedirects(podConfigIdMap map[string]kubefloworgv1beta1.Pod // check if there is a cycle involving the current node if cycle := helper.DetectGraphCycle(id, checkedNodes, podConfigRedirectMap); cycle != nil { redirectToPath := field.NewPath("spec", "podTemplate", "options", "podConfig", "values").Key(id).Child("redirect", "to") - errs = append(errs, field.Invalid(redirectToPath, redirectTo, fmt.Sprintf("cycle detected: %v", cycle))) + errs = append(errs, field.Invalid(redirectToPath, redirectTo, fmt.Sprintf("podConfig redirect cycle detected: %v", cycle))) break // stop checking redirects if a cycle is detected } // ensure the target of the redirect exists if _, exists := podConfigIdMap[redirectTo]; !exists { redirectToPath := field.NewPath("spec", "podTemplate", "options", "podConfig", "values").Key(id).Child("redirect", "to") - errs = append(errs, field.Invalid(redirectToPath, redirectTo, fmt.Sprintf("invalid redirect target %q", redirectTo))) + errs = append(errs, field.Invalid(redirectToPath, redirectTo, fmt.Sprintf("target podConfig %q does not exist", redirectTo))) } } return errs } - -// normalizePodConfigSpec normalizes a PodConfigSpec so that it can be compared with reflect.DeepEqual -func normalizePodConfigSpec(spec kubefloworgv1beta1.PodConfigSpec) (err error) { - // Normalize Affinity - if spec.Affinity != nil && reflect.DeepEqual(spec.Affinity, corev1.Affinity{}) { - spec.Affinity = nil - } - - // Normalize NodeSelector - if spec.NodeSelector != nil && len(spec.NodeSelector) == 0 { - spec.NodeSelector = nil - } - - // Normalize Tolerations - if spec.Tolerations != nil && len(spec.Tolerations) == 0 { - spec.Tolerations = nil - } - - // Normalize Resources.Requests - if reflect.DeepEqual(spec.Resources.Requests, corev1.ResourceList{}) { - spec.Resources.Requests = nil - } - if spec.Resources.Requests != nil { - for key, value := range spec.Resources.Requests { - q, err := resource.ParseQuantity(value.String()) - if err != nil { - return err - } - spec.Resources.Requests[key] = q - } - } - - // Normalize Resources.Limits - if reflect.DeepEqual(spec.Resources.Limits, corev1.ResourceList{}) { - spec.Resources.Limits = nil - } - if spec.Resources.Limits != nil { - for key, value := range spec.Resources.Limits { - q, err := resource.ParseQuantity(value.String()) - if err != nil { - return err - } - spec.Resources.Limits[key] = q - } - } - - return nil -} diff --git a/workspaces/controller/internal/webhook/workspacekind_webhook_test.go b/workspaces/controller/internal/webhook/workspacekind_webhook_test.go index 9ef7f968..1a13a94f 100644 --- a/workspaces/controller/internal/webhook/workspacekind_webhook_test.go +++ b/workspaces/controller/internal/webhook/workspacekind_webhook_test.go @@ -18,78 +18,74 @@ package webhook import ( "fmt" - "time" - "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + + kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" ) var _ = Describe("WorkspaceKind Webhook", func() { const ( namespaceName = "default" - - // how long to wait in "Eventually" blocks - timeout = time.Second * 10 - - // how long to wait in "Consistently" blocks - duration = time.Second * 10 - - // how frequently to poll for conditions - interval = time.Millisecond * 250 ) - Context("When creating WorkspaceKind under Validating Webhook", Ordered, func() { + Context("When creating a WorkspaceKind", Ordered, func() { testCases := []struct { description string - workspaceKind *v1beta1.WorkspaceKind + workspaceKind *kubefloworgv1beta1.WorkspaceKind shouldSucceed bool }{ { - description: "should reject WorkspaceKind creation with cycles in ImageConfig options", + description: "should accept creation of a valid WorkspaceKind", + workspaceKind: NewExampleWorkspaceKind("wsk-webhook-create-test"), + shouldSucceed: true, + }, + { + description: "should reject creation with cycle in imageConfig redirects", workspaceKind: NewExampleWorkspaceKindWithImageConfigCycle("wsk-webhook-image-config-cycle-test"), shouldSucceed: false, }, { - description: "should reject WorkspaceKind creation with cycles in PodConfig options", + description: "should reject creation with cycle in podConfig redirects", workspaceKind: NewExampleWorkspaceKindWithPodConfigCycle("wsk-webhook-pod-config-cycle-test"), shouldSucceed: false, }, { - description: "should reject WorkspaceKind creation with invalid redirects in ImageConfig options", - workspaceKind: NewExampleWorkspaceKindWithInvalidImageConfig("wsk-webhook-image-config-invalid-test"), + description: "should reject creation with invalid redirect target in imageConfig options", + workspaceKind: NewExampleWorkspaceKindWithInvalidImageConfigRedirect("wsk-webhook-image-config-invalid-redirect-test"), shouldSucceed: false, }, { - description: "should reject WorkspaceKind creation with invalid redirects in PodConfig options", - workspaceKind: NewExampleWorkspaceKindWithInvalidPodConfig("wsk-webhook-pod-config-invalid-test"), + description: "should reject creation with invalid redirect target in podConfig options", + workspaceKind: NewExampleWorkspaceKindWithInvalidPodConfigRedirect("wsk-webhook-pod-config-invalid-redirect-test"), shouldSucceed: false, }, { - description: "should reject WorkspaceKind creation if the default ImageConfig option is missing", + description: "should reject creation with invalid default imageConfig", workspaceKind: NewExampleWorkspaceKindWithInvalidDefaultImageConfig("wsk-webhook-image-config-default-test"), shouldSucceed: false, }, { - description: "should reject WorkspaceKind creation if the default PodConfig option is missing", + description: "should reject creation with invalid default podConfig", workspaceKind: NewExampleWorkspaceKindWithInvalidDefaultPodConfig("wsk-webhook-pod-config-default-test"), shouldSucceed: false, }, { - description: "should reject WorkspaceKind creation with non-unique ports in PodConfig", - workspaceKind: NewExampleWorkspaceKindWithDuplicatePorts("wsk-webhook-ports-port-not-unique-test"), + description: "should reject creation with duplicate ports in imageConfig", + workspaceKind: NewExampleWorkspaceKindWithDuplicatePorts("wsk-webhook-image-config-duplicate-ports-test"), shouldSucceed: false, }, { - description: "should reject WorkspaceKind creation if extraEnv[].value is not a valid Go template", - workspaceKind: NewExampleWorkspaceKindWithInvalidExtraEnvValue("wsk-webhook-extra-env-value-invalid-test"), + description: "should reject creation if extraEnv[].value is not a valid Go template", + workspaceKind: NewExampleWorkspaceKindWithInvalidExtraEnvValue("wsk-webhook-invalid-extra-env-value-test"), shouldSucceed: false, }, } @@ -97,22 +93,25 @@ var _ = Describe("WorkspaceKind Webhook", func() { for _, tc := range testCases { tc := tc // Create a new instance of tc to avoid capturing the loop variable. It(tc.description, func() { - By("creating the WorkspaceKind") if tc.shouldSucceed { + By("creating the WorkspaceKind") Expect(k8sClient.Create(ctx, tc.workspaceKind)).To(Succeed()) + + By("deleting the WorkspaceKind") + Expect(k8sClient.Delete(ctx, tc.workspaceKind)).To(Succeed()) } else { + By("creating the WorkspaceKind") Expect(k8sClient.Create(ctx, tc.workspaceKind)).NotTo(Succeed()) } }) } - }) - Context("When updating WorkspaceKind under Validating Webhook", Ordered, func() { + Context("When updating a WorkspaceKind", Ordered, func() { var ( workspaceKindName string workspaceKindKey types.NamespacedName - workspaceKind *v1beta1.WorkspaceKind + workspaceKind *kubefloworgv1beta1.WorkspaceKind ) BeforeAll(func() { @@ -125,10 +124,8 @@ var _ = Describe("WorkspaceKind Webhook", func() { Expect(k8sClient.Create(ctx, createdWorkspaceKind)).To(Succeed()) By("getting the created WorkspaceKind") - workspaceKind = &v1beta1.WorkspaceKind{} - Eventually(func() error { - return k8sClient.Get(ctx, workspaceKindKey, workspaceKind) - }, timeout, interval).Should(Succeed()) + workspaceKind = &kubefloworgv1beta1.WorkspaceKind{} + Expect(k8sClient.Get(ctx, workspaceKindKey, workspaceKind)).To(Succeed()) }) AfterAll(func() { @@ -137,125 +134,201 @@ var _ = Describe("WorkspaceKind Webhook", func() { }) testCases := []struct { - description string - modifyKindFn func(*v1beta1.WorkspaceKind) - workspaceName *string + description string + // modifyKindFn is a function that modifies the WorkspaceKind in some way + // and returns a string matcher for the expected error message (if any) + modifyKindFn func(*kubefloworgv1beta1.WorkspaceKind) string + workspaceName string + + // if shouldSucceed is true, the test is expected to succeed shouldSucceed bool }{ { - description: "should reject updates to used imageConfig spec", - modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { + description: "should reject updates to in-use imageConfig spec", + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { wsk.Spec.PodTemplate.Options.ImageConfig.Values[0].Spec.Image = "new-image:latest" + return fmt.Sprintf("imageConfig value %q is in use and cannot be changed", wsk.Spec.PodTemplate.Options.ImageConfig.Values[0].Id) }, - workspaceName: ptr.To("ws-webhook-update-image-config-spec-test"), + workspaceName: "wsk-webhook-update-image-config-test", shouldSucceed: false, }, { - description: "should reject updates to used podConfig spec", - modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { + description: "should reject updates to in-use podConfig spec", + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Spec.Resources = &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1.5"), }, } + return fmt.Sprintf("podConfig value %q is in use and cannot be changed", wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Id) }, - workspaceName: ptr.To("ws-webhook-update-pod-config-spec-test"), + workspaceName: "ws-webhook-update-pod-config-test", shouldSucceed: false, }, { - description: "should reject WorkspaceKind update with cycles in imageConfig options", - modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { - wsk.Spec.PodTemplate.Options.ImageConfig.Values[1].Redirect = &v1beta1.OptionRedirect{To: "jupyterlab_scipy_190"} + description: "should reject removing in-use imageConfig values", + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + toBeRemoved := "jupyterlab_scipy_180" + newValues := make([]kubefloworgv1beta1.ImageConfigValue, 0) + for _, value := range wsk.Spec.PodTemplate.Options.ImageConfig.Values { + if value.Id != toBeRemoved { + newValues = append(newValues, value) + } + } + wsk.Spec.PodTemplate.Options.ImageConfig.Values = newValues + return fmt.Sprintf("imageConfig value %q is in use and cannot be removed", toBeRemoved) }, + workspaceName: "ws-webhook-update-image-config-test", shouldSucceed: false, }, { - description: "should reject WorkspaceKind update with invalid redirects in ImageConfig options", - modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { - wsk.Spec.PodTemplate.Options.ImageConfig.Values[1].Redirect = &v1beta1.OptionRedirect{To: "invalid-image-config"} + description: "should reject removing in-use podConfig values", + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + toBeRemoved := "tiny_cpu" + newValues := make([]kubefloworgv1beta1.PodConfigValue, 0) + for _, value := range wsk.Spec.PodTemplate.Options.PodConfig.Values { + if value.Id != toBeRemoved { + newValues = append(newValues, value) + } + } + wsk.Spec.PodTemplate.Options.PodConfig.Values = newValues + return fmt.Sprintf("podConfig value %q is in use and cannot be removed", toBeRemoved) }, + workspaceName: "ws-webhook-update-pod-config-test", shouldSucceed: false, }, { - description: "should reject WorkspaceKind update with cycles in PodConfig options", - modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { - wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Redirect = &v1beta1.OptionRedirect{To: "small_cpu"} - wsk.Spec.PodTemplate.Options.PodConfig.Values[1].Redirect = &v1beta1.OptionRedirect{To: "tiny_cpu"} + description: "should reject updating an imageConfig value to create a self-cycle", + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + valueId := wsk.Spec.PodTemplate.Options.ImageConfig.Values[1].Id + wsk.Spec.PodTemplate.Options.ImageConfig.Values[1].Redirect = &kubefloworgv1beta1.OptionRedirect{To: valueId} + return fmt.Sprintf("imageConfig redirect cycle detected: [%s]", valueId) }, shouldSucceed: false, }, { - description: "should reject WorkspaceKind creation with invalid redirects in PodConfig options", - modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { - wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Redirect = &v1beta1.OptionRedirect{To: "invalid-pod-config"} + description: "should reject updating a podConfig value to create a 2-step cycle", + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + step1 := wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Id + step2 := wsk.Spec.PodTemplate.Options.PodConfig.Values[1].Id + wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Redirect = &kubefloworgv1beta1.OptionRedirect{To: step2} + wsk.Spec.PodTemplate.Options.PodConfig.Values[1].Redirect = &kubefloworgv1beta1.OptionRedirect{To: step1} + return "podConfig redirect cycle detected: [" // there is no guarantee on which element will be first }, shouldSucceed: false, }, { - description: "should reject updates to WorkspaceKind with missing default imageConfig", - modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { - wsk.Spec.PodTemplate.Options.ImageConfig.Spawner.Default = "invalid-image-config" + description: "should reject updating an imageConfig to redirect to a non-existent value", + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + invalidTarget := "invalid_image_config" + wsk.Spec.PodTemplate.Options.ImageConfig.Values[1].Redirect = &kubefloworgv1beta1.OptionRedirect{To: invalidTarget} + return fmt.Sprintf("target imageConfig %q does not exist", invalidTarget) }, shouldSucceed: false, }, { - description: "should reject updates to WorkspaceKind with missing default podConfig", - modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { - wsk.Spec.PodTemplate.Options.PodConfig.Spawner.Default = "invalid-pod-config" + description: "should reject updating a podConfig to redirect to a non-existent value", + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + invalidTarget := "invalid_pod_config" + wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Redirect = &kubefloworgv1beta1.OptionRedirect{To: invalidTarget} + return fmt.Sprintf("target podConfig %q does not exist", invalidTarget) }, + shouldSucceed: false, }, { - description: "should reject updates to WorkspaceKind if extraEnv[].value is not a valid Go template", - modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { - wsk.Spec.PodTemplate.ExtraEnv[0].Value = `{{ httpPathPrefix "jupyterlab" }` + description: "should reject updating the default imageConfig value to a non-existent value", + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + invalidDefault := "invalid_image_config" + wsk.Spec.PodTemplate.Options.ImageConfig.Spawner.Default = invalidDefault + return fmt.Sprintf("default imageConfig %q not found", invalidDefault) }, shouldSucceed: false, }, { - description: "should accept updates to WorkspaceKind with valid extraEnv[].value Go template", - modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { - wsk.Spec.PodTemplate.ExtraEnv[0].Value = `{{ httpPathPrefix "jupyterlab" }}` + description: "should reject updating the default podConfig value to a non-existent value", + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + invalidDefault := "invalid_pod_config" + wsk.Spec.PodTemplate.Options.PodConfig.Spawner.Default = invalidDefault + return fmt.Sprintf("default podConfig %q not found", invalidDefault) }, - shouldSucceed: true, + shouldSucceed: false, }, { - description: "should reject updates that remove ImageConfig in use", - modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { - wsk.Spec.PodTemplate.Options.ImageConfig.Values = wsk.Spec.PodTemplate.Options.ImageConfig.Values[1:] + description: "should reject updating an imageConfig to have duplicate ports", + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + duplicatePortNumber := int32(8888) + wsk.Spec.PodTemplate.Options.ImageConfig.Values[1].Spec.Ports = []kubefloworgv1beta1.ImagePort{ + { + Id: "jupyterlab", + DisplayName: "JupyterLab", + Port: duplicatePortNumber, + Protocol: "HTTP", + }, + { + Id: "jupyterlab2", + DisplayName: "JupyterLab2", + Port: duplicatePortNumber, + Protocol: "HTTP", + }, + } + return fmt.Sprintf("port %d is defined more than once", duplicatePortNumber) }, - workspaceName: ptr.To("ws-webhook-update-image-config-test"), shouldSucceed: false, }, { - description: "should reject updates that remove podConfig in use", - modifyKindFn: func(wsk *v1beta1.WorkspaceKind) { - wsk.Spec.PodTemplate.Options.PodConfig.Values = wsk.Spec.PodTemplate.Options.PodConfig.Values[1:] + description: "should reject updating an extraEnv[].value to an invalid Go template", + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + invalidValue := `{{ httpPathPrefix "jupyterlab" }` + wsk.Spec.PodTemplate.ExtraEnv[0].Value = invalidValue + return fmt.Sprintf("failed to parse template %q", invalidValue) }, - workspaceName: ptr.To("ws-webhook-update-pod-config-test"), shouldSucceed: false, }, + { + description: "should accept updating an extraEnv[].value to a valid Go template", + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + wsk.Spec.PodTemplate.ExtraEnv[0].Value = `{{ httpPathPrefix "jupyterlab" }}` + return "" + }, + shouldSucceed: true, + }, } for _, tc := range testCases { tc := tc // Create a new instance of tc to avoid capturing the loop variable. It(tc.description, func() { - if tc.workspaceName != nil { - By("creating a Workspace with the WorkspaceKind") - workspace := NewExampleWorkspace(*tc.workspaceName, namespaceName, workspaceKind.Name) + if tc.workspaceName != "" { + By("creating a Workspace that uses the WorkspaceKind") + workspace := NewExampleWorkspace(tc.workspaceName, namespaceName, workspaceKindName) Expect(k8sClient.Create(ctx, workspace)).To(Succeed()) } patch := client.MergeFrom(workspaceKind.DeepCopy()) modifiedWorkspaceKind := workspaceKind.DeepCopy() + expectedErrorMessage := tc.modifyKindFn(modifiedWorkspaceKind) - tc.modifyKindFn(modifiedWorkspaceKind) + By("updating the WorkspaceKind") if tc.shouldSucceed { Expect(k8sClient.Patch(ctx, modifiedWorkspaceKind, patch)).To(Succeed()) } else { - Expect(k8sClient.Patch(ctx, modifiedWorkspaceKind, patch)).NotTo(Succeed()) + err := k8sClient.Patch(ctx, modifiedWorkspaceKind, patch) + Expect(err).NotTo(Succeed()) + if expectedErrorMessage != "" { + Expect(err.Error()).To(ContainSubstring(expectedErrorMessage)) + } + } + + if tc.workspaceName != "" { + By("deleting the Workspace that uses the WorkspaceKind") + workspace := &kubefloworgv1beta1.Workspace{ + ObjectMeta: metav1.ObjectMeta{ + Name: tc.workspaceName, + Namespace: namespaceName, + }, + } + Expect(k8sClient.Delete(ctx, workspace)).To(Succeed()) } }) } }) - }) diff --git a/workspaces/controller/test/e2e/e2e_test.go b/workspaces/controller/test/e2e/e2e_test.go index 0e2d2f2a..52c7493b 100644 --- a/workspaces/controller/test/e2e/e2e_test.go +++ b/workspaces/controller/test/e2e/e2e_test.go @@ -43,6 +43,9 @@ const ( workspacePortInt = 8888 workspacePortId = "jupyterlab" + // workspacekind configs + workspaceKindName = "jupyterlab" + // curl image curlImage = "curlimages/curl:8.9.1" @@ -50,7 +53,7 @@ const ( timeout = time.Second * 60 // how long to wait in "Consistently" blocks - duration = time.Second * 10 + duration = time.Second * 10 // nolint:unused // how frequently to poll for conditions interval = time.Second * 1 @@ -311,14 +314,58 @@ var _ = Describe("controller", Ordered, func() { } Eventually(curlService, timeout, interval).Should(Succeed()) - By("ensuring that an option in WorkspaceKind cannot be removed if it is currently in use") + By("ensuring in-use imageConfig values cannot be removed from WorkspaceKind") EventuallyWithOffset(1, func() error { - // Attempt to remove an option from the WorkspaceKind that is currently in use - cmd := exec.Command("kubectl", "patch", "workspacekind", "jupyterlab", + cmd := exec.Command("kubectl", "patch", "workspacekind", workspaceKindName, "--type=json", "-p", `[{"op": "remove", "path": "/spec/podTemplate/options/imageConfig/values/1"}]`) _, err := utils.Run(cmd) return err }, timeout, interval).ShouldNot(Succeed()) + + By("ensuring unused imageConfig values can be removed from WorkspaceKind") + EventuallyWithOffset(1, func() error { + cmd := exec.Command("kubectl", "patch", "workspacekind", workspaceKindName, + "--type=json", "-p", `[{"op": "remove", "path": "/spec/podTemplate/options/imageConfig/values/0"}]`) + _, err := utils.Run(cmd) + return err + }, timeout, interval).Should(Succeed()) + + By("ensuring in-use podConfig values cannot be removed from WorkspaceKind") + EventuallyWithOffset(1, func() error { + cmd := exec.Command("kubectl", "patch", "workspacekind", workspaceKindName, + "--type=json", "-p", `[{"op": "remove", "path": "/spec/podTemplate/options/podConfig/values/0"}]`) + _, err := utils.Run(cmd) + return err + }, timeout, interval).ShouldNot(Succeed()) + + By("ensuring unused podConfig values can be removed from WorkspaceKind") + EventuallyWithOffset(1, func() error { + cmd := exec.Command("kubectl", "patch", "workspacekind", workspaceKindName, + "--type=json", "-p", `[{"op": "remove", "path": "/spec/podTemplate/options/podConfig/values/1"}]`) + _, err := utils.Run(cmd) + return err + }, timeout, interval).Should(Succeed()) + + By("failing to delete an in-use WorkspaceKind") + EventuallyWithOffset(1, func() error { + cmd = exec.Command("kubectl", "delete", "workspacekind", workspaceKindName) + _, err := utils.Run(cmd) + return err + }, timeout, interval).ShouldNot(Succeed()) + + By("deleting the Workspace") + EventuallyWithOffset(1, func() error { + cmd := exec.Command("kubectl", "delete", "workspace", workspaceName, "-n", workspaceNamespace) + _, err := utils.Run(cmd) + return err + }, timeout, interval).Should(Succeed()) + + By("deleting an unused WorkspaceKind") + EventuallyWithOffset(1, func() error { + cmd = exec.Command("kubectl", "delete", "workspacekind", workspaceKindName) + _, err := utils.Run(cmd) + return err + }, timeout, interval).Should(Succeed()) }) }) }) From 4487a2c2388f41b257ec3db124e55eb3da040e62 Mon Sep 17 00:00:00 2001 From: Mathew Wicks <5735406+thesuperzapper@users.noreply.github.com> Date: Wed, 28 Aug 2024 16:52:18 -0700 Subject: [PATCH 8/8] mathew refactor 3 Signed-off-by: Mathew Wicks <5735406+thesuperzapper@users.noreply.github.com> --- .../internal/controller/suite_test.go | 5 + .../controller/internal/helper/helper.go | 77 +-- .../controller/internal/webhook/suite_test.go | 95 ++++ .../webhook/workspacekind_webhook_test.go | 458 +++++++++++++----- 4 files changed, 491 insertions(+), 144 deletions(-) diff --git a/workspaces/controller/internal/controller/suite_test.go b/workspaces/controller/internal/controller/suite_test.go index a4444fd6..7d6a6b90 100644 --- a/workspaces/controller/internal/controller/suite_test.go +++ b/workspaces/controller/internal/controller/suite_test.go @@ -262,6 +262,7 @@ func NewExampleWorkspaceKind1(name string) *kubefloworgv1beta1.WorkspaceKind { }, Values: []kubefloworgv1beta1.ImageConfigValue{ { + // WARNING: do not change the ID of this value or remove it, it is used in the tests Id: "jupyterlab_scipy_180", Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ DisplayName: "jupyter-scipy:v1.8.0", @@ -294,6 +295,7 @@ func NewExampleWorkspaceKind1(name string) *kubefloworgv1beta1.WorkspaceKind { }, }, { + // WARNING: do not change the ID of this value or remove it, it is used in the tests Id: "jupyterlab_scipy_190", Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ DisplayName: "jupyter-scipy:v1.9.0", @@ -325,6 +327,7 @@ func NewExampleWorkspaceKind1(name string) *kubefloworgv1beta1.WorkspaceKind { }, Values: []kubefloworgv1beta1.PodConfigValue{ { + // WARNING: do not change the ID of this value or remove it, it is used in the tests Id: "tiny_cpu", Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ DisplayName: "Tiny CPU", @@ -350,6 +353,7 @@ func NewExampleWorkspaceKind1(name string) *kubefloworgv1beta1.WorkspaceKind { }, }, { + // WARNING: do not change the ID of this value or remove it, it is used in the tests Id: "small_cpu", Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ DisplayName: "Small CPU", @@ -375,6 +379,7 @@ func NewExampleWorkspaceKind1(name string) *kubefloworgv1beta1.WorkspaceKind { }, }, { + // WARNING: do not change the ID of this value or remove it, it is used in the tests Id: "big_gpu", Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ DisplayName: "Big GPU", diff --git a/workspaces/controller/internal/helper/helper.go b/workspaces/controller/internal/helper/helper.go index ca68176f..f166a0e8 100644 --- a/workspaces/controller/internal/helper/helper.go +++ b/workspaces/controller/internal/helper/helper.go @@ -104,46 +104,63 @@ func CopyServiceFields(desired *corev1.Service, target *corev1.Service) bool { // NormalizePodConfigSpec normalizes a PodConfigSpec so that it can be compared with reflect.DeepEqual func NormalizePodConfigSpec(spec kubefloworgv1beta1.PodConfigSpec) error { - // Normalize Affinity - if spec.Affinity != nil && reflect.DeepEqual(spec.Affinity, corev1.Affinity{}) { - spec.Affinity = nil - } - // Normalize NodeSelector - if spec.NodeSelector != nil && len(spec.NodeSelector) == 0 { - spec.NodeSelector = nil + // normalize Affinity + if spec.Affinity != nil { + + // set Affinity to nil if it is empty + if reflect.DeepEqual(spec.Affinity, corev1.Affinity{}) { + spec.Affinity = nil + } } - // Normalize Tolerations - if spec.Tolerations != nil && len(spec.Tolerations) == 0 { - spec.Tolerations = nil + // normalize NodeSelector + if spec.NodeSelector != nil { + + // set NodeSelector to nil if it is empty + if len(spec.NodeSelector) == 0 { + spec.NodeSelector = nil + } } - // Normalize Resources.Requests - if reflect.DeepEqual(spec.Resources.Requests, corev1.ResourceList{}) { - spec.Resources.Requests = nil + // normalize Tolerations + if spec.Tolerations != nil { + + // set Tolerations to nil if it is empty + if len(spec.Tolerations) == 0 { + spec.Tolerations = nil + } } - if spec.Resources.Requests != nil { - for key, value := range spec.Resources.Requests { - q, err := resource.ParseQuantity(value.String()) - if err != nil { - return err + + // normalize Resources + if spec.Resources != nil { + + // if Resources.Requests is empty, set it to nil + if len(spec.Resources.Requests) == 0 { + spec.Resources.Requests = nil + } else { + // otherwise, normalize the values in Resources.Requests + for key, value := range spec.Resources.Requests { + q, err := resource.ParseQuantity(value.String()) + if err != nil { + return err + } + spec.Resources.Requests[key] = q } - spec.Resources.Requests[key] = q } - } - // Normalize Resources.Limits - if reflect.DeepEqual(spec.Resources.Limits, corev1.ResourceList{}) { - spec.Resources.Limits = nil - } - if spec.Resources.Limits != nil { - for key, value := range spec.Resources.Limits { - q, err := resource.ParseQuantity(value.String()) - if err != nil { - return err + // if Resources.Limits is empty, set it to nil + if len(spec.Resources.Limits) == 0 { + spec.Resources.Limits = nil + } else { + // otherwise, normalize the values in Resources.Limits + for key, value := range spec.Resources.Limits { + q, err := resource.ParseQuantity(value.String()) + if err != nil { + return err + } + spec.Resources.Limits[key] = q } - spec.Resources.Limits[key] = q } } diff --git a/workspaces/controller/internal/webhook/suite_test.go b/workspaces/controller/internal/webhook/suite_test.go index 579123c7..a4e9e077 100644 --- a/workspaces/controller/internal/webhook/suite_test.go +++ b/workspaces/controller/internal/webhook/suite_test.go @@ -256,6 +256,7 @@ func NewExampleWorkspaceKind(name string) *kubefloworgv1beta1.WorkspaceKind { }, Values: []kubefloworgv1beta1.ImageConfigValue{ { + // WARNING: do not change the ID of this value or remove it, it is used in the tests Id: "jupyterlab_scipy_180", Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ DisplayName: "jupyter-scipy:v1.8.0", @@ -288,6 +289,7 @@ func NewExampleWorkspaceKind(name string) *kubefloworgv1beta1.WorkspaceKind { }, }, { + // WARNING: do not change the ID of this value or remove it, it is used in the tests Id: "jupyterlab_scipy_190", Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ DisplayName: "jupyter-scipy:v1.9.0", @@ -311,6 +313,66 @@ func NewExampleWorkspaceKind(name string) *kubefloworgv1beta1.WorkspaceKind { }, }, }, + { + // WARNING: do not change the ID of this value or remove it, it is used in the tests + Id: "redirect_step_1", + Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ + DisplayName: "redirect_step_1", + }, + Redirect: &kubefloworgv1beta1.OptionRedirect{ + To: "redirect_step_2", + }, + Spec: kubefloworgv1beta1.ImageConfigSpec{ + Image: "redirect-test:step-1", + Ports: []kubefloworgv1beta1.ImagePort{ + { + Id: "my_port", + DisplayName: "something", + Port: 1234, + Protocol: "HTTP", + }, + }, + }, + }, + { + // WARNING: do not change the ID of this value or remove it, it is used in the tests + Id: "redirect_step_2", + Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ + DisplayName: "redirect_step_2", + }, + Redirect: &kubefloworgv1beta1.OptionRedirect{ + To: "redirect_step_3", + }, + Spec: kubefloworgv1beta1.ImageConfigSpec{ + Image: "redirect-test:step-2", + Ports: []kubefloworgv1beta1.ImagePort{ + { + Id: "my_port", + DisplayName: "something", + Port: 1234, + Protocol: "HTTP", + }, + }, + }, + }, + { + // WARNING: do not change the ID of this value or remove it, it is used in the tests + Id: "redirect_step_3", + Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ + DisplayName: "redirect_step_3", + }, + Spec: kubefloworgv1beta1.ImageConfigSpec{ + Image: "redirect-test:step-3", + Ports: []kubefloworgv1beta1.ImagePort{ + { + Id: "my_port", + DisplayName: "something", + Port: 1234, + Protocol: "HTTP", + }, + }, + }, + }, }, }, PodConfig: kubefloworgv1beta1.PodConfig{ @@ -319,6 +381,7 @@ func NewExampleWorkspaceKind(name string) *kubefloworgv1beta1.WorkspaceKind { }, Values: []kubefloworgv1beta1.PodConfigValue{ { + // WARNING: do not change the ID of this value or remove it, it is used in the tests Id: "tiny_cpu", Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ DisplayName: "Tiny CPU", @@ -344,6 +407,7 @@ func NewExampleWorkspaceKind(name string) *kubefloworgv1beta1.WorkspaceKind { }, }, { + // WARNING: do not change the ID of this value or remove it, it is used in the tests Id: "small_cpu", Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ DisplayName: "Small CPU", @@ -369,6 +433,7 @@ func NewExampleWorkspaceKind(name string) *kubefloworgv1beta1.WorkspaceKind { }, }, { + // WARNING: do not change the ID of this value or remove it, it is used in the tests Id: "big_gpu", Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ DisplayName: "Big GPU", @@ -409,6 +474,36 @@ func NewExampleWorkspaceKind(name string) *kubefloworgv1beta1.WorkspaceKind { }, }, }, + { + // WARNING: do not change the ID of this value or remove it, it is used in the tests + Id: "redirect_step_1", + Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ + DisplayName: "redirect_step_1", + }, + Redirect: &kubefloworgv1beta1.OptionRedirect{ + To: "redirect_step_2", + }, + Spec: kubefloworgv1beta1.PodConfigSpec{}, + }, + { + // WARNING: do not change the ID of this value or remove it, it is used in the tests + Id: "redirect_step_2", + Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ + DisplayName: "redirect_step_2", + }, + Redirect: &kubefloworgv1beta1.OptionRedirect{ + To: "redirect_step_3", + }, + Spec: kubefloworgv1beta1.PodConfigSpec{}, + }, + { + // WARNING: do not change the ID of this value or remove it, it is used in the tests + Id: "redirect_step_3", + Spawner: kubefloworgv1beta1.OptionSpawnerInfo{ + DisplayName: "redirect_step_3", + }, + Spec: kubefloworgv1beta1.PodConfigSpec{}, + }, }, }, }, diff --git a/workspaces/controller/internal/webhook/workspacekind_webhook_test.go b/workspaces/controller/internal/webhook/workspacekind_webhook_test.go index 1a13a94f..a412bf89 100644 --- a/workspaces/controller/internal/webhook/workspacekind_webhook_test.go +++ b/workspaces/controller/internal/webhook/workspacekind_webhook_test.go @@ -17,14 +17,14 @@ limitations under the License. package webhook import ( - "fmt" + "slices" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + gomegaTypes "github.com/onsi/gomega/types" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1" @@ -39,53 +39,58 @@ var _ = Describe("WorkspaceKind Webhook", func() { Context("When creating a WorkspaceKind", Ordered, func() { testCases := []struct { - description string + // the "Should()" description of the test + description string + + // the WorkspaceKind to attempt to create workspaceKind *kubefloworgv1beta1.WorkspaceKind + + // if the test should succeed shouldSucceed bool }{ { description: "should accept creation of a valid WorkspaceKind", - workspaceKind: NewExampleWorkspaceKind("wsk-webhook-create-test"), + workspaceKind: NewExampleWorkspaceKind("wsk-webhook-create--valid"), shouldSucceed: true, }, { description: "should reject creation with cycle in imageConfig redirects", - workspaceKind: NewExampleWorkspaceKindWithImageConfigCycle("wsk-webhook-image-config-cycle-test"), + workspaceKind: NewExampleWorkspaceKindWithImageConfigCycle("wsk-webhook-create--image-config-cycle"), shouldSucceed: false, }, { description: "should reject creation with cycle in podConfig redirects", - workspaceKind: NewExampleWorkspaceKindWithPodConfigCycle("wsk-webhook-pod-config-cycle-test"), + workspaceKind: NewExampleWorkspaceKindWithPodConfigCycle("wsk-webhook-create--pod-config-cycle"), shouldSucceed: false, }, { description: "should reject creation with invalid redirect target in imageConfig options", - workspaceKind: NewExampleWorkspaceKindWithInvalidImageConfigRedirect("wsk-webhook-image-config-invalid-redirect-test"), + workspaceKind: NewExampleWorkspaceKindWithInvalidImageConfigRedirect("wsk-webhook-create--image-config-invalid-redirect"), shouldSucceed: false, }, { description: "should reject creation with invalid redirect target in podConfig options", - workspaceKind: NewExampleWorkspaceKindWithInvalidPodConfigRedirect("wsk-webhook-pod-config-invalid-redirect-test"), + workspaceKind: NewExampleWorkspaceKindWithInvalidPodConfigRedirect("wsk-webhook-create--pod-config-invalid-redirect"), shouldSucceed: false, }, { description: "should reject creation with invalid default imageConfig", - workspaceKind: NewExampleWorkspaceKindWithInvalidDefaultImageConfig("wsk-webhook-image-config-default-test"), + workspaceKind: NewExampleWorkspaceKindWithInvalidDefaultImageConfig("wsk-webhook-create--image-config-invalid-default"), shouldSucceed: false, }, { description: "should reject creation with invalid default podConfig", - workspaceKind: NewExampleWorkspaceKindWithInvalidDefaultPodConfig("wsk-webhook-pod-config-default-test"), + workspaceKind: NewExampleWorkspaceKindWithInvalidDefaultPodConfig("wsk-webhook-create--pod-config-invalid-default"), shouldSucceed: false, }, { description: "should reject creation with duplicate ports in imageConfig", - workspaceKind: NewExampleWorkspaceKindWithDuplicatePorts("wsk-webhook-image-config-duplicate-ports-test"), + workspaceKind: NewExampleWorkspaceKindWithDuplicatePorts("wsk-webhook-create--image-config-duplicate-ports"), shouldSucceed: false, }, { description: "should reject creation if extraEnv[].value is not a valid Go template", - workspaceKind: NewExampleWorkspaceKindWithInvalidExtraEnvValue("wsk-webhook-invalid-extra-env-value-test"), + workspaceKind: NewExampleWorkspaceKindWithInvalidExtraEnvValue("wsk-webhook-create--extra-invalid-env-value"), shouldSucceed: false, }, } @@ -108,66 +113,132 @@ var _ = Describe("WorkspaceKind Webhook", func() { }) Context("When updating a WorkspaceKind", Ordered, func() { - var ( - workspaceKindName string - workspaceKindKey types.NamespacedName - workspaceKind *kubefloworgv1beta1.WorkspaceKind + const ( + workspaceName = "wsk-webhook-update-test" + workspaceKindName = "wsk-webhook-update-test" ) - BeforeAll(func() { - uniqueName := "wsk-webhook-update-test" - workspaceKindName = fmt.Sprintf("workspacekind-%s", uniqueName) - workspaceKindKey = types.NamespacedName{Name: workspaceKindName} - - By("creating the WorkspaceKind") - createdWorkspaceKind := NewExampleWorkspaceKind(workspaceKindName) - Expect(k8sClient.Create(ctx, createdWorkspaceKind)).To(Succeed()) - - By("getting the created WorkspaceKind") - workspaceKind = &kubefloworgv1beta1.WorkspaceKind{} - Expect(k8sClient.Get(ctx, workspaceKindKey, workspaceKind)).To(Succeed()) - }) + AfterEach(func() { + By("deleting the Workspace, if it exists") + workspace := &kubefloworgv1beta1.Workspace{ + ObjectMeta: metav1.ObjectMeta{ + Name: workspaceName, + Namespace: namespaceName, + }, + } + _ = k8sClient.Delete(ctx, workspace) - AfterAll(func() { - By("deleting the WorkspaceKind") - Expect(k8sClient.Delete(ctx, workspaceKind)).To(Succeed()) + By("deleting the WorkspaceKind, if it exists") + workspaceKind := &kubefloworgv1beta1.WorkspaceKind{ + ObjectMeta: metav1.ObjectMeta{ + Name: workspaceKindName, + }, + } + _ = k8sClient.Delete(ctx, workspaceKind) }) testCases := []struct { + // the "Should()" description of the test description string - // modifyKindFn is a function that modifies the WorkspaceKind in some way - // and returns a string matcher for the expected error message (if any) - modifyKindFn func(*kubefloworgv1beta1.WorkspaceKind) string - workspaceName string - // if shouldSucceed is true, the test is expected to succeed + // if the test should succeed shouldSucceed bool + + // the initial state of the WorkspaceKind (required) + workspaceKind *kubefloworgv1beta1.WorkspaceKind + + // the initial state of the Workspace, if any + workspace *kubefloworgv1beta1.Workspace + + // modifyKindFn modifies the WorkspaceKind in some way. + // returns a string matcher for the error message (only used if `shouldSucceed` is false) + modifyKindFn func(*kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher }{ { - description: "should reject updates to in-use imageConfig spec", - modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { - wsk.Spec.PodTemplate.Options.ImageConfig.Values[0].Spec.Image = "new-image:latest" - return fmt.Sprintf("imageConfig value %q is in use and cannot be changed", wsk.Spec.PodTemplate.Options.ImageConfig.Values[0].Id) + description: "should accept re-ordering in-use imageConfig values", + shouldSucceed: true, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + workspace: NewExampleWorkspace(workspaceName, namespaceName, workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { + // reverse the imageConfig values list + slices.Reverse(wsk.Spec.PodTemplate.Options.ImageConfig.Values) + return ContainSubstring("") }, - workspaceName: "wsk-webhook-update-image-config-test", + }, + { + description: "should accept re-ordering in-use podConfig values", + shouldSucceed: true, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + workspace: NewExampleWorkspace(workspaceName, namespaceName, workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { + // reverse the podConfig values list + slices.Reverse(wsk.Spec.PodTemplate.Options.PodConfig.Values) + return ContainSubstring("") + }, + }, + { + description: "should reject updates to in-use imageConfig spec", shouldSucceed: false, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + workspace: NewExampleWorkspace(workspaceName, namespaceName, workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { + inUseId := wsk.Spec.PodTemplate.Options.ImageConfig.Values[0].Id + wsk.Spec.PodTemplate.Options.ImageConfig.Values[0].Spec.Image = "new-image:latest" + return ContainSubstring("imageConfig value %q is in use and cannot be changed", inUseId) + }, }, { - description: "should reject updates to in-use podConfig spec", - modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + description: "should reject updates to in-use podConfig spec", + shouldSucceed: false, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + workspace: NewExampleWorkspace(workspaceName, namespaceName, workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { + inUseId := wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Id wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Spec.Resources = &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1.5"), }, } - return fmt.Sprintf("podConfig value %q is in use and cannot be changed", wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Id) + return ContainSubstring("podConfig value %q is in use and cannot be changed", inUseId) + }, + }, + { + description: "should accept updates to unused imageConfig spec", + shouldSucceed: true, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + workspace: NewExampleWorkspace(workspaceName, namespaceName, workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { + wsk.Spec.PodTemplate.Options.ImageConfig.Values[1].Spec.Image = "new-image:latest" + return ContainSubstring("") + }, + }, + { + description: "should accept updates to unused podConfig spec", + shouldSucceed: true, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + workspace: NewExampleWorkspace(workspaceName, namespaceName, workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { + wsk.Spec.PodTemplate.Options.PodConfig.Values[1].Spec.Resources = &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1.5"), + }, + } + return ContainSubstring("") }, - workspaceName: "ws-webhook-update-pod-config-test", - shouldSucceed: false, }, { - description: "should reject removing in-use imageConfig values", - modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + description: "should reject removing in-use imageConfig values", + shouldSucceed: false, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + workspace: NewExampleWorkspace(workspaceName, namespaceName, workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { toBeRemoved := "jupyterlab_scipy_180" newValues := make([]kubefloworgv1beta1.ImageConfigValue, 0) for _, value := range wsk.Spec.PodTemplate.Options.ImageConfig.Values { @@ -176,14 +247,87 @@ var _ = Describe("WorkspaceKind Webhook", func() { } } wsk.Spec.PodTemplate.Options.ImageConfig.Values = newValues - return fmt.Sprintf("imageConfig value %q is in use and cannot be removed", toBeRemoved) + return ContainSubstring("imageConfig value %q is in use and cannot be removed", toBeRemoved) }, - workspaceName: "ws-webhook-update-image-config-test", + }, + { + description: "should reject removing in-use podConfig values", shouldSucceed: false, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + workspace: NewExampleWorkspace(workspaceName, namespaceName, workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { + toBeRemoved := "tiny_cpu" + newValues := make([]kubefloworgv1beta1.PodConfigValue, 0) + for _, value := range wsk.Spec.PodTemplate.Options.PodConfig.Values { + if value.Id != toBeRemoved { + newValues = append(newValues, value) + } + } + wsk.Spec.PodTemplate.Options.PodConfig.Values = newValues + return ContainSubstring("podConfig value %q is in use and cannot be removed", toBeRemoved) + }, + }, + { + description: "should accept removing an unused imageConfig value", + shouldSucceed: true, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + workspace: NewExampleWorkspace(workspaceName, namespaceName, workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { + toBeRemoved := "redirect_step_1" + newValues := make([]kubefloworgv1beta1.ImageConfigValue, 0) + for _, value := range wsk.Spec.PodTemplate.Options.ImageConfig.Values { + if value.Id != toBeRemoved { + newValues = append(newValues, value) + } + } + wsk.Spec.PodTemplate.Options.ImageConfig.Values = newValues + return ContainSubstring("") + }, }, { - description: "should reject removing in-use podConfig values", - modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + description: "should accept removing an unused podConfig value", + shouldSucceed: true, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + workspace: NewExampleWorkspace(workspaceName, namespaceName, workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { + toBeRemoved := "redirect_step_1" + newValues := make([]kubefloworgv1beta1.PodConfigValue, 0) + for _, value := range wsk.Spec.PodTemplate.Options.PodConfig.Values { + if value.Id != toBeRemoved { + newValues = append(newValues, value) + } + } + wsk.Spec.PodTemplate.Options.PodConfig.Values = newValues + return ContainSubstring("") + }, + }, + { + description: "should reject removing the default imageConfig value", + shouldSucceed: false, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { + toBeRemoved := "jupyterlab_scipy_190" + newValues := make([]kubefloworgv1beta1.ImageConfigValue, 0) + for _, value := range wsk.Spec.PodTemplate.Options.ImageConfig.Values { + if value.Id != toBeRemoved { + newValues = append(newValues, value) + } + } + wsk.Spec.PodTemplate.Options.ImageConfig.Values = newValues + return ContainSubstring("default imageConfig %q not found", toBeRemoved) + }, + }, + { + description: "should reject removing the default podConfig value", + shouldSucceed: false, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + workspace: NewExampleWorkspace(workspaceName, namespaceName, workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { toBeRemoved := "tiny_cpu" newValues := make([]kubefloworgv1beta1.PodConfigValue, 0) for _, value := range wsk.Spec.PodTemplate.Options.PodConfig.Values { @@ -192,70 +336,151 @@ var _ = Describe("WorkspaceKind Webhook", func() { } } wsk.Spec.PodTemplate.Options.PodConfig.Values = newValues - return fmt.Sprintf("podConfig value %q is in use and cannot be removed", toBeRemoved) + return ContainSubstring("default podConfig %q not found", toBeRemoved) }, - workspaceName: "ws-webhook-update-pod-config-test", + }, + { + description: "should reject removing the target of an imageConfig redirect", shouldSucceed: false, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { + toBeRemoved := "redirect_step_2" + newValues := make([]kubefloworgv1beta1.ImageConfigValue, 0) + for _, value := range wsk.Spec.PodTemplate.Options.ImageConfig.Values { + if value.Id != toBeRemoved { + newValues = append(newValues, value) + } + } + wsk.Spec.PodTemplate.Options.ImageConfig.Values = newValues + return ContainSubstring("target imageConfig %q does not exist", toBeRemoved) + }, }, { - description: "should reject updating an imageConfig value to create a self-cycle", - modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + description: "should reject removing the target of a podConfig redirect", + shouldSucceed: false, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { + toBeRemoved := "redirect_step_2" + newValues := make([]kubefloworgv1beta1.PodConfigValue, 0) + for _, value := range wsk.Spec.PodTemplate.Options.PodConfig.Values { + if value.Id != toBeRemoved { + newValues = append(newValues, value) + } + } + wsk.Spec.PodTemplate.Options.PodConfig.Values = newValues + return ContainSubstring("target podConfig %q does not exist", toBeRemoved) + }, + }, + { + description: "should accept removing an entire imageConfig redirect chain", + shouldSucceed: true, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { + toBeRemoved := map[string]bool{"redirect_step_1": true, "redirect_step_2": true, "redirect_step_3": true} + newValues := make([]kubefloworgv1beta1.ImageConfigValue, 0) + for _, value := range wsk.Spec.PodTemplate.Options.ImageConfig.Values { + if !toBeRemoved[value.Id] { + newValues = append(newValues, value) + } + } + wsk.Spec.PodTemplate.Options.ImageConfig.Values = newValues + return ContainSubstring("") + }, + }, + { + description: "should accept removing an entire podConfig redirect chain", + shouldSucceed: true, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { + toBeRemoved := map[string]bool{"redirect_step_1": true, "redirect_step_2": true, "redirect_step_3": true} + newValues := make([]kubefloworgv1beta1.PodConfigValue, 0) + for _, value := range wsk.Spec.PodTemplate.Options.PodConfig.Values { + if !toBeRemoved[value.Id] { + newValues = append(newValues, value) + } + } + wsk.Spec.PodTemplate.Options.PodConfig.Values = newValues + return ContainSubstring("") + }, + }, + { + description: "should reject updating an imageConfig value to create a self-cycle", + shouldSucceed: false, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { valueId := wsk.Spec.PodTemplate.Options.ImageConfig.Values[1].Id wsk.Spec.PodTemplate.Options.ImageConfig.Values[1].Redirect = &kubefloworgv1beta1.OptionRedirect{To: valueId} - return fmt.Sprintf("imageConfig redirect cycle detected: [%s]", valueId) + return ContainSubstring("imageConfig redirect cycle detected: [%s]", valueId) }, - shouldSucceed: false, }, { - description: "should reject updating a podConfig value to create a 2-step cycle", - modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + description: "should reject updating a podConfig value to create a 2-step cycle", + shouldSucceed: false, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { step1 := wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Id step2 := wsk.Spec.PodTemplate.Options.PodConfig.Values[1].Id wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Redirect = &kubefloworgv1beta1.OptionRedirect{To: step2} wsk.Spec.PodTemplate.Options.PodConfig.Values[1].Redirect = &kubefloworgv1beta1.OptionRedirect{To: step1} - return "podConfig redirect cycle detected: [" // there is no guarantee on which element will be first + return ContainSubstring("podConfig redirect cycle detected: [") // there is no guarantee on which element will be first }, - shouldSucceed: false, }, { - description: "should reject updating an imageConfig to redirect to a non-existent value", - modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + description: "should reject updating an imageConfig to redirect to a non-existent value", + shouldSucceed: false, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { invalidTarget := "invalid_image_config" wsk.Spec.PodTemplate.Options.ImageConfig.Values[1].Redirect = &kubefloworgv1beta1.OptionRedirect{To: invalidTarget} - return fmt.Sprintf("target imageConfig %q does not exist", invalidTarget) + return ContainSubstring("target imageConfig %q does not exist", invalidTarget) }, - shouldSucceed: false, }, { - description: "should reject updating a podConfig to redirect to a non-existent value", - modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + description: "should reject updating a podConfig to redirect to a non-existent value", + shouldSucceed: false, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { invalidTarget := "invalid_pod_config" wsk.Spec.PodTemplate.Options.PodConfig.Values[0].Redirect = &kubefloworgv1beta1.OptionRedirect{To: invalidTarget} - return fmt.Sprintf("target podConfig %q does not exist", invalidTarget) + return ContainSubstring("target podConfig %q does not exist", invalidTarget) }, - shouldSucceed: false, }, { - description: "should reject updating the default imageConfig value to a non-existent value", - modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + description: "should reject updating the default imageConfig value to a non-existent value", + shouldSucceed: false, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { invalidDefault := "invalid_image_config" wsk.Spec.PodTemplate.Options.ImageConfig.Spawner.Default = invalidDefault - return fmt.Sprintf("default imageConfig %q not found", invalidDefault) + return ContainSubstring("default imageConfig %q not found", invalidDefault) }, - shouldSucceed: false, }, { - description: "should reject updating the default podConfig value to a non-existent value", - modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + description: "should reject updating the default podConfig value to a non-existent value", + shouldSucceed: false, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { invalidDefault := "invalid_pod_config" wsk.Spec.PodTemplate.Options.PodConfig.Spawner.Default = invalidDefault - return fmt.Sprintf("default podConfig %q not found", invalidDefault) + return ContainSubstring("default podConfig %q not found", invalidDefault) }, - shouldSucceed: false, }, { - description: "should reject updating an imageConfig to have duplicate ports", - modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + description: "should reject updating an imageConfig to have duplicate ports", + shouldSucceed: false, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { duplicatePortNumber := int32(8888) wsk.Spec.PodTemplate.Options.ImageConfig.Values[1].Spec.Ports = []kubefloworgv1beta1.ImagePort{ { @@ -271,62 +496,67 @@ var _ = Describe("WorkspaceKind Webhook", func() { Protocol: "HTTP", }, } - return fmt.Sprintf("port %d is defined more than once", duplicatePortNumber) + return ContainSubstring("port %d is defined more than once", duplicatePortNumber) }, - shouldSucceed: false, }, { - description: "should reject updating an extraEnv[].value to an invalid Go template", - modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + description: "should reject updating an extraEnv[].value to an invalid Go template", + shouldSucceed: false, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { invalidValue := `{{ httpPathPrefix "jupyterlab" }` wsk.Spec.PodTemplate.ExtraEnv[0].Value = invalidValue - return fmt.Sprintf("failed to parse template %q", invalidValue) + return ContainSubstring("failed to parse template %q", invalidValue) }, - shouldSucceed: false, }, { - description: "should accept updating an extraEnv[].value to a valid Go template", - modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) string { + description: "should accept updating an extraEnv[].value to a valid Go template", + shouldSucceed: true, + + workspaceKind: NewExampleWorkspaceKind(workspaceKindName), + modifyKindFn: func(wsk *kubefloworgv1beta1.WorkspaceKind) gomegaTypes.GomegaMatcher { wsk.Spec.PodTemplate.ExtraEnv[0].Value = `{{ httpPathPrefix "jupyterlab" }}` - return "" + return ContainSubstring("") }, - shouldSucceed: true, }, } for _, tc := range testCases { tc := tc // Create a new instance of tc to avoid capturing the loop variable. It(tc.description, func() { - if tc.workspaceName != "" { - By("creating a Workspace that uses the WorkspaceKind") - workspace := NewExampleWorkspace(tc.workspaceName, namespaceName, workspaceKindName) - Expect(k8sClient.Create(ctx, workspace)).To(Succeed()) + if tc.workspaceKind == nil { + Fail("invalid test case definition: workspaceKind is required") } - patch := client.MergeFrom(workspaceKind.DeepCopy()) - modifiedWorkspaceKind := workspaceKind.DeepCopy() - expectedErrorMessage := tc.modifyKindFn(modifiedWorkspaceKind) + By("creating the WorkspaceKind") + // NOTE: cleanup is handled in the AfterEach() + Expect(k8sClient.Create(ctx, tc.workspaceKind)).To(Succeed()) + + By("retrieving the WorkspaceKind") + workspaceKind := &kubefloworgv1beta1.WorkspaceKind{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(tc.workspaceKind), workspaceKind)).To(Succeed()) + + if tc.workspace != nil { + By("creating the Workspace") + // NOTE: cleanup is handled in the AfterEach() + Expect(k8sClient.Create(ctx, tc.workspace)).To(Succeed()) + + By("retrieving the Workspace") + workspace := &kubefloworgv1beta1.Workspace{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(tc.workspace), workspace)).To(Succeed()) + } By("updating the WorkspaceKind") + patch := client.MergeFrom(workspaceKind.DeepCopy()) + modifiedWorkspaceKind := workspaceKind.DeepCopy() + errMatcher := tc.modifyKindFn(modifiedWorkspaceKind) + err := k8sClient.Patch(ctx, modifiedWorkspaceKind, patch) if tc.shouldSucceed { - Expect(k8sClient.Patch(ctx, modifiedWorkspaceKind, patch)).To(Succeed()) + Expect(err).To(Succeed()) } else { - err := k8sClient.Patch(ctx, modifiedWorkspaceKind, patch) Expect(err).NotTo(Succeed()) - if expectedErrorMessage != "" { - Expect(err.Error()).To(ContainSubstring(expectedErrorMessage)) - } - } - - if tc.workspaceName != "" { - By("deleting the Workspace that uses the WorkspaceKind") - workspace := &kubefloworgv1beta1.Workspace{ - ObjectMeta: metav1.ObjectMeta{ - Name: tc.workspaceName, - Namespace: namespaceName, - }, - } - Expect(k8sClient.Delete(ctx, workspace)).To(Succeed()) + Expect(err.Error()).To(errMatcher) } }) }