diff --git a/workspaces/controller/api/v1beta1/workspacekind_types.go b/workspaces/controller/api/v1beta1/workspacekind_types.go index ee06d557..7c3068e2 100644 --- a/workspaces/controller/api/v1beta1/workspacekind_types.go +++ b/workspaces/controller/api/v1beta1/workspacekind_types.go @@ -312,7 +312,6 @@ type ImageConfigValue struct { Redirect *OptionRedirect `json:"redirect,omitempty"` // the spec of the image config - //+kubebuilder:validation:XValidation:rule="self == oldSelf",message="ImageConfig 'spec' is immutable" Spec ImageConfigSpec `json:"spec"` } @@ -396,7 +395,7 @@ type PodConfigValue struct { Redirect *OptionRedirect `json:"redirect,omitempty"` // the spec of the pod config - //+kubebuilder:validation:XValidation:rule="self == oldSelf",message="PodConfig 'spec' is immutable" + Spec PodConfigSpec `json:"spec"` } diff --git a/workspaces/controller/config/crd/bases/kubeflow.org_workspacekinds.yaml b/workspaces/controller/config/crd/bases/kubeflow.org_workspacekinds.yaml index 41b71234..375548f6 100644 --- a/workspaces/controller/config/crd/bases/kubeflow.org_workspacekinds.yaml +++ b/workspaces/controller/config/crd/bases/kubeflow.org_workspacekinds.yaml @@ -2368,9 +2368,6 @@ spec: - image - ports type: object - x-kubernetes-validations: - - message: ImageConfig 'spec' is immutable - rule: self == oldSelf required: - id - spawner @@ -2494,7 +2491,6 @@ spec: - displayName type: object spec: - description: the spec of the pod config properties: affinity: description: affinity configs for the pod @@ -3532,9 +3528,6 @@ spec: type: object type: array type: object - x-kubernetes-validations: - - message: PodConfig 'spec' is immutable - rule: self == oldSelf required: - id - spawner diff --git a/workspaces/controller/config/manager/kustomization.yaml b/workspaces/controller/config/manager/kustomization.yaml index 5c5f0b84..0f89648a 100644 --- a/workspaces/controller/config/manager/kustomization.yaml +++ b/workspaces/controller/config/manager/kustomization.yaml @@ -1,2 +1,8 @@ resources: - manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: example.com/workspace-controller + newTag: v0.0.1 diff --git a/workspaces/controller/config/samples/v1beta1_workspace.yaml b/workspaces/controller/config/samples/v1beta1_workspace.yaml index 13386e71..62549999 100644 --- a/workspaces/controller/config/samples/v1beta1_workspace.yaml +++ b/workspaces/controller/config/samples/v1beta1_workspace.yaml @@ -65,7 +65,7 @@ spec: ## - options are defined in WorkspaceKind under ## `spec.podTemplate.options.imageConfig.values[]` ## - imageConfig: "jupyterlab_scipy_180" + imageConfig: "jupyterlab_scipy_190" ## the id of a podConfig option ## - options are defined in WorkspaceKind under diff --git a/workspaces/controller/config/samples/v1beta1_workspacekind.yaml b/workspaces/controller/config/samples/v1beta1_workspacekind.yaml index 44cacc63..4b73578b 100644 --- a/workspaces/controller/config/samples/v1beta1_workspacekind.yaml +++ b/workspaces/controller/config/samples/v1beta1_workspacekind.yaml @@ -111,9 +111,9 @@ spec: ## https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#probe-v1-core ## probes: - startupProbe: {} - livenessProbe: {} - readinessProbe: {} +# startupProbe: {} +# livenessProbe: {} +# readinessProbe: {} ## volume mount paths ## @@ -158,7 +158,7 @@ spec: ## https://github.com/kubeflow/kubeflow/blob/v1.8.0/components/example-notebook-servers/jupyter/s6/services.d/jupyterlab/run#L12 - name: "NB_PREFIX" value: |- - {{ httpPathPrefix "juptyerlab" }} + {{ httpPathPrefix "jupyterlab" }} ## extra volume mounts for Workspace Pods (MUTABLE) ## - spec for VolumeMount: diff --git a/workspaces/controller/config/samples/workspace_data_pvc.yaml b/workspaces/controller/config/samples/workspace_data_pvc.yaml new file mode 100644 index 00000000..43c774d5 --- /dev/null +++ b/workspaces/controller/config/samples/workspace_data_pvc.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: my-data-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi \ No newline at end of file diff --git a/workspaces/controller/config/samples/workspace_home_pvc.yaml b/workspaces/controller/config/samples/workspace_home_pvc.yaml new file mode 100644 index 00000000..075ed8c7 --- /dev/null +++ b/workspaces/controller/config/samples/workspace_home_pvc.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: my-home-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi \ No newline at end of file diff --git a/workspaces/controller/test/e2e/e2e_test.go b/workspaces/controller/test/e2e/e2e_test.go index 55cb7f75..28f84b5c 100644 --- a/workspaces/controller/test/e2e/e2e_test.go +++ b/workspaces/controller/test/e2e/e2e_test.go @@ -19,6 +19,8 @@ package e2e import ( "fmt" "os/exec" + "path/filepath" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -29,28 +31,67 @@ import ( const namespace = "workspace-controller-system" +var ( + projectDir = "" +) var _ = Describe("controller", Ordered, func() { BeforeAll(func() { - By("installing prometheus operator") - Expect(utils.InstallPrometheusOperator()).To(Succeed()) - - By("installing the cert-manager") - Expect(utils.InstallCertManager()).To(Succeed()) + projectDir, _ = utils.GetProjectDir() By("creating manager namespace") cmd := exec.Command("kubectl", "create", "ns", namespace) _, _ = utils.Run(cmd) + + By("creating service account") + cmd = exec.Command("kubectl", "create", "sa", "default-editor") + _, _ = utils.Run(cmd) + + By("creating workspace home pvc") + cmd = exec.Command("kubectl", "apply", "-f", filepath.Join(projectDir, + "config/samples/workspace_home_pvc.yaml")) + _, _ = utils.Run(cmd) + + By("creating workspace data pvc") + cmd = exec.Command("kubectl", "apply", "-f", filepath.Join(projectDir, + "config/samples/workspace_data_pvc.yaml")) + _, _ = utils.Run(cmd) }) AfterAll(func() { - By("uninstalling the Prometheus manager bundle") - utils.UninstallPrometheusOperator() + By("deleting workspace CR") + cmd := exec.Command("kubectl", "delete", "-f", filepath.Join(projectDir, + "config/samples/v1beta1_workspace.yaml")) + _, _ = utils.Run(cmd) + + By("deleting workspaceKind CR") + cmd = exec.Command("kubectl", "delete", "-f", filepath.Join(projectDir, + "config/samples/v1beta1_workspacekind.yaml")) + _, _ = utils.Run(cmd) + + By("deleting manager namespace") + cmd = exec.Command("kubectl", "delete", "ns", namespace) + _, _ = utils.Run(cmd) + + By("deleting service account") + cmd = exec.Command("kubectl", "delete", "sa", "default-editor") + _, _ = utils.Run(cmd) - By("uninstalling the cert-manager bundle") - utils.UninstallCertManager() + By("deleting workspace home pvc") + cmd = exec.Command("kubectl", "delete", "-f", filepath.Join(projectDir, + "config/samples/workspace_home_pvc.yaml")) + _, _ = utils.Run(cmd) + + By("deleting workspace data pvc") + cmd = exec.Command("kubectl", "delete", "-f", filepath.Join(projectDir, + "config/samples/workspace_data_pvc.yaml")) + _, _ = utils.Run(cmd) + + By("deleting the controller-manager") + cmd = exec.Command("make", "undeploy") + _, _ = utils.Run(cmd) - By("removing manager namespace") - cmd := exec.Command("kubectl", "delete", "ns", namespace) + By("deleting CRDs") + cmd = exec.Command("make", "uninstall") _, _ = utils.Run(cmd) }) @@ -117,6 +158,100 @@ var _ = Describe("controller", Ordered, func() { } EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) + By("creating an instance of the WorkspaceKind CR") + EventuallyWithOffset(1, func() error { + cmd = exec.Command("kubectl", "apply", "-f", filepath.Join(projectDir, + "config/samples/v1beta1_workspacekind.yaml")) + _, err = utils.Run(cmd) + return err + }, time.Minute, time.Second).Should(Succeed()) + + By("creating an instance of the Workspace CR") + EventuallyWithOffset(1, func() error { + cmd = exec.Command("kubectl", "apply", "-f", filepath.Join(projectDir, + "config/samples/v1beta1_workspace.yaml")) + _, err = utils.Run(cmd) + return err + }, time.Minute, time.Second).Should(Succeed()) + + By("validating that workspace pod is running as expected") + verifyWorkspacePod := func() error { + // Get workspace pod name + cmd = exec.Command("kubectl", "get", + "pods", "-l", "statefulset=my-workspace", + "-o", "go-template={{ range .items }}"+ + "{{ if not .metadata.deletionTimestamp }}"+ + "{{ .metadata.name }}"+ + "{{ \"\\n\" }}{{ end }}{{ end }}", + ) + + podOutput, err := utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + podNames := utils.GetNonEmptyLines(string(podOutput)) + if len(podNames) != 1 { + return fmt.Errorf("expect 1 workspace pod running, but got %d", len(podNames)) + } + workspacePodName := podNames[0] + ExpectWithOffset(2, workspacePodName).Should(ContainSubstring("ws-my-workspace")) + + // Validate pod status + cmd = exec.Command("kubectl", "get", + "pods", workspacePodName, "-o", "jsonpath={.status.phase}", + ) + status, err := utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + if string(status) != "Running" { + return fmt.Errorf("workspace pod in %s status", status) + } + return nil + } + EventuallyWithOffset(1, verifyWorkspacePod, time.Minute, time.Second).Should(Succeed()) + + By("CURL the workspace pod") + getServiceName := func() (string, error) { + cmd := exec.Command("kubectl", "get", "services", "-l", "notebooks.kubeflow.org/workspace-name=my-workspace", "-o", "jsonpath={.items[0].metadata.name}") + output, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("failed to get service name: %v", err) + } + serviceName := strings.TrimSpace(string(output)) + if serviceName == "" { + return "", fmt.Errorf("no service found with label notebooks.kubeflow.org/workspace-name=my-workspace") + } + return serviceName, nil + } + serviceName, err := getServiceName() + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + // Construct the service endpoint + const servicePort = 8888 + serviceEndpoint := fmt.Sprintf("http://%s:%d/workspace/default/my-workspace/jupyterlab/lab", serviceName, servicePort) + + // Function to run the curl command inside the cluster and return the status code + curlService := func() (int, error) { + cmd := exec.Command("kubectl", "run", "tmp-curl", "--restart=Never", "--rm", "-i", "--image=appropriate/curl", "--", + "curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", serviceEndpoint) + + // Execute the curl command + output, err := cmd.CombinedOutput() + if err != nil { + return 0, fmt.Errorf("failed to execute curl command: %v", err) + } + + // Parse the HTTP status code from the output + var statusCode int + if _, err := fmt.Sscanf(string(output), "%d", &statusCode); err != nil { + return 0, fmt.Errorf("failed to parse status code: %v", err) + } + + return statusCode, nil + } + + // Check that the curl command returns a 200-status code + Eventually(func() (int, error) { + return curlService() + }, 2*time.Minute, 10*time.Second).Should(Equal(200), "Expected status code to be 200") + }) }) })