From 843319fd589548001d6059e3b111ae7068415ccd Mon Sep 17 00:00:00 2001 From: yanghua Date: Tue, 7 Feb 2023 18:16:42 +0800 Subject: [PATCH 01/10] [Addon #603] Add Apache Spark as a experimental addon Signed-off-by: yanghua --- .../spark-kubernetes-operator/README.md | 170 ++++++++++++++++++ .../definitions/spark-application.cue | 90 ++++++++++ .../spark-kubernetes-operator/metadata.yaml | 15 ++ .../spark-kubernetes-operator/parameter.cue | 20 +++ .../spark-kubernetes-operator/template.cue | 101 +++++++++++ 5 files changed, 396 insertions(+) create mode 100644 experimental/addons/spark-kubernetes-operator/README.md create mode 100644 experimental/addons/spark-kubernetes-operator/definitions/spark-application.cue create mode 100644 experimental/addons/spark-kubernetes-operator/metadata.yaml create mode 100644 experimental/addons/spark-kubernetes-operator/parameter.cue create mode 100644 experimental/addons/spark-kubernetes-operator/template.cue diff --git a/experimental/addons/spark-kubernetes-operator/README.md b/experimental/addons/spark-kubernetes-operator/README.md new file mode 100644 index 00000000..bce23450 --- /dev/null +++ b/experimental/addons/spark-kubernetes-operator/README.md @@ -0,0 +1,170 @@ +# spark-kubernetes-operator + +A kubernetes operator for Apache Spark(https://github.com/GoogleCloudPlatform/spark-on-k8s-operator), it allows users to manage Spark applications and their lifecycle through native K8S tooling like `kubectl`. + +> Note: It's not provided by Apache Spark. But widely used by a large number of companies(https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/docs/who-is-using.md). + +# Install + +``` +#The following steps are for enabling fluxcd and spark-kubernetes-operator in namespace called "spark-operator". + +vela addon enable fluxcd +vela addon enable spark-kubernetes-operator +``` + +# Uninstall + +``` +vela addon disable spark-kubernetes-operator +vela addon disable fluxcd +``` + +# To check the spark-kubernetes-operator running status + +* Firstly, check the spark-kubernetes-operator (and the fluxcd and we need to deploy by helm) running status + +``` +vela addon status spark-kubernetes-operator +vela ls -A | grep spark +``` + +* Secondly, show the component type `spark-cluster`, so we know how to use it in one application. As a spark user, you can choose the parameter to set for your spark cluster. + +``` +vela show spark-application +# Specification ++---------------------+------------------------------------------------------------------------------------------------------+-------------------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++---------------------+------------------------------------------------------------------------------------------------------+-------------------+----------+---------+ +| name | Specify the spark application name. | string | true | | +| namespace | Specify the namespace for spark application to install. | string | true | | +| type | Specify the application language type, e.g. "Scala", "Python", "Java" or "R". | string | true | | +| pythonVersion | Specify the python version. | string | false | | +| mode | Specify the deploy mode, e.go "cluster", "client" or "in-cluster-client". | string | true | | +| image | Specify the container image for the driver, executor, and init-container. | string | true | | +| imagePullPolicy | Specify the image pull policy for the driver, executor, and init-container. | string | true | | +| mainClass | Specify the fully-qualified main class of the Spark application. | string | true | | +| mainApplicationFile | Specify the path to a bundled JAR, Python, or R file of the application. | string | true | | +| sparkVersion | Specify the version of Spark the application uses. | string | true | | +| driverCores | Specify the number of CPU cores to request for the driver pod. | int | true | | +| executorCores | Specify the number of CPU cores to request for the executor pod. | int | true | | +| arguments | Specify a list of arguments to be passed to the application. | []string | false | | +| sparkConf | Specify the config information carries user-specified Spark configuration properties as they would | map[string]string | false | | +| | use the "--conf" option in spark-submit. | | | | +| hadoopConf | Specify the config information carries user-specified Hadoop configuration properties as they would | map[string]string | false | | +| | use the the "--conf" option in spark-submit. The SparkApplication controller automatically adds | | | | +| | prefix "spark.hadoop." to Hadoop configuration properties. | | | | +| sparkConfigMap | Specify the name of the ConfigMap containing Spark configuration files such as log4j.properties. The | string | false | | +| | controller will add environment variable SPARK_CONF_DIR to the path where the ConfigMap is mounted | | | | +| | to. | | | | +| hadoopConfigMap | Specify the name of the ConfigMap containing Hadoop configuration files such as core-site.xml. The | string | false | | +| | controller will add environment variable HADOOP_CONF_DIR to the path where the ConfigMap is mounted | | | | +| | to. | | | | ++---------------------+------------------------------------------------------------------------------------------------------+-------------------+----------+---------+ + +``` + +# Example for how to run a component typed spark-cluster in application + +1. Firstly, copy the following example to "spark-app-v1.yaml": + +> The addon will create a namespace named "spark-cluster" + +``` +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: spark-app-v1 + namespace: spark-cluster +spec: + components: + - name: my-spark-application-component + type: spark-application + properties: + name: my-spark-app + namespace: spark-cluster + type: Scala + mode: cluster + image: "gcr.io/spark-operator/spark:v3.1.1" + imagePullPolicy: Always + mainClass: org.apache.spark.examples.streaming.JavaQueueStream + mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.12-3.1.1.jar" + sparkVersion: "3.1.1" + driverCores: 1 + executorCores: 1 +``` + +2. Secondly, start the application: + +``` +vela up -f spark-app-v1.yaml +``` + +You will see the stdout like this: + +``` +Applying an application in vela K8s object format... +I0227 16:54:37.069480 361176 apply.go:121] "creating object" name="spark-app-v1" resource="core.oam.dev/v1beta1, Kind=Application" +✅ App has been deployed 🚀🚀🚀 + Port forward: vela port-forward spark-app-v1 -n spark-cluster + SSH: vela exec spark-app-v1 -n spark-cluster + Logging: vela logs spark-app-v1 -n spark-cluster + App status: vela status spark-app-v1 -n spark-cluster + Endpoint: vela status spark-app-v1 -n spark-cluster --endpoint +Application spark-cluster/spark-app-v1 applied. +``` + +3. Then, you can use the native command to check the status of the Spark applicaiton: + +``` +$ kubectl get sparkapplications -n spark-cluster +NAME STATUS ATTEMPTS START FINISH AGE +my-spark-app RUNNING 1 2023-02-27T08:54:40Z 2m33s +``` + +or get the application detail via this command: + +``` +$ kubectl describe sparkapplication my-spark-app -n spark-cluster +Name: my-spark-app +Namespace: spark-cluster +Labels: app.oam.dev/app-revision-hash=4e5592aea53a5961 + app.oam.dev/appRevision=spark-app-v1-v1 + app.oam.dev/cluster=local + app.oam.dev/component=my-spark-application-component + app.oam.dev/name=spark-app-v1 + app.oam.dev/namespace=spark-cluster + app.oam.dev/resourceType=TRAIT + app.oam.dev/revision= + oam.dev/render-hash=640a3298d803274e + trait.oam.dev/resource=spark + trait.oam.dev/type=AuxiliaryWorkload +Annotations: app.oam.dev/last-applied-configuration: + {"apiVersion":"sparkoperator.k8s.io/v1beta2","kind":"SparkApplication","metadata":{"annotations":{"app.oam.dev/last-applied-time":"2023-02... + app.oam.dev/last-applied-time: 2023-02-27T16:54:37+08:00 + oam.dev/kubevela-version: v1.7.0 +API Version: sparkoperator.k8s.io/v1beta2 +Kind: SparkApplication +Metadata: +...... +``` + +or get the general purpose detail information via this command: + +``` +$ kubectl get app spark-app-v1 -n spark-cluster -oyaml +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: +...... +``` + +4. Show the service of spark application via this command: + +``` +$ kubectl get svc -n spark-cluster +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +my-spark-app-c58a1c869214bfe5-driver-svc ClusterIP None 7078/TCP,7079/TCP,4040/TCP 19m +my-spark-app-ui-svc ClusterIP xx.xx.xx.xx 4040/TCP 19m +``` diff --git a/experimental/addons/spark-kubernetes-operator/definitions/spark-application.cue b/experimental/addons/spark-kubernetes-operator/definitions/spark-application.cue new file mode 100644 index 00000000..44b786b4 --- /dev/null +++ b/experimental/addons/spark-kubernetes-operator/definitions/spark-application.cue @@ -0,0 +1,90 @@ +"spark-application": { + annotations: {} + attributes: workload: type: "autodetects.core.oam.dev" + description: "Describes a containerized spark application that can specify resource spec." + labels: {} + type: "component" +} + +template: { + parameter: { + // +usage=Specify the spark application name + name: string + // +usage=Specify the namespace for spark application to install + namespace: string + // +usage=Specify the application language type, e.g. "Scala", "Python", "Java" or "R" + type: string + // +usage=Specify the python version + pythonVersion ?: string + // +usage=Specify the deploy mode, e.go "cluster", "client" or "in-cluster-client" + mode: string + // +usage=Specify the container image for the driver, executor, and init-container + image: string + // +usage=Specify the image pull policy for the driver, executor, and init-container + imagePullPolicy: string + // +usage=Specify the fully-qualified main class of the Spark application + mainClass: string + // +usage=Specify the path to a bundled JAR, Python, or R file of the application + mainApplicationFile: string + // +usage=Specify the version of Spark the application uses + sparkVersion: string + // +usage=Specify the number of CPU cores to request for the driver pod + driverCores: int + // +usage=Specify the number of CPU cores to request for the executor pod + executorCores: int + // +usage=Specify a list of arguments to be passed to the application + arguments ?: [...string] + // +usage=Specify the config information carries user-specified Spark configuration properties as they would use the "--conf" option in spark-submit + sparkConf ?: [string]: string + // +usage=Specify the config information carries user-specified Hadoop configuration properties as they would use the the "--conf" option in spark-submit. The SparkApplication controller automatically adds prefix "spark.hadoop." to Hadoop configuration properties + hadoopConf ?: [string]: string + // +usage=Specify the name of the ConfigMap containing Spark configuration files such as log4j.properties. The controller will add environment variable SPARK_CONF_DIR to the path where the ConfigMap is mounted to + sparkConfigMap ?: string + // +usage=Specify the name of the ConfigMap containing Hadoop configuration files such as core-site.xml. The controller will add environment variable HADOOP_CONF_DIR to the path where the ConfigMap is mounted to + hadoopConfigMap ?: string + + } + + output: { + kind: "ClusterRoleBinding" + apiVersion: "rbac.authorization.k8s.io/v1" + metadata: name: parameter.name + roleRef: { + name: "edit" + apiGroup: "rbac.authorization.k8s.io" + kind: "ClusterRole" + } + subjects: [{ + name: "default" + kind: "ServiceAccount" + namespace: parameter.namespace + }] + } + + outputs: { + + "spark": { + kind: "SparkApplication" + apiVersion: "sparkoperator.k8s.io/v1beta2" + metadata: { + name: parameter.name + namespace: parameter.namespace + } + spec: { + type: parameter.type + mode: parameter.mode + image: parameter.image + imagePullPolicy: parameter.imagePullPolicy + mainClass: parameter.mainClass + mainApplicationFile: parameter.mainApplicationFile + sparkVersion: parameter.sparkVersion + driver: { + cores: parameter.driverCores + } + executor: { + cores: parameter.executorCores + } + } + } + } +} diff --git a/experimental/addons/spark-kubernetes-operator/metadata.yaml b/experimental/addons/spark-kubernetes-operator/metadata.yaml new file mode 100644 index 00000000..65d95dd6 --- /dev/null +++ b/experimental/addons/spark-kubernetes-operator/metadata.yaml @@ -0,0 +1,15 @@ +description: A kubernetes operator for Apache Spark +icon: "https://spark.apache.org/images/spark-logo.png" +invisible: false +name: spark-kubernetes-operator +tags: + - GoogleCloudPlatform/spark-on-k8s-operator +version: v1beta2-1.3.8-3.1.1 +url: https://github.com/GoogleCloudPlatform/spark-on-k8s-operator + +dependencies: + - name: fluxcd + +system: + vela: ">=1.5.0-beta.3" + kubernetes: ">=1.16" diff --git a/experimental/addons/spark-kubernetes-operator/parameter.cue b/experimental/addons/spark-kubernetes-operator/parameter.cue new file mode 100644 index 00000000..17de99c8 --- /dev/null +++ b/experimental/addons/spark-kubernetes-operator/parameter.cue @@ -0,0 +1,20 @@ +// parameter.cue is used to store addon parameters. +// +// You can use these parameters in template.cue or in resources/ by 'parameter.myparam' +// +// For example, you can use parameters to allow the user to customize +// container images, ports, and etc. +parameter: { + // +usage=Deploy to specified clusters. Leave empty to deploy to all clusters. + clusters?: [...string] + // +usage=Namespace to deploy to + namespace: *"spark-operator" | string + // +usage=Specify if create the webhook or not + "createWebhook": *false | bool + // +usage=Specify the image repository + "imageRepository": *"ghcr.io/googlecloudplatform/spark-operator" | string + // +usage=Specify the image tag + "imageTag": *"v1beta2-1.3.8-3.1.1" | string + // +usage=Specify if create the sa for job or not + "createSparkServiceAccount": *false|bool +} diff --git a/experimental/addons/spark-kubernetes-operator/template.cue b/experimental/addons/spark-kubernetes-operator/template.cue new file mode 100644 index 00000000..55ac84bd --- /dev/null +++ b/experimental/addons/spark-kubernetes-operator/template.cue @@ -0,0 +1,101 @@ +package main +output: { + apiVersion: "core.oam.dev/v1beta1" + kind: "Application" + spec: { + components: [ + { + type: "k8s-objects" + name: "spark-operator-ns" + properties: objects: [{ + apiVersion: "v1" + kind: "Namespace" + metadata: name: parameter.namespace + }] + }, + { + type: "k8s-objects" + name: "spark-cluster-ns" + properties: objects: [{ + apiVersion: "v1" + kind: "Namespace" + metadata: name: "spark-cluster" + }, + { + apiVersion: "v1" + kind: "ServiceAccount" + metadata: { + name: "spark" + namespace: "spark-cluster" + } + }] + }, + { + name: "spark-operator-helm" + type: "helm" + dependsOn: ["spark-operator-ns"] + type: "helm" + properties: { + repoType: "helm" + url: "https://googlecloudplatform.github.io/spark-on-k8s-operator/" + chart: "spark-operator" + targetNamespace: parameter["namespace"] + version: "1.1.26" + values: { + image: { + repository: parameter["imageRepository"] + tag: parameter["imageTag"] + } + + serviceAccounts: { + spark: { + create: parameter["createSparkServiceAccount"] + } + } + + serviceAccounts: { + sparkoperator: { + name: "spark-kubernetes-operator" + } + } + + webhook: { + enable: parameter["createWebhook"] + } + } + } + }, + ] + + policies: [ + { + name: "gc-dependency", + type: "garbage-collect", + properties: { + order: "dependency" + } + } + { + type: "shared-resource" + name: "shared-resource-via-namespace" + properties: rules: [{ + selector: resourceTypes: ["Namespace"] + }] + } + { + type: "topology" + name: "deploy-operator" + properties: { + namespace: parameter.namespace + if parameter.clusters != _|_ { + clusters: parameter.clusters + } + + if parameter.clusters == _|_ { + clusterLabelSelector: {} + } + } + } + ] + } +} From 06c27950fcbc3bb95e2de9a42204504e483347c4 Mon Sep 17 00:00:00 2001 From: yanghua Date: Tue, 28 Feb 2023 22:34:05 +0800 Subject: [PATCH 02/10] [Addon #579] Reformat cue file via cue fmt command Signed-off-by: yanghua --- .../definitions/spark-application.cue | 75 ++++---- .../spark-kubernetes-operator/parameter.cue | 14 +- .../spark-kubernetes-operator/template.cue | 173 +++++++++--------- 3 files changed, 131 insertions(+), 131 deletions(-) diff --git a/experimental/addons/spark-kubernetes-operator/definitions/spark-application.cue b/experimental/addons/spark-kubernetes-operator/definitions/spark-application.cue index 44b786b4..de5c5e65 100644 --- a/experimental/addons/spark-kubernetes-operator/definitions/spark-application.cue +++ b/experimental/addons/spark-kubernetes-operator/definitions/spark-application.cue @@ -11,11 +11,11 @@ template: { // +usage=Specify the spark application name name: string // +usage=Specify the namespace for spark application to install - namespace: string + namespace: string // +usage=Specify the application language type, e.g. "Scala", "Python", "Java" or "R" type: string // +usage=Specify the python version - pythonVersion ?: string + pythonVersion?: string // +usage=Specify the deploy mode, e.go "cluster", "client" or "in-cluster-client" mode: string // +usage=Specify the container image for the driver, executor, and init-container @@ -33,58 +33,57 @@ template: { // +usage=Specify the number of CPU cores to request for the executor pod executorCores: int // +usage=Specify a list of arguments to be passed to the application - arguments ?: [...string] + arguments?: [...string] // +usage=Specify the config information carries user-specified Spark configuration properties as they would use the "--conf" option in spark-submit - sparkConf ?: [string]: string + sparkConf?: [string]: string // +usage=Specify the config information carries user-specified Hadoop configuration properties as they would use the the "--conf" option in spark-submit. The SparkApplication controller automatically adds prefix "spark.hadoop." to Hadoop configuration properties - hadoopConf ?: [string]: string + hadoopConf?: [string]: string // +usage=Specify the name of the ConfigMap containing Spark configuration files such as log4j.properties. The controller will add environment variable SPARK_CONF_DIR to the path where the ConfigMap is mounted to - sparkConfigMap ?: string + sparkConfigMap?: string // +usage=Specify the name of the ConfigMap containing Hadoop configuration files such as core-site.xml. The controller will add environment variable HADOOP_CONF_DIR to the path where the ConfigMap is mounted to - hadoopConfigMap ?: string - + hadoopConfigMap?: string } - output: { - kind: "ClusterRoleBinding" - apiVersion: "rbac.authorization.k8s.io/v1" - metadata: name: parameter.name - roleRef: { - name: "edit" - apiGroup: "rbac.authorization.k8s.io" - kind: "ClusterRole" - } - subjects: [{ - name: "default" - kind: "ServiceAccount" - namespace: parameter.namespace - }] - } + output: { + kind: "ClusterRoleBinding" + apiVersion: "rbac.authorization.k8s.io/v1" + metadata: name: parameter.name + roleRef: { + name: "edit" + apiGroup: "rbac.authorization.k8s.io" + kind: "ClusterRole" + } + subjects: [{ + name: "default" + kind: "ServiceAccount" + namespace: parameter.namespace + }] + } outputs: { "spark": { - kind: "SparkApplication" - apiVersion: "sparkoperator.k8s.io/v1beta2" - metadata: { - name: parameter.name + kind: "SparkApplication" + apiVersion: "sparkoperator.k8s.io/v1beta2" + metadata: { + name: parameter.name namespace: parameter.namespace - } - spec: { - type: parameter.type - mode: parameter.mode - image: parameter.image - imagePullPolicy: parameter.imagePullPolicy - mainClass: parameter.mainClass + } + spec: { + type: parameter.type + mode: parameter.mode + image: parameter.image + imagePullPolicy: parameter.imagePullPolicy + mainClass: parameter.mainClass mainApplicationFile: parameter.mainApplicationFile - sparkVersion: parameter.sparkVersion + sparkVersion: parameter.sparkVersion driver: { - cores: parameter.driverCores + cores: parameter.driverCores } executor: { - cores: parameter.executorCores + cores: parameter.executorCores } - } + } } } } diff --git a/experimental/addons/spark-kubernetes-operator/parameter.cue b/experimental/addons/spark-kubernetes-operator/parameter.cue index 17de99c8..55765a2e 100644 --- a/experimental/addons/spark-kubernetes-operator/parameter.cue +++ b/experimental/addons/spark-kubernetes-operator/parameter.cue @@ -6,15 +6,15 @@ // container images, ports, and etc. parameter: { // +usage=Deploy to specified clusters. Leave empty to deploy to all clusters. - clusters?: [...string] + clusters?: [...string] // +usage=Namespace to deploy to - namespace: *"spark-operator" | string + namespace: *"spark-operator" | string // +usage=Specify if create the webhook or not - "createWebhook": *false | bool + "createWebhook": *false | bool // +usage=Specify the image repository - "imageRepository": *"ghcr.io/googlecloudplatform/spark-operator" | string - // +usage=Specify the image tag - "imageTag": *"v1beta2-1.3.8-3.1.1" | string + "imageRepository": *"ghcr.io/googlecloudplatform/spark-operator" | string + // +usage=Specify the image tag + "imageTag": *"v1beta2-1.3.8-3.1.1" | string // +usage=Specify if create the sa for job or not - "createSparkServiceAccount": *false|bool + "createSparkServiceAccount": *false | bool } diff --git a/experimental/addons/spark-kubernetes-operator/template.cue b/experimental/addons/spark-kubernetes-operator/template.cue index 55ac84bd..6d9d6af4 100644 --- a/experimental/addons/spark-kubernetes-operator/template.cue +++ b/experimental/addons/spark-kubernetes-operator/template.cue @@ -1,101 +1,102 @@ package main + output: { apiVersion: "core.oam.dev/v1beta1" kind: "Application" spec: { components: [ { - type: "k8s-objects" - name: "spark-operator-ns" - properties: objects: [{ - apiVersion: "v1" - kind: "Namespace" - metadata: name: parameter.namespace - }] - }, - { - type: "k8s-objects" - name: "spark-cluster-ns" - properties: objects: [{ - apiVersion: "v1" - kind: "Namespace" - metadata: name: "spark-cluster" - }, - { - apiVersion: "v1" - kind: "ServiceAccount" - metadata: { - name: "spark" - namespace: "spark-cluster" - } - }] - }, - { - name: "spark-operator-helm" - type: "helm" - dependsOn: ["spark-operator-ns"] - type: "helm" - properties: { - repoType: "helm" - url: "https://googlecloudplatform.github.io/spark-on-k8s-operator/" - chart: "spark-operator" - targetNamespace: parameter["namespace"] - version: "1.1.26" - values: { - image: { - repository: parameter["imageRepository"] - tag: parameter["imageTag"] - } + type: "k8s-objects" + name: "spark-operator-ns" + properties: objects: [{ + apiVersion: "v1" + kind: "Namespace" + metadata: name: parameter.namespace + }] + }, + { + type: "k8s-objects" + name: "spark-cluster-ns" + properties: objects: [{ + apiVersion: "v1" + kind: "Namespace" + metadata: name: "spark-cluster" + }, + { + apiVersion: "v1" + kind: "ServiceAccount" + metadata: { + name: "spark" + namespace: "spark-cluster" + } + }] + }, + { + name: "spark-operator-helm" + type: "helm" + dependsOn: ["spark-operator-ns"] + type: "helm" + properties: { + repoType: "helm" + url: "https://googlecloudplatform.github.io/spark-on-k8s-operator/" + chart: "spark-operator" + targetNamespace: parameter["namespace"] + version: "1.1.26" + values: { + image: { + repository: parameter["imageRepository"] + tag: parameter["imageTag"] + } - serviceAccounts: { - spark: { - create: parameter["createSparkServiceAccount"] - } - } + serviceAccounts: { + spark: { + create: parameter["createSparkServiceAccount"] + } + } - serviceAccounts: { - sparkoperator: { - name: "spark-kubernetes-operator" - } - } + serviceAccounts: { + sparkoperator: { + name: "spark-kubernetes-operator" + } + } - webhook: { - enable: parameter["createWebhook"] - } - } - } - }, + webhook: { + enable: parameter["createWebhook"] + } + } + } + }, ] - + policies: [ - { - name: "gc-dependency", - type: "garbage-collect", - properties: { - order: "dependency" - } - } - { - type: "shared-resource" - name: "shared-resource-via-namespace" - properties: rules: [{ - selector: resourceTypes: ["Namespace"] - }] - } - { - type: "topology" - name: "deploy-operator" - properties: { - namespace: parameter.namespace - if parameter.clusters != _|_ { - clusters: parameter.clusters - } - - if parameter.clusters == _|_ { - clusterLabelSelector: {} - } - } - } + { + name: "gc-dependency" + type: "garbage-collect" + properties: { + order: "dependency" + } + }, + { + type: "shared-resource" + name: "shared-resource-via-namespace" + properties: rules: [{ + selector: resourceTypes: ["Namespace"] + }] + }, + { + type: "topology" + name: "deploy-operator" + properties: { + namespace: parameter.namespace + if parameter.clusters != _|_ { + clusters: parameter.clusters + } + + if parameter.clusters == _|_ { + clusterLabelSelector: {} + } + } + }, ] } } From b19a329b3f5f5ac41af7b315c614a57d4ef108cc Mon Sep 17 00:00:00 2001 From: yanghua Date: Tue, 28 Feb 2023 22:51:57 +0800 Subject: [PATCH 03/10] [Addon #579] Address the review suggestion Signed-off-by: yanghua --- .../spark-kubernetes-operator/README.md | 66 +++++++++++++++++-- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/experimental/addons/spark-kubernetes-operator/README.md b/experimental/addons/spark-kubernetes-operator/README.md index bce23450..8541a256 100644 --- a/experimental/addons/spark-kubernetes-operator/README.md +++ b/experimental/addons/spark-kubernetes-operator/README.md @@ -115,7 +115,47 @@ I0227 16:54:37.069480 361176 apply.go:121] "creating object" name="spark-app-v1 Application spark-cluster/spark-app-v1 applied. ``` -3. Then, you can use the native command to check the status of the Spark applicaiton: +3. Then, there are serval ways to check the status(or detail information) of the Spark applicaiton: + +option 1: + +``` +$ vela status spark-app-v1 -n spark-cluster +About: + + Name: spark-app-v1 + Namespace: spark-cluster + Created at: 2023-02-27 16:54:37 +0800 CST + Status: running + +Workflow: + + mode: DAG-DAG + finished: true + Suspend: false + Terminated: false + Steps + - id: r9day16jok + name: deploy-topology-two-clouds + type: deploy + phase: succeeded + +Services: + + - Name: my-spark-application-component + Cluster: huaweibigdatak8s Namespace: spark-cluster + Type: spark-application + Healthy + No trait applied + + - Name: my-spark-application-component + Cluster: local Namespace: spark-cluster + Type: spark-application + Healthy + No trait applied +``` + +option 2: ``` $ kubectl get sparkapplications -n spark-cluster @@ -123,7 +163,7 @@ NAME STATUS ATTEMPTS START FINISH AGE my-spark-app RUNNING 1 2023-02-27T08:54:40Z 2m33s ``` -or get the application detail via this command: +option 3: ``` $ kubectl describe sparkapplication my-spark-app -n spark-cluster @@ -150,7 +190,7 @@ Metadata: ...... ``` -or get the general purpose detail information via this command: +option 4: ``` $ kubectl get app spark-app-v1 -n spark-cluster -oyaml @@ -166,5 +206,23 @@ metadata: $ kubectl get svc -n spark-cluster NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE my-spark-app-c58a1c869214bfe5-driver-svc ClusterIP None 7078/TCP,7079/TCP,4040/TCP 19m -my-spark-app-ui-svc ClusterIP xx.xx.xx.xx 4040/TCP 19m +my-spark-app-ui-svc ClusterIP xx.xx.xx.xx 4040/TCP 19m ``` + +# Architecture + +> The following content is quoted from the [original design document](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/docs/design.md). + +The operator consists of: +* a `SparkApplication` controller that watches events of creation, updates, and deletion of +`SparkApplication` objects and acts on the watch events, +* a *submission runner* that runs `spark-submit` for submissions received from the controller, +* a *Spark pod monitor* that watches for Spark pods and sends pod status updates to the controller, +* a [Mutating Admission Webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/) that handles customizations for Spark driver and executor pods based on the annotations on the pods added by the controller, +* and also a command-line tool named `sparkctl` for working with the operator. + +The following diagram shows how different components interact and work together. + +![Architecture Diagram](https://raw.githubusercontent.com/GoogleCloudPlatform/spark-on-k8s-operator/master/docs/architecture-diagram.png) + +Specifically, a user uses the `sparkctl` (or `kubectl`) to create a `SparkApplication` object. The `SparkApplication` controller receives the object through a watcher from the API server, creates a submission carrying the `spark-submit` arguments, and sends the submission to the *submission runner*. The submission runner submits the application to run and creates the driver pod of the application. Upon starting, the driver pod creates the executor pods. While the application is running, the *Spark pod monitor* watches the pods of the application and sends status updates of the pods back to the controller, which then updates the status of the application accordingly. From fc5d00b7d7c7b9dba0eb7f56a8d873c99688c02b Mon Sep 17 00:00:00 2001 From: yanghua Date: Wed, 1 Mar 2023 14:50:10 +0800 Subject: [PATCH 04/10] [Addon #579] Address the review suggestion Signed-off-by: yanghua --- .../spark-kubernetes-operator/sparkapp.yaml | 21 +++++++++++++++++++ .../spark-kubernetes-operator/README.md | 8 +++---- ...ark-application.cue => spark-workload.cue} | 2 +- 3 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 examples/spark-kubernetes-operator/sparkapp.yaml rename experimental/addons/spark-kubernetes-operator/definitions/{spark-application.cue => spark-workload.cue} (99%) diff --git a/examples/spark-kubernetes-operator/sparkapp.yaml b/examples/spark-kubernetes-operator/sparkapp.yaml new file mode 100644 index 00000000..9af21a6e --- /dev/null +++ b/examples/spark-kubernetes-operator/sparkapp.yaml @@ -0,0 +1,21 @@ +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: spark-app-v1 + namespace: spark-cluster +spec: + components: + - name: spark-workload-component + type: spark-workload + properties: + name: my-spark-app + namespace: spark-cluster + type: Scala + mode: cluster + image: "gcr.io/spark-operator/spark:v3.1.1" + imagePullPolicy: Always + mainClass: org.apache.spark.examples.streaming.JavaQueueStream + mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.12-3.1.1.jar" + sparkVersion: "3.1.1" + driverCores: 1 + executorCores: 1 diff --git a/experimental/addons/spark-kubernetes-operator/README.md b/experimental/addons/spark-kubernetes-operator/README.md index 8541a256..3ee33c8e 100644 --- a/experimental/addons/spark-kubernetes-operator/README.md +++ b/experimental/addons/spark-kubernetes-operator/README.md @@ -29,10 +29,10 @@ vela addon status spark-kubernetes-operator vela ls -A | grep spark ``` -* Secondly, show the component type `spark-cluster`, so we know how to use it in one application. As a spark user, you can choose the parameter to set for your spark cluster. +* Secondly, show the component type `spark-workload`, so we know how to use it in one application. As a spark user, you can choose the parameter to set for your spark cluster. ``` -vela show spark-application +vela show spark-workload # Specification +---------------------+------------------------------------------------------------------------------------------------------+-------------------+----------+---------+ | NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | @@ -79,8 +79,8 @@ metadata: namespace: spark-cluster spec: components: - - name: my-spark-application-component - type: spark-application + - name: spark-workload-component + type: spark-workload properties: name: my-spark-app namespace: spark-cluster diff --git a/experimental/addons/spark-kubernetes-operator/definitions/spark-application.cue b/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue similarity index 99% rename from experimental/addons/spark-kubernetes-operator/definitions/spark-application.cue rename to experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue index de5c5e65..44bcbe1b 100644 --- a/experimental/addons/spark-kubernetes-operator/definitions/spark-application.cue +++ b/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue @@ -1,4 +1,4 @@ -"spark-application": { +"spark-workload": { annotations: {} attributes: workload: type: "autodetects.core.oam.dev" description: "Describes a containerized spark application that can specify resource spec." From 5e615b61677edfb8e81db28517b573776919716c Mon Sep 17 00:00:00 2001 From: yanghua Date: Wed, 1 Mar 2023 14:54:46 +0800 Subject: [PATCH 05/10] [Addon #579] Address the review suggestion Signed-off-by: yanghua --- .../spark-kubernetes-operator/README.md | 40 ++++--------------- 1 file changed, 7 insertions(+), 33 deletions(-) diff --git a/experimental/addons/spark-kubernetes-operator/README.md b/experimental/addons/spark-kubernetes-operator/README.md index 3ee33c8e..1bba950e 100644 --- a/experimental/addons/spark-kubernetes-operator/README.md +++ b/experimental/addons/spark-kubernetes-operator/README.md @@ -67,38 +67,12 @@ vela show spark-workload # Example for how to run a component typed spark-cluster in application -1. Firstly, copy the following example to "spark-app-v1.yaml": - -> The addon will create a namespace named "spark-cluster" - -``` -apiVersion: core.oam.dev/v1beta1 -kind: Application -metadata: - name: spark-app-v1 - namespace: spark-cluster -spec: - components: - - name: spark-workload-component - type: spark-workload - properties: - name: my-spark-app - namespace: spark-cluster - type: Scala - mode: cluster - image: "gcr.io/spark-operator/spark:v3.1.1" - imagePullPolicy: Always - mainClass: org.apache.spark.examples.streaming.JavaQueueStream - mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.12-3.1.1.jar" - sparkVersion: "3.1.1" - driverCores: 1 - executorCores: 1 -``` +1. Firstly, download or copy `catalog/examples/spark-kubernetes-operator/sparkapp.yaml` 2. Secondly, start the application: ``` -vela up -f spark-app-v1.yaml +vela up -f sparkapp.yaml ``` You will see the stdout like this: @@ -120,7 +94,7 @@ Application spark-cluster/spark-app-v1 applied. option 1: ``` -$ vela status spark-app-v1 -n spark-cluster +vela status spark-app-v1 -n spark-cluster About: Name: spark-app-v1 @@ -158,7 +132,7 @@ Services: option 2: ``` -$ kubectl get sparkapplications -n spark-cluster +kubectl get sparkapplications -n spark-cluster NAME STATUS ATTEMPTS START FINISH AGE my-spark-app RUNNING 1 2023-02-27T08:54:40Z 2m33s ``` @@ -166,7 +140,7 @@ my-spark-app RUNNING 1 2023-02-27T08:54:40Z 2m33s option 3: ``` -$ kubectl describe sparkapplication my-spark-app -n spark-cluster +kubectl describe sparkapplication my-spark-app -n spark-cluster Name: my-spark-app Namespace: spark-cluster Labels: app.oam.dev/app-revision-hash=4e5592aea53a5961 @@ -193,7 +167,7 @@ Metadata: option 4: ``` -$ kubectl get app spark-app-v1 -n spark-cluster -oyaml +kubectl get app spark-app-v1 -n spark-cluster -oyaml apiVersion: core.oam.dev/v1beta1 kind: Application metadata: @@ -203,7 +177,7 @@ metadata: 4. Show the service of spark application via this command: ``` -$ kubectl get svc -n spark-cluster +kubectl get svc -n spark-cluster NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE my-spark-app-c58a1c869214bfe5-driver-svc ClusterIP None 7078/TCP,7079/TCP,4040/TCP 19m my-spark-app-ui-svc ClusterIP xx.xx.xx.xx 4040/TCP 19m From 00a08aa73933053f5e16a214cb017f0cdb8a8192 Mon Sep 17 00:00:00 2001 From: yanghua Date: Wed, 1 Mar 2023 17:40:44 +0800 Subject: [PATCH 06/10] [Addon #579] Address the review suggestion Signed-off-by: yanghua --- .../spark-kubernetes-operator/sparkapp.yaml | 11 ++ .../spark-kubernetes-operator/README.md | 116 ++++++++++++------ .../definitions/spark-workload.cue | 28 +++++ .../spark-kubernetes-operator/parameter.cue | 2 +- 4 files changed, 116 insertions(+), 41 deletions(-) diff --git a/examples/spark-kubernetes-operator/sparkapp.yaml b/examples/spark-kubernetes-operator/sparkapp.yaml index 9af21a6e..487c8e4b 100644 --- a/examples/spark-kubernetes-operator/sparkapp.yaml +++ b/examples/spark-kubernetes-operator/sparkapp.yaml @@ -17,5 +17,16 @@ spec: mainClass: org.apache.spark.examples.streaming.JavaQueueStream mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.12-3.1.1.jar" sparkVersion: "3.1.1" + volumes: + - name: "test-volume" + hostPath: + path: "/tmp" + type: Directory driverCores: 1 executorCores: 1 + driverVolumeMounts: + - name: "test-volume" + mountPath: "/tmp" + executorVolumeMounts: + - name: "test-volume" + mountPath: "/tmp" diff --git a/experimental/addons/spark-kubernetes-operator/README.md b/experimental/addons/spark-kubernetes-operator/README.md index 1bba950e..1168136f 100644 --- a/experimental/addons/spark-kubernetes-operator/README.md +++ b/experimental/addons/spark-kubernetes-operator/README.md @@ -34,39 +34,81 @@ vela ls -A | grep spark ``` vela show spark-workload # Specification -+---------------------+------------------------------------------------------------------------------------------------------+-------------------+----------+---------+ -| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | -+---------------------+------------------------------------------------------------------------------------------------------+-------------------+----------+---------+ -| name | Specify the spark application name. | string | true | | -| namespace | Specify the namespace for spark application to install. | string | true | | -| type | Specify the application language type, e.g. "Scala", "Python", "Java" or "R". | string | true | | -| pythonVersion | Specify the python version. | string | false | | -| mode | Specify the deploy mode, e.go "cluster", "client" or "in-cluster-client". | string | true | | -| image | Specify the container image for the driver, executor, and init-container. | string | true | | -| imagePullPolicy | Specify the image pull policy for the driver, executor, and init-container. | string | true | | -| mainClass | Specify the fully-qualified main class of the Spark application. | string | true | | -| mainApplicationFile | Specify the path to a bundled JAR, Python, or R file of the application. | string | true | | -| sparkVersion | Specify the version of Spark the application uses. | string | true | | -| driverCores | Specify the number of CPU cores to request for the driver pod. | int | true | | -| executorCores | Specify the number of CPU cores to request for the executor pod. | int | true | | -| arguments | Specify a list of arguments to be passed to the application. | []string | false | | -| sparkConf | Specify the config information carries user-specified Spark configuration properties as they would | map[string]string | false | | -| | use the "--conf" option in spark-submit. | | | | -| hadoopConf | Specify the config information carries user-specified Hadoop configuration properties as they would | map[string]string | false | | -| | use the the "--conf" option in spark-submit. The SparkApplication controller automatically adds | | | | -| | prefix "spark.hadoop." to Hadoop configuration properties. | | | | -| sparkConfigMap | Specify the name of the ConfigMap containing Spark configuration files such as log4j.properties. The | string | false | | -| | controller will add environment variable SPARK_CONF_DIR to the path where the ConfigMap is mounted | | | | -| | to. | | | | -| hadoopConfigMap | Specify the name of the ConfigMap containing Hadoop configuration files such as core-site.xml. The | string | false | | -| | controller will add environment variable HADOOP_CONF_DIR to the path where the ConfigMap is mounted | | | | -| | to. | | | | -+---------------------+------------------------------------------------------------------------------------------------------+-------------------+----------+---------+ - ++----------------------+------------------------------------------------------------------------------------------------------+-------------------------------------------------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++----------------------+------------------------------------------------------------------------------------------------------+-------------------------------------------------+----------+---------+ +| name | Specify the spark application name. | string | true | | +| namespace | Specify the namespace for spark application to install. | string | true | | +| type | Specify the application language type, e.g. "Scala", "Python", "Java" or "R". | string | true | | +| pythonVersion | Specify the python version. | string | false | | +| mode | Specify the deploy mode, e.go "cluster", "client" or "in-cluster-client". | string | true | | +| image | Specify the container image for the driver, executor, and init-container. | string | true | | +| imagePullPolicy | Specify the image pull policy for the driver, executor, and init-container. | string | true | | +| mainClass | Specify the fully-qualified main class of the Spark application. | string | true | | +| mainApplicationFile | Specify the path to a bundled JAR, Python, or R file of the application. | string | true | | +| sparkVersion | Specify the version of Spark the application uses. | string | true | | +| driverCores | Specify the number of CPU cores to request for the driver pod. | int | true | | +| executorCores | Specify the number of CPU cores to request for the executor pod. | int | true | | +| arguments | Specify a list of arguments to be passed to the application. | []string | false | | +| sparkConf | Specify the config information carries user-specified Spark configuration properties as they would | map[string]string | false | | +| | use the "--conf" option in spark-submit. | | | | +| hadoopConf | Specify the config information carries user-specified Hadoop configuration properties as they would | map[string]string | false | | +| | use the the "--conf" option in spark-submit. The SparkApplication controller automatically adds | | | | +| | prefix "spark.hadoop." to Hadoop configuration properties. | | | | +| sparkConfigMap | Specify the name of the ConfigMap containing Spark configuration files such as log4j.properties. The | string | false | | +| | controller will add environment variable SPARK_CONF_DIR to the path where the ConfigMap is mounted | | | | +| | to. | | | | +| hadoopConfigMap | Specify the name of the ConfigMap containing Hadoop configuration files such as core-site.xml. The | string | false | | +| | controller will add environment variable HADOOP_CONF_DIR to the path where the ConfigMap is mounted | | | | +| | to. | | | | +| volumes | Specify the list of Kubernetes volumes that can be mounted by the driver and/or executors. | [[]volumes](#volumes) | false | | +| driverVolumeMounts | Specify the volumes listed in "parameter.volumes" to mount into the main container’s filesystem for | [[]driverVolumeMounts](#drivervolumemounts) | false | | +| | driver pod. | | | | +| executorVolumeMounts | Specify the volumes listed in "parameter.volumes" to mount into the main container’s filesystem for | [[]executorVolumeMounts](#executorvolumemounts) | false | | +| | executor pod. | | | | ++----------------------+------------------------------------------------------------------------------------------------------+-------------------------------------------------+----------+---------+ + + +## volumes ++----------+-------------+-----------------------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++----------+-------------+-----------------------+----------+---------+ +| name | | string | true | | +| hostPath | | [hostPath](#hostpath) | true | | ++----------+-------------+-----------------------+----------+---------+ + + +### hostPath ++------+-------------+--------+----------+-----------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++------+-------------+--------+----------+-----------+ +| path | | string | true | | +| type | | string | false | Directory | ++------+-------------+--------+----------+-----------+ + + +## driverVolumeMounts ++-----------+-------------+--------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++-----------+-------------+--------+----------+---------+ +| name | | string | true | | +| mountPath | | string | true | | ++-----------+-------------+--------+----------+---------+ + + +## executorVolumeMounts ++-----------+-------------+--------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++-----------+-------------+--------+----------+---------+ +| name | | string | true | | +| mountPath | | string | true | | ++-----------+-------------+--------+----------+---------+ ``` # Example for how to run a component typed spark-cluster in application +**Note**: If we want to check and verify the mount volume, we need to specify `parameter.createWebhook` to be `true` + 1. Firstly, download or copy `catalog/examples/spark-kubernetes-operator/sparkapp.yaml` 2. Secondly, start the application: @@ -109,22 +151,16 @@ Workflow: Suspend: false Terminated: false Steps - - id: r9day16jok - name: deploy-topology-two-clouds - type: deploy + - id: vfgjkrxvih + name: spark-workload-component + type: apply-component phase: succeeded Services: - - Name: my-spark-application-component - Cluster: huaweibigdatak8s Namespace: spark-cluster - Type: spark-application - Healthy - No trait applied - - - Name: my-spark-application-component + - Name: spark-workload-component Cluster: local Namespace: spark-cluster - Type: spark-application + Type: spark-workload Healthy No trait applied ``` diff --git a/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue b/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue index 44bcbe1b..616e2391 100644 --- a/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue +++ b/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue @@ -42,6 +42,24 @@ template: { sparkConfigMap?: string // +usage=Specify the name of the ConfigMap containing Hadoop configuration files such as core-site.xml. The controller will add environment variable HADOOP_CONF_DIR to the path where the ConfigMap is mounted to hadoopConfigMap?: string + // +usage=Specify the list of Kubernetes volumes that can be mounted by the driver and/or executors + volumes?: [...{ + name: string + hostPath: { + path: string + type: *"Directory" | string + } + }] + // +usage=Specify the volumes listed in "parameter.volumes" to mount into the main container’s filesystem for driver pod + driverVolumeMounts?: [...{ + name: string + mountPath: string + }] + // +usage=Specify the volumes listed in "parameter.volumes" to mount into the main container’s filesystem for executor pod + executorVolumeMounts?: [...{ + name: string + mountPath: string + }] } output: { @@ -79,10 +97,20 @@ template: { sparkVersion: parameter.sparkVersion driver: { cores: parameter.driverCores + if parameter.driverVolumeMounts != _|_ { + volumeMounts: parameter.driverVolumeMounts + } } executor: { cores: parameter.executorCores + if parameter.executorVolumeMounts != _|_ { + volumeMounts: parameter.executorVolumeMounts + } + } + if parameter.volumes != _|_ { + volumes: parameter.volumes } + } } } diff --git a/experimental/addons/spark-kubernetes-operator/parameter.cue b/experimental/addons/spark-kubernetes-operator/parameter.cue index 55765a2e..0a47568e 100644 --- a/experimental/addons/spark-kubernetes-operator/parameter.cue +++ b/experimental/addons/spark-kubernetes-operator/parameter.cue @@ -9,7 +9,7 @@ parameter: { clusters?: [...string] // +usage=Namespace to deploy to namespace: *"spark-operator" | string - // +usage=Specify if create the webhook or not + // +usage=Specify if create the webhook or not "createWebhook": *false | bool // +usage=Specify the image repository "imageRepository": *"ghcr.io/googlecloudplatform/spark-operator" | string From 1c0cf185bce1e9ca105ea07630f2018ede808131 Mon Sep 17 00:00:00 2001 From: yanghua Date: Wed, 1 Mar 2023 17:46:07 +0800 Subject: [PATCH 07/10] [Addon #579] Add more details about mount volume Signed-off-by: yanghua --- experimental/addons/spark-kubernetes-operator/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experimental/addons/spark-kubernetes-operator/README.md b/experimental/addons/spark-kubernetes-operator/README.md index 1168136f..2299d023 100644 --- a/experimental/addons/spark-kubernetes-operator/README.md +++ b/experimental/addons/spark-kubernetes-operator/README.md @@ -107,7 +107,7 @@ vela show spark-workload # Example for how to run a component typed spark-cluster in application -**Note**: If we want to check and verify the mount volume, we need to specify `parameter.createWebhook` to be `true` +**Note**: If we want to check and verify the mount volume, we need to specify `parameter.createWebhook` to be `true`. For more details, please check the [official documentation](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/docs/user-guide.md#mounting-volumes). 1. Firstly, download or copy `catalog/examples/spark-kubernetes-operator/sparkapp.yaml` From 8916021e187054bb285548886158d68308ae77ae Mon Sep 17 00:00:00 2001 From: yanghua Date: Wed, 1 Mar 2023 23:09:35 +0800 Subject: [PATCH 08/10] [Addon #579] Refactor the spark-workload parameter definition Signed-off-by: yanghua --- .../spark-kubernetes-operator/sparkapp.yaml | 18 +-- .../spark-kubernetes-operator/README.md | 116 ++++++++++-------- .../definitions/spark-workload.cue | 44 +++---- 3 files changed, 93 insertions(+), 85 deletions(-) diff --git a/examples/spark-kubernetes-operator/sparkapp.yaml b/examples/spark-kubernetes-operator/sparkapp.yaml index 487c8e4b..34966ac4 100644 --- a/examples/spark-kubernetes-operator/sparkapp.yaml +++ b/examples/spark-kubernetes-operator/sparkapp.yaml @@ -22,11 +22,13 @@ spec: hostPath: path: "/tmp" type: Directory - driverCores: 1 - executorCores: 1 - driverVolumeMounts: - - name: "test-volume" - mountPath: "/tmp" - executorVolumeMounts: - - name: "test-volume" - mountPath: "/tmp" + driver: + cores: 1 + volumeMounts: + - name: "test-volume" + mountPath: "/tmp" + executor: + cores: 1 + volumeMounts: + - name: "test-volume" + mountPath: "/tmp" diff --git a/experimental/addons/spark-kubernetes-operator/README.md b/experimental/addons/spark-kubernetes-operator/README.md index 2299d023..eac7402f 100644 --- a/experimental/addons/spark-kubernetes-operator/README.md +++ b/experimental/addons/spark-kubernetes-operator/README.md @@ -34,39 +34,71 @@ vela ls -A | grep spark ``` vela show spark-workload # Specification -+----------------------+------------------------------------------------------------------------------------------------------+-------------------------------------------------+----------+---------+ -| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | -+----------------------+------------------------------------------------------------------------------------------------------+-------------------------------------------------+----------+---------+ -| name | Specify the spark application name. | string | true | | -| namespace | Specify the namespace for spark application to install. | string | true | | -| type | Specify the application language type, e.g. "Scala", "Python", "Java" or "R". | string | true | | -| pythonVersion | Specify the python version. | string | false | | -| mode | Specify the deploy mode, e.go "cluster", "client" or "in-cluster-client". | string | true | | -| image | Specify the container image for the driver, executor, and init-container. | string | true | | -| imagePullPolicy | Specify the image pull policy for the driver, executor, and init-container. | string | true | | -| mainClass | Specify the fully-qualified main class of the Spark application. | string | true | | -| mainApplicationFile | Specify the path to a bundled JAR, Python, or R file of the application. | string | true | | -| sparkVersion | Specify the version of Spark the application uses. | string | true | | -| driverCores | Specify the number of CPU cores to request for the driver pod. | int | true | | -| executorCores | Specify the number of CPU cores to request for the executor pod. | int | true | | -| arguments | Specify a list of arguments to be passed to the application. | []string | false | | -| sparkConf | Specify the config information carries user-specified Spark configuration properties as they would | map[string]string | false | | -| | use the "--conf" option in spark-submit. | | | | -| hadoopConf | Specify the config information carries user-specified Hadoop configuration properties as they would | map[string]string | false | | -| | use the the "--conf" option in spark-submit. The SparkApplication controller automatically adds | | | | -| | prefix "spark.hadoop." to Hadoop configuration properties. | | | | -| sparkConfigMap | Specify the name of the ConfigMap containing Spark configuration files such as log4j.properties. The | string | false | | -| | controller will add environment variable SPARK_CONF_DIR to the path where the ConfigMap is mounted | | | | -| | to. | | | | -| hadoopConfigMap | Specify the name of the ConfigMap containing Hadoop configuration files such as core-site.xml. The | string | false | | -| | controller will add environment variable HADOOP_CONF_DIR to the path where the ConfigMap is mounted | | | | -| | to. | | | | -| volumes | Specify the list of Kubernetes volumes that can be mounted by the driver and/or executors. | [[]volumes](#volumes) | false | | -| driverVolumeMounts | Specify the volumes listed in "parameter.volumes" to mount into the main container’s filesystem for | [[]driverVolumeMounts](#drivervolumemounts) | false | | -| | driver pod. | | | | -| executorVolumeMounts | Specify the volumes listed in "parameter.volumes" to mount into the main container’s filesystem for | [[]executorVolumeMounts](#executorvolumemounts) | false | | -| | executor pod. | | | | -+----------------------+------------------------------------------------------------------------------------------------------+-------------------------------------------------+----------+---------+ ++---------------------+------------------------------------------------------------------------------------------------------+-----------------------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++---------------------+------------------------------------------------------------------------------------------------------+-----------------------+----------+---------+ +| name | Specify the spark application name. | string | true | | +| namespace | Specify the namespace for spark application to install. | string | true | | +| type | Specify the application language type, e.g. "Scala", "Python", "Java" or "R". | string | true | | +| pythonVersion | Specify the python version. | string | false | | +| mode | Specify the deploy mode, e.go "cluster", "client" or "in-cluster-client". | string | true | | +| image | Specify the container image for the driver, executor, and init-container. | string | true | | +| imagePullPolicy | Specify the image pull policy for the driver, executor, and init-container. | string | true | | +| mainClass | Specify the fully-qualified main class of the Spark application. | string | true | | +| mainApplicationFile | Specify the path to a bundled JAR, Python, or R file of the application. | string | true | | +| sparkVersion | Specify the version of Spark the application uses. | string | true | | +| driver | Specify the driver sepc request for the driver pod. | [driver](#driver) | true | | +| executor | Specify the executor spec request for the executor pod. | [executor](#executor) | true | | +| arguments | Specify a list of arguments to be passed to the application. | []string | false | | +| sparkConf | Specify the config information carries user-specified Spark configuration properties as they would | map[string]string | false | | +| | use the "--conf" option in spark-submit. | | | | +| hadoopConf | Specify the config information carries user-specified Hadoop configuration properties as they would | map[string]string | false | | +| | use the the "--conf" option in spark-submit. The SparkApplication controller automatically adds | | | | +| | prefix "spark.hadoop." to Hadoop configuration properties. | | | | +| sparkConfigMap | Specify the name of the ConfigMap containing Spark configuration files such as log4j.properties. The | string | false | | +| | controller will add environment variable SPARK_CONF_DIR to the path where the ConfigMap is mounted | | | | +| | to. | | | | +| hadoopConfigMap | Specify the name of the ConfigMap containing Hadoop configuration files such as core-site.xml. The | string | false | | +| | controller will add environment variable HADOOP_CONF_DIR to the path where the ConfigMap is mounted | | | | +| | to. | | | | +| volumes | Specify the list of Kubernetes volumes that can be mounted by the driver and/or executors. | [[]volumes](#volumes) | false | | ++---------------------+------------------------------------------------------------------------------------------------------+-----------------------+----------+---------+ + + +## driver ++--------------+-------------+---------------------------------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++--------------+-------------+---------------------------------+----------+---------+ +| cores | | int | true | | +| volumeMounts | | [[]volumeMounts](#volumemounts) | false | | ++--------------+-------------+---------------------------------+----------+---------+ + + +### volumeMounts ++-----------+-------------+--------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++-----------+-------------+--------+----------+---------+ +| name | | string | true | | +| mountPath | | string | true | | ++-----------+-------------+--------+----------+---------+ + + +## executor ++--------------+-------------+---------------------------------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++--------------+-------------+---------------------------------+----------+---------+ +| cores | | int | true | | +| volumeMounts | | [[]volumeMounts](#volumemounts) | false | | ++--------------+-------------+---------------------------------+----------+---------+ + + +### volumeMounts ++-----------+-------------+--------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++-----------+-------------+--------+----------+---------+ +| name | | string | true | | +| mountPath | | string | true | | ++-----------+-------------+--------+----------+---------+ ## volumes @@ -85,24 +117,6 @@ vela show spark-workload | path | | string | true | | | type | | string | false | Directory | +------+-------------+--------+----------+-----------+ - - -## driverVolumeMounts -+-----------+-------------+--------+----------+---------+ -| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | -+-----------+-------------+--------+----------+---------+ -| name | | string | true | | -| mountPath | | string | true | | -+-----------+-------------+--------+----------+---------+ - - -## executorVolumeMounts -+-----------+-------------+--------+----------+---------+ -| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | -+-----------+-------------+--------+----------+---------+ -| name | | string | true | | -| mountPath | | string | true | | -+-----------+-------------+--------+----------+---------+ ``` # Example for how to run a component typed spark-cluster in application diff --git a/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue b/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue index 616e2391..f7959eb8 100644 --- a/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue +++ b/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue @@ -28,10 +28,22 @@ template: { mainApplicationFile: string // +usage=Specify the version of Spark the application uses sparkVersion: string - // +usage=Specify the number of CPU cores to request for the driver pod - driverCores: int - // +usage=Specify the number of CPU cores to request for the executor pod - executorCores: int + // +usage=Specify the driver sepc request for the driver pod + driver: { + cores: int + volumeMounts?: [...{ + name: string + mountPath: string + }] + } + // +usage=Specify the executor spec request for the executor pod + executor: { + cores: int + volumeMounts?: [...{ + name: string + mountPath: string + }] + } // +usage=Specify a list of arguments to be passed to the application arguments?: [...string] // +usage=Specify the config information carries user-specified Spark configuration properties as they would use the "--conf" option in spark-submit @@ -50,16 +62,6 @@ template: { type: *"Directory" | string } }] - // +usage=Specify the volumes listed in "parameter.volumes" to mount into the main container’s filesystem for driver pod - driverVolumeMounts?: [...{ - name: string - mountPath: string - }] - // +usage=Specify the volumes listed in "parameter.volumes" to mount into the main container’s filesystem for executor pod - executorVolumeMounts?: [...{ - name: string - mountPath: string - }] } output: { @@ -95,18 +97,8 @@ template: { mainClass: parameter.mainClass mainApplicationFile: parameter.mainApplicationFile sparkVersion: parameter.sparkVersion - driver: { - cores: parameter.driverCores - if parameter.driverVolumeMounts != _|_ { - volumeMounts: parameter.driverVolumeMounts - } - } - executor: { - cores: parameter.executorCores - if parameter.executorVolumeMounts != _|_ { - volumeMounts: parameter.executorVolumeMounts - } - } + driver: parameter.driver + executor: parameter.executor if parameter.volumes != _|_ { volumes: parameter.volumes } From 7036a637fea9e53a0c871d1abd88b061bfd838f5 Mon Sep 17 00:00:00 2001 From: yanghua Date: Thu, 2 Mar 2023 16:14:25 +0800 Subject: [PATCH 09/10] [Addon #579] Refactor the spark-workload parameter definition and add spark-py example Signed-off-by: yanghua --- .../sparkapp-py.yaml | 49 +++++++ .../spark-kubernetes-operator/sparkapp.yaml | 10 ++ .../spark-kubernetes-operator/README.md | 124 ++++++++++++------ .../definitions/spark-workload.cue | 49 ++++++- 4 files changed, 189 insertions(+), 43 deletions(-) create mode 100644 examples/spark-kubernetes-operator/sparkapp-py.yaml diff --git a/examples/spark-kubernetes-operator/sparkapp-py.yaml b/examples/spark-kubernetes-operator/sparkapp-py.yaml new file mode 100644 index 00000000..1731b23e --- /dev/null +++ b/examples/spark-kubernetes-operator/sparkapp-py.yaml @@ -0,0 +1,49 @@ +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: spark-app-v1 + namespace: spark-cluster +spec: + components: + - name: spark-workload-component + type: spark-workload + properties: + name: my-spark-py-app + namespace: spark-cluster + type: Python + pythonVersion: "3" + mode: cluster + image: "gcr.io/spark-operator/spark-py:v3.1.1" + imagePullPolicy: Always + mainClass: org.apache.spark.examples.streaming.JavaQueueStream + mainApplicationFile: "local:///opt/spark/examples/src/main/python/pi.py" + sparkVersion: "3.1.1" + restartPolicy: + type: OnFailure + onFailureRetries: 3 + onFailureRetryInterval: 10 + onSubmissionFailureRetries: 5 + onSubmissionFailureRetryInterval: 20 + volumes: + - name: "test-volume" + hostPath: + path: "/tmp" + type: Directory + driver: + cores: 1 + coreLimit: "1200m" + memory: "1024m" + labels: + version: 3.1.1 + volumeMounts: + - name: "test-volume" + mountPath: "/tmp" + executor: + cores: 1 + instances: 1 + memory: "1024m" + labels: + version: 3.1.1 + volumeMounts: + - name: "test-volume" + mountPath: "/tmp" diff --git a/examples/spark-kubernetes-operator/sparkapp.yaml b/examples/spark-kubernetes-operator/sparkapp.yaml index 34966ac4..920b68d1 100644 --- a/examples/spark-kubernetes-operator/sparkapp.yaml +++ b/examples/spark-kubernetes-operator/sparkapp.yaml @@ -17,6 +17,8 @@ spec: mainClass: org.apache.spark.examples.streaming.JavaQueueStream mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.12-3.1.1.jar" sparkVersion: "3.1.1" + restartPolicy: + type: Never volumes: - name: "test-volume" hostPath: @@ -24,11 +26,19 @@ spec: type: Directory driver: cores: 1 + coreLimit: "1200m" + memory: "1024m" + labels: + version: 3.1.1 volumeMounts: - name: "test-volume" mountPath: "/tmp" executor: cores: 1 + instances: 1 + memory: "1024m" + labels: + version: 3.1.1 volumeMounts: - name: "test-volume" mountPath: "/tmp" diff --git a/experimental/addons/spark-kubernetes-operator/README.md b/experimental/addons/spark-kubernetes-operator/README.md index eac7402f..b307c63c 100644 --- a/experimental/addons/spark-kubernetes-operator/README.md +++ b/experimental/addons/spark-kubernetes-operator/README.md @@ -34,44 +34,64 @@ vela ls -A | grep spark ``` vela show spark-workload # Specification -+---------------------+------------------------------------------------------------------------------------------------------+-----------------------+----------+---------+ -| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | -+---------------------+------------------------------------------------------------------------------------------------------+-----------------------+----------+---------+ -| name | Specify the spark application name. | string | true | | -| namespace | Specify the namespace for spark application to install. | string | true | | -| type | Specify the application language type, e.g. "Scala", "Python", "Java" or "R". | string | true | | -| pythonVersion | Specify the python version. | string | false | | -| mode | Specify the deploy mode, e.go "cluster", "client" or "in-cluster-client". | string | true | | -| image | Specify the container image for the driver, executor, and init-container. | string | true | | -| imagePullPolicy | Specify the image pull policy for the driver, executor, and init-container. | string | true | | -| mainClass | Specify the fully-qualified main class of the Spark application. | string | true | | -| mainApplicationFile | Specify the path to a bundled JAR, Python, or R file of the application. | string | true | | -| sparkVersion | Specify the version of Spark the application uses. | string | true | | -| driver | Specify the driver sepc request for the driver pod. | [driver](#driver) | true | | -| executor | Specify the executor spec request for the executor pod. | [executor](#executor) | true | | -| arguments | Specify a list of arguments to be passed to the application. | []string | false | | -| sparkConf | Specify the config information carries user-specified Spark configuration properties as they would | map[string]string | false | | -| | use the "--conf" option in spark-submit. | | | | -| hadoopConf | Specify the config information carries user-specified Hadoop configuration properties as they would | map[string]string | false | | -| | use the the "--conf" option in spark-submit. The SparkApplication controller automatically adds | | | | -| | prefix "spark.hadoop." to Hadoop configuration properties. | | | | -| sparkConfigMap | Specify the name of the ConfigMap containing Spark configuration files such as log4j.properties. The | string | false | | -| | controller will add environment variable SPARK_CONF_DIR to the path where the ConfigMap is mounted | | | | -| | to. | | | | -| hadoopConfigMap | Specify the name of the ConfigMap containing Hadoop configuration files such as core-site.xml. The | string | false | | -| | controller will add environment variable HADOOP_CONF_DIR to the path where the ConfigMap is mounted | | | | -| | to. | | | | -| volumes | Specify the list of Kubernetes volumes that can be mounted by the driver and/or executors. | [[]volumes](#volumes) | false | | -+---------------------+------------------------------------------------------------------------------------------------------+-----------------------+----------+---------+ ++---------------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++---------------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ +| name | Specify the spark application name. | string | true | | +| namespace | Specify the namespace for spark application to install. | string | true | | +| type | Specify the application language type, e.g. "Scala", "Python", "Java" or "R". | string | true | | +| pythonVersion | Specify the python version. | string | false | | +| mode | Specify the deploy mode, e.go "cluster", "client" or "in-cluster-client". | string | true | | +| image | Specify the container image for the driver, executor, and init-container. | string | true | | +| imagePullPolicy | Specify the image pull policy for the driver, executor, and init-container. | string | true | | +| mainClass | Specify the fully-qualified main class of the Spark application. | string | true | | +| mainApplicationFile | Specify the path to a bundled JAR, Python, or R file of the application. | string | true | | +| sparkVersion | Specify the version of Spark the application uses. | string | true | | +| restartPolicy | Specify the policy on if and in which conditions the controller should restart an application. | [restartPolicy](#restartpolicy) | false | | +| driver | Specify the driver sepc request for the driver pod. | [driver](#driver) | true | | +| executor | Specify the executor spec request for the executor pod. | [executor](#executor) | true | | +| arguments | Specify a list of arguments to be passed to the application. | []string | false | | +| sparkConf | Specify the config information carries user-specified Spark configuration properties as they would | map[string]string | false | | +| | use the "--conf" option in spark-submit. | | | | +| hadoopConf | Specify the config information carries user-specified Hadoop configuration properties as they would | map[string]string | false | | +| | use the the "--conf" option in spark-submit. The SparkApplication controller automatically adds | | | | +| | prefix "spark.hadoop." to Hadoop configuration properties. | | | | +| sparkConfigMap | Specify the name of the ConfigMap containing Spark configuration files such as log4j.properties. The | string | false | | +| | controller will add environment variable SPARK_CONF_DIR to the path where the ConfigMap is mounted | | | | +| | to. | | | | +| hadoopConfigMap | Specify the name of the ConfigMap containing Hadoop configuration files such as core-site.xml. The | string | false | | +| | controller will add environment variable HADOOP_CONF_DIR to the path where the ConfigMap is mounted | | | | +| | to. | | | | +| volumes | Specify the list of Kubernetes volumes that can be mounted by the driver and/or executors. | [[]volumes](#volumes) | false | | +| deps | Specify the dependencies captures all possible types of dependencies of a Spark application. | [deps](#deps) | false | | ++---------------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ + + +## restartPolicy ++----------------------------------+------------------------------------------------------------------------------------------------------+--------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++----------------------------------+------------------------------------------------------------------------------------------------------+--------+----------+---------+ +| type | Type value option: "Always", "Never", "OnFailure". | string | true | | +| onSubmissionFailureRetries | Specify the number of times to retry submitting an application before giving up. This is best | int | false | | +| | effort and actual retry attempts can be >= the value specified due to caching. These are required if | | | | +| | RestartPolicy is OnFailure. | | | | +| onFailureRetries | Specify the number of times to retry running an application before giving up. | int | false | | +| onSubmissionFailureRetryInterval | Specify the interval in seconds between retries on failed submissions. | int | false | | +| onFailureRetryInterval | Specify the interval in seconds between retries on failed runs. | int | false | | ++----------------------------------+------------------------------------------------------------------------------------------------------+--------+----------+---------+ ## driver -+--------------+-------------+---------------------------------+----------+---------+ -| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | -+--------------+-------------+---------------------------------+----------+---------+ -| cores | | int | true | | -| volumeMounts | | [[]volumeMounts](#volumemounts) | false | | -+--------------+-------------+---------------------------------+----------+---------+ ++--------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++--------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ +| cores | Specify the cores maps to spark.driver.cores or spark.executor.cores for the driver and executors, | int | false | | +| | respectively. | | | | +| coreLimit | Specify a hard limit on CPU cores for the pod. | string | false | | +| memory | Specify the amount of memory to request for the pod. | string | false | | +| labels | Specify the Kubernetes labels to be added to the pod. | map[string]string | false | | +| volumeMounts | Specify the volumes listed in “.spec.volumes” to mount into the main container’s filesystem. | [[]volumeMounts](#volumemounts) | false | | ++--------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ ### volumeMounts @@ -84,12 +104,17 @@ vela show spark-workload ## executor -+--------------+-------------+---------------------------------+----------+---------+ -| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | -+--------------+-------------+---------------------------------+----------+---------+ -| cores | | int | true | | -| volumeMounts | | [[]volumeMounts](#volumemounts) | false | | -+--------------+-------------+---------------------------------+----------+---------+ ++--------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++--------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ +| cores | Specify the cores maps to spark.driver.cores or spark.executor.cores for the driver and executors, | int | false | | +| | respectively. | | | | +| coreLimit | Specify a hard limit on CPU cores for the pod. | string | false | | +| memory | Specify the amount of memory to request for the pod. | string | false | | +| instances | | int | false | | +| labels | Specify the Kubernetes labels to be added to the pod. | map[string]string | false | | +| volumeMounts | Specify the volumes listed in “.spec.volumes” to mount into the main container’s filesystem. | [[]volumeMounts](#volumemounts) | false | | ++--------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ ### volumeMounts @@ -117,6 +142,23 @@ vela show spark-workload | path | | string | true | | | type | | string | false | Directory | +------+-------------+--------+----------+-----------+ + + +## deps ++-----------------+------------------------------------------------------------------------------------------------------+----------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++-----------------+------------------------------------------------------------------------------------------------------+----------+----------+---------+ +| jars | Specify a list of JAR files the Spark application depends on. | []string | false | | +| files | Specify a list of files the Spark application depends on. | []string | false | | +| pyFiles | Specify a list of Python files the Spark application depends on. | []string | false | | +| packages | Specify a list of maven coordinates of jars to include on the driver and executor classpaths. This | []string | false | | +| | will search the local maven repo, then maven central and any additional remote repositories given by | | | | +| | the “repositories” option. Each package should be of the form “groupId:artifactId:version”. | | | | +| excludePackages | Specify a list of “groupId:artifactId”, to exclude while resolving the dependencies provided in | []string | false | | +| | Packages to avoid dependency conflicts. | | | | +| repositories | Specify a list of additional remote repositories to search for the maven coordinate given with the | []string | false | | +| | “packages” option. | | | | ++-----------------+------------------------------------------------------------------------------------------------------+----------+----------+---------+ ``` # Example for how to run a component typed spark-cluster in application diff --git a/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue b/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue index f7959eb8..b3efe9b0 100644 --- a/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue +++ b/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue @@ -28,9 +28,30 @@ template: { mainApplicationFile: string // +usage=Specify the version of Spark the application uses sparkVersion: string + // +usage=Specify the policy on if and in which conditions the controller should restart an application + restartPolicy?: { + // +usage=Type value option: "Always", "Never", "OnFailure" + type: string + // +usage=Specify the number of times to retry submitting an application before giving up. This is best effort and actual retry attempts can be >= the value specified due to caching. These are required if RestartPolicy is OnFailure + onSubmissionFailureRetries?: int + // +usage=Specify the number of times to retry running an application before giving up + onFailureRetries?: int + // +usage=Specify the interval in seconds between retries on failed submissions + onSubmissionFailureRetryInterval?: int + // +usage=Specify the interval in seconds between retries on failed runs + onFailureRetryInterval?: int + } // +usage=Specify the driver sepc request for the driver pod driver: { - cores: int + // +usage=Specify the cores maps to spark.driver.cores or spark.executor.cores for the driver and executors, respectively + cores?: int + // +usage=Specify a hard limit on CPU cores for the pod + coreLimit?: string + // +usage=Specify the amount of memory to request for the pod + memory?: string + // +usage=Specify the Kubernetes labels to be added to the pod + labels?: [string]: string + // +usage=Specify the volumes listed in “.spec.volumes” to mount into the main container’s filesystem volumeMounts?: [...{ name: string mountPath: string @@ -38,7 +59,16 @@ template: { } // +usage=Specify the executor spec request for the executor pod executor: { - cores: int + // +usage=Specify the cores maps to spark.driver.cores or spark.executor.cores for the driver and executors, respectively + cores?: int + // +usage=Specify a hard limit on CPU cores for the pod + coreLimit?: string + // +usage=Specify the amount of memory to request for the pod + memory?: string + instances?: int + // +usage=Specify the Kubernetes labels to be added to the pod + labels?: [string]: string + // +usage=Specify the volumes listed in “.spec.volumes” to mount into the main container’s filesystem volumeMounts?: [...{ name: string mountPath: string @@ -62,6 +92,21 @@ template: { type: *"Directory" | string } }] + // +usage=Specify the dependencies captures all possible types of dependencies of a Spark application + deps?: { + // +usage=Specify a list of JAR files the Spark application depends on + jars?: [...string] + // +usage=Specify a list of files the Spark application depends on + files?: [...string] + // +usage=Specify a list of Python files the Spark application depends on + pyFiles?: [...string] + // +usage=Specify a list of maven coordinates of jars to include on the driver and executor classpaths. This will search the local maven repo, then maven central and any additional remote repositories given by the “repositories” option. Each package should be of the form “groupId:artifactId:version” + packages?: [...string] + // +usage=Specify a list of “groupId:artifactId”, to exclude while resolving the dependencies provided in Packages to avoid dependency conflicts + excludePackages?: [...string] + // +usage=Specify a list of additional remote repositories to search for the maven coordinate given with the “packages” option + repositories?: [...string] + } } output: { From 4e5c475a63f06701e29ca2b97fd8498aadccad3f Mon Sep 17 00:00:00 2001 From: yanghua Date: Thu, 2 Mar 2023 17:53:32 +0800 Subject: [PATCH 10/10] [Addon #603] Refactor the spark-workload parameter definition (add proxyUser option) Signed-off-by: yanghua --- experimental/addons/spark-kubernetes-operator/README.md | 5 +++-- .../spark-kubernetes-operator/definitions/spark-workload.cue | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/experimental/addons/spark-kubernetes-operator/README.md b/experimental/addons/spark-kubernetes-operator/README.md index b307c63c..fc715626 100644 --- a/experimental/addons/spark-kubernetes-operator/README.md +++ b/experimental/addons/spark-kubernetes-operator/README.md @@ -33,7 +33,6 @@ vela ls -A | grep spark ``` vela show spark-workload -# Specification +---------------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ | NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | +---------------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ @@ -47,6 +46,8 @@ vela show spark-workload | mainClass | Specify the fully-qualified main class of the Spark application. | string | true | | | mainApplicationFile | Specify the path to a bundled JAR, Python, or R file of the application. | string | true | | | sparkVersion | Specify the version of Spark the application uses. | string | true | | +| proxyUser | Specify the user to impersonate when submitting the application. It maps to the command-line flag | string | false | | +| | “–proxy-user” in spark-submit. | | | | | restartPolicy | Specify the policy on if and in which conditions the controller should restart an application. | [restartPolicy](#restartpolicy) | false | | | driver | Specify the driver sepc request for the driver pod. | [driver](#driver) | true | | | executor | Specify the executor spec request for the executor pod. | [executor](#executor) | true | | @@ -111,7 +112,7 @@ vela show spark-workload | | respectively. | | | | | coreLimit | Specify a hard limit on CPU cores for the pod. | string | false | | | memory | Specify the amount of memory to request for the pod. | string | false | | -| instances | | int | false | | +| instances | Specify the number of executor instances. | int | false | | | labels | Specify the Kubernetes labels to be added to the pod. | map[string]string | false | | | volumeMounts | Specify the volumes listed in “.spec.volumes” to mount into the main container’s filesystem. | [[]volumeMounts](#volumemounts) | false | | +--------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ diff --git a/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue b/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue index b3efe9b0..e655e211 100644 --- a/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue +++ b/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue @@ -28,6 +28,8 @@ template: { mainApplicationFile: string // +usage=Specify the version of Spark the application uses sparkVersion: string + // +usage=Specify the user to impersonate when submitting the application. It maps to the command-line flag “–proxy-user” in spark-submit + proxyUser?: string // +usage=Specify the policy on if and in which conditions the controller should restart an application restartPolicy?: { // +usage=Type value option: "Always", "Never", "OnFailure" @@ -64,7 +66,8 @@ template: { // +usage=Specify a hard limit on CPU cores for the pod coreLimit?: string // +usage=Specify the amount of memory to request for the pod - memory?: string + memory?: string + // +usage=Specify the number of executor instances instances?: int // +usage=Specify the Kubernetes labels to be added to the pod labels?: [string]: string