apache · Mighten · Aug 19, 2024
@@ -18,6 +18,8 @@ K8S task type used to execute a batch task. In this task, the worker submits the
 
 | **Parameter** | **Description** |
 |-------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| customConfig | The switch enables user-customized Kubernetes YAML mode when the default low-code Kubernetes Job provided does not meet the requirements. |
+| yamlContent | The YAML configuration file content for user-customized Kubernetes task. |
 | Namespace | The namespace for running k8s task. |
 | Min CPU | Minimum CPU requirement for running k8s task. |
 | Min Memory | Minimum memory requirement for running k8s task. |
@@ -41,6 +43,10 @@ Configure the required content according to the parameter descriptions above.
 
 ![K8S](../../../../img/tasks/demo/kubernetes-task-en.png)
 
+User-customized Kubernetes YAML mode can be turned on by switching to "Custom Template".
+
+![K8S-YAML](../../../../img/tasks/demo/kubernetes-yaml-task-en.png)
+
 ## Note
 
 Task name contains only lowercase alphanumeric characters or '-'
@@ -18,6 +18,8 @@ kubernetes任务类型，用于在kubernetes上执行一个短时和批处理的
 
 | **任务参数** | **描述** |
 |----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 自定义模板 | 当默认的低代码 Kubernetes 任务不符合要求时，该开关可以启用用户自定义的 Kubernetes YAML 模式。 |
+| YAML | 用户自定义 Kubernetes 任务的 YAML 配置文件内容 |
 | 命名空间 | 选择kubernetes集群上存在的命名空间 |
 | 最小CPU | 任务在kubernetes上运行所需的最小CPU |
 | 最小内存 | 任务在kubernetes上运行所需的最小内存 |
@@ -41,6 +43,10 @@ kubernetes任务类型，用于在kubernetes上执行一个短时和批处理的
 
 ![kubernetes](../../../../img/tasks/demo/kubernetes-task-en.png)
 
+切换到用户自定义模板，即可使用 YAML 配置文件启动自定义 kubernetes 任务。
+
+![K8S-YAML](../../../../img/tasks/demo/kubernetes-yaml-task-en.png)
+
 ## 注意事项
 
 任务名字限制在小写字母、数字和-这三种字符之中
@@ -365,6 +365,12 @@
  <artifactId>jackson-datatype-jsr310</artifactId>
  <version>${jackson.version}</version>
  </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-yaml</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+
  <!--protostuff-->
  <dependency>
  <groupId>io.protostuff</groupId>

@@ -74,6 +74,12 @@
  <groupId>com.fasterxml.jackson.core</groupId>
  <artifactId>jackson-databind</artifactId>
  </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-yaml</artifactId>
+ </dependency>
+
  <dependency>
  <groupId>org.apache.commons</groupId>
  <artifactId>commons-collections4</artifactId>

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.common.utils;
+
+import java.io.File;
+
+import lombok.extern.slf4j.Slf4j;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator;
+import com.fasterxml.jackson.dataformat.yaml.YAMLMapper;
+
+/**
+ * YAML Utilities
+ */
+@Slf4j
+public class YamlUtils {
+
+ // YAML parser
+ private static final ObjectMapper objectMapper = YAMLMapper.builder()
+ .enable(SerializationFeature.INDENT_OUTPUT)
+ .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER)
+ .build();
+
+ // ensure Singleton Pattern of `YamlUtils`
+ private YamlUtils() {
+ throw new UnsupportedOperationException("Construct YamlUtils");
+ }
+
+ /**
+ * parse the YAML String
+ *
+ * @param yamlString YAML string to load
+ * @param typeReference the type reference specifying the type of the object to parse into
+ * @param <T> the type of the object
+ * @return an object of type T parsed from the YAML file, or null if parsing fails
+ */
+ public static <T> T load(String yamlString, TypeReference<T> typeReference) {
+ try {
+ return objectMapper.readValue(yamlString, typeReference);
+ } catch (Exception exception) {
+ log.error("failed to parse YAML String ({}):" + "\n" +
+ "```yaml" + "\n" +
+ "{}" + "\n" +
+ "```" + "\n" +
+ "\n",
+ exception.getMessage(), yamlString);
+ return null;
+ }
+ }
+
+ /**
+ * Loads and parses a YAML file into an object of the specified class.
+ *
+ * @param file YAML file to load
+ * @param typeReference the type reference specifying the type of the object to parse into
+ * @param <T> the type of the object
+ * @return an object of type T parsed from the YAML file, or null if parsing fails
+ */
+ public static <T> T load(File file, TypeReference<T> typeReference) {
+ try {
+ return objectMapper.readValue(file, typeReference);
+ } catch (Exception exception) {
+ log.error("failed to parse YAML file `{}`: {}", file.getName(), exception.getMessage());
+ return null;
+ }
+ }
+}
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.common.utils;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+
+public class YamlUtilsTest {
+
+ private static final String yamlStringExpected =
+ "name: Yaml Parser" + '\n' +
+ "age: 30" + '\n' +
+ "address:" + '\n' +
+ " city: New York" + '\n' +
+ " state: NY" + '\n' +
+ " zipcode: 10001" + '\n';
+
+ private static final Map<String, Object> yamlDataMapExpected = new HashMap<String, Object>() {
+
+ {
+ put("name", "Yaml Parser");
+ put("age", 30);
+ put("address", new HashMap<String, Object>() {
+
+ {
+ put("city", "New York");
+ put("state", "NY");
+ put("zipcode", 10001);
+ }
+ });
+ }
+ };
+
+ @Test
+ public void testParseYamlString() {
+
+ Assertions.assertNull(YamlUtils.load("", new TypeReference<Map<String, Object>>() {
+ }));
+
+ Assertions.assertNull(YamlUtils.load(yamlStringExpected, new TypeReference<Assertions>() {
+ }));
+
+ Map<String, Object> yamlDataMapActual =
+ YamlUtils.load(yamlStringExpected, new TypeReference<Map<String, Object>>() {
+ });
+ Assertions.assertEquals(
+ yamlDataMapExpected, yamlDataMapActual,
+ "[!] Test FAILED: expected YAML data: " + yamlDataMapExpected +
+ ", but actual data parsed: " + yamlDataMapActual);
+ }
+
+ @Test
+ public void testParseYamlFile() {
+
+ File emptyFile = new File("");
+ Assertions.assertNull(YamlUtils.load(emptyFile, new TypeReference<Assertions>() {
+ }));
+
+ isFileTestcase01AddressYamlPassed();
+ isFileTestcase02SimpleK8sPodYamlPassed();
+ }
+
+ /*
+ * The following methods are helpers and should be kept private
+ */
+
+ private void isFileTestcase01AddressYamlPassed() {
+ String filePathRelative = "yaml/testcase-01-address.yaml";
+ String filePathAbsolute =
+ Objects.requireNonNull(getClass().getClassLoader().getResource(filePathRelative)).getFile();
+ File file = new File(filePathAbsolute);
+ Map<String, Object> yamlDataMapActual = YamlUtils.load(file, new TypeReference<Map<String, Object>>() {
+ });
+ Assertions.assertEquals(
+ yamlDataMapExpected, yamlDataMapActual,
+ "[!] Test FAILED on YAML file: yaml/testcase-01-address.yaml");
+ }
+
+ private void isFileTestcase02SimpleK8sPodYamlPassed() {
+ String filePathRelative = "yaml/testcase-02-simple-k8s-pod.yaml";
+ String filePathAbsolute =
+ Objects.requireNonNull(getClass().getClassLoader().getResource(filePathRelative)).getFile();
+
+ Map<String, Object> yamlDataK8sPodExpected = new HashMap<String, Object>() {
+
+ {
+ put("apiVersion", "v1");
+ put("kind", "Pod");
+ put("metadata", new HashMap<String, Object>() {
+
+ {
+ put("name", "testcase-02-simple-k8s-pod-nginx");
+ put("labels", new HashMap<String, Object>() {
+
+ {
+ put("app", "nginx");
+ }
+ });
+ }
+ });
+ put("spec", new HashMap<String, Object>() {
+
+ {
+ put("containers", new ArrayList<Object>() {
+
+ {
+ add(new HashMap<String, Object>() {
+
+ {
+ put("name", "nginx-container");
+ put("image", "nginx:1.10");
+ put("ports", new ArrayList<Object>() {
+
+ {
+ add(new HashMap<String, Object>() {
+
+ {
+ put("containerPort", 80);
+ }
+ });
+ }
+ });
+ }
+ });
+ }
+ });
+ }
+ });
+ }
+ };
+ File file = new File(filePathAbsolute);
+ Map<String, Object> yamlDataK8sPodActual = YamlUtils.load(file, new TypeReference<Map<String, Object>>() {
+ });
+ Assertions.assertEquals(
+ yamlDataK8sPodExpected, yamlDataK8sPodActual,
+ "[!] Test FAILED on YAML file: yaml/testcase-02-simple-k8s-pod.yaml");
+ }
+}
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# testcase-01-address.yaml
+name: Yaml Parser
+age: 30
+address:
+ city: New York
+ state: NY
+ zipcode: 10001
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# testcase-02-simple-k8s-pod.yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: testcase-02-simple-k8s-pod-nginx
+ labels:
+ app: nginx
+spec:
+ containers:
+ - name: nginx-container
+ image: nginx:1.10
+ ports:
+ - containerPort: 80
@@ -434,7 +434,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
  kubernetes-model-scheduling 5.10.2: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-scheduling/5.10.2, Apache 2.0
  kubernetes-model-storageclass 5.10.2: https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-storageclass/5.10.2, Apache 2.0
  zjsonpatch 0.3.0 https://mvnrepository.com/artifact/io.fabric8/zjsonpatch/0.3.0, Apache 2.0
- jackson-dataformat-yaml 2.13.0 https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/2.13.0, Apache 2.0
+ jackson-dataformat-yaml 2.13.4 https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/2.13.4, Apache 2.0
  logging-interceptor 4.9.3 https://mvnrepository.com/artifact/com.squareup.okhttp3/logging-interceptor/4.9.3, Apache 2.0
  okio 3.6.0 https://mvnrepository.com/artifact/com.squareup.okio/okio/3.6.0, Apache 2.0
  okio-jvm 3.6.0 https://repo1.maven.org/maven2/com/squareup/okio/okio-jvm/3.6.0, Apache 2.0