-
Notifications
You must be signed in to change notification settings - Fork 4.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
DSIP-63 YAML Pod for User Customization
* Added `YamlUtils` with `jackson-dataformat-yaml`(new dependency in `pom.xml`)* * Updated `K8sUtils` with default namespace injection* * Added user-customized YAML task mode, with frontend and backend*
- Loading branch information
Showing
32 changed files
with
1,629 additions
and
25 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
85 changes: 85 additions & 0 deletions
85
...hinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/YamlUtils.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.dolphinscheduler.common.utils; | ||
|
|
||
| import java.io.File; | ||
|
|
||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
| import com.fasterxml.jackson.core.type.TypeReference; | ||
| import com.fasterxml.jackson.databind.ObjectMapper; | ||
| import com.fasterxml.jackson.databind.SerializationFeature; | ||
| import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; | ||
| import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; | ||
|
|
||
| /** | ||
| * YAML Utilities | ||
| */ | ||
| @Slf4j | ||
| public class YamlUtils { | ||
|
|
||
| // YAML parser | ||
| private static final ObjectMapper objectMapper = YAMLMapper.builder() | ||
| .enable(SerializationFeature.INDENT_OUTPUT) | ||
| .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER) | ||
| .build(); | ||
|
|
||
| // ensure Singleton Pattern of `YamlUtils` | ||
| private YamlUtils() { | ||
| throw new UnsupportedOperationException("Construct YamlUtils"); | ||
| } | ||
|
|
||
| /** | ||
| * parse the YAML String | ||
| * | ||
| * @param yamlString YAML string to load | ||
| * @param typeReference the type reference specifying the type of the object to parse into | ||
| * @param <T> the type of the object | ||
| * @return an object of type T parsed from the YAML file, or null if parsing fails | ||
| */ | ||
| public static <T> T load(String yamlString, TypeReference<T> typeReference) { | ||
| try { | ||
| return objectMapper.readValue(yamlString, typeReference); | ||
| } catch (Exception exception) { | ||
| log.error("failed to parse YAML String ({}):" + "\n" + | ||
| "```yaml" + "\n" + | ||
| "{}" + "\n" + | ||
| "```" + "\n" + | ||
| "\n", | ||
| exception.getMessage(), yamlString); | ||
| return null; | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Loads and parses a YAML file into an object of the specified class. | ||
| * | ||
| * @param file YAML file to load | ||
| * @param typeReference the type reference specifying the type of the object to parse into | ||
| * @param <T> the type of the object | ||
| * @return an object of type T parsed from the YAML file, or null if parsing fails | ||
| */ | ||
| public static <T> T load(File file, TypeReference<T> typeReference) { | ||
| try { | ||
| return objectMapper.readValue(file, typeReference); | ||
| } catch (Exception exception) { | ||
| log.error("failed to parse YAML file `{}`: {}", file.getName(), exception.getMessage()); | ||
| return null; | ||
| } | ||
| } | ||
| } |
161 changes: 161 additions & 0 deletions
161
...cheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/YamlUtilsTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,161 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.dolphinscheduler.common.utils; | ||
|
|
||
| import java.io.File; | ||
| import java.util.ArrayList; | ||
| import java.util.HashMap; | ||
| import java.util.Map; | ||
| import java.util.Objects; | ||
|
|
||
| import org.junit.jupiter.api.Assertions; | ||
| import org.junit.jupiter.api.Test; | ||
|
|
||
| import com.fasterxml.jackson.core.type.TypeReference; | ||
|
|
||
| public class YamlUtilsTest { | ||
|
|
||
| private static final String yamlStringExpected = | ||
| "name: Yaml Parser" + '\n' + | ||
| "age: 30" + '\n' + | ||
| "address:" + '\n' + | ||
| " city: New York" + '\n' + | ||
| " state: NY" + '\n' + | ||
| " zipcode: 10001" + '\n'; | ||
|
|
||
| private static final Map<String, Object> yamlDataMapExpected = new HashMap<String, Object>() { | ||
|
|
||
| { | ||
| put("name", "Yaml Parser"); | ||
| put("age", 30); | ||
| put("address", new HashMap<String, Object>() { | ||
|
|
||
| { | ||
| put("city", "New York"); | ||
| put("state", "NY"); | ||
| put("zipcode", 10001); | ||
| } | ||
| }); | ||
| } | ||
| }; | ||
|
|
||
| @Test | ||
| public void testParseYamlString() { | ||
|
|
||
| Assertions.assertNull(YamlUtils.load("", new TypeReference<Map<String, Object>>() { | ||
| })); | ||
|
|
||
| Assertions.assertNull(YamlUtils.load(yamlStringExpected, new TypeReference<Assertions>() { | ||
| })); | ||
|
|
||
| Map<String, Object> yamlDataMapActual = | ||
| YamlUtils.load(yamlStringExpected, new TypeReference<Map<String, Object>>() { | ||
| }); | ||
| Assertions.assertEquals( | ||
| yamlDataMapExpected, yamlDataMapActual, | ||
| "[!] Test FAILED: expected YAML data: " + yamlDataMapExpected + | ||
| ", but actual data parsed: " + yamlDataMapActual); | ||
| } | ||
|
|
||
| @Test | ||
| public void testParseYamlFile() { | ||
|
|
||
| File emptyFile = new File(""); | ||
| Assertions.assertNull(YamlUtils.load(emptyFile, new TypeReference<Assertions>() { | ||
| })); | ||
|
|
||
| isFileTestcase01AddressYamlPassed(); | ||
| isFileTestcase02SimpleK8sPodYamlPassed(); | ||
| } | ||
|
|
||
| /* | ||
| * The following methods are helpers and should be kept private | ||
| */ | ||
|
|
||
| private void isFileTestcase01AddressYamlPassed() { | ||
| String filePathRelative = "yaml/testcase-01-address.yaml"; | ||
| String filePathAbsolute = | ||
| Objects.requireNonNull(getClass().getClassLoader().getResource(filePathRelative)).getFile(); | ||
| File file = new File(filePathAbsolute); | ||
| Map<String, Object> yamlDataMapActual = YamlUtils.load(file, new TypeReference<Map<String, Object>>() { | ||
| }); | ||
| Assertions.assertEquals( | ||
| yamlDataMapExpected, yamlDataMapActual, | ||
| "[!] Test FAILED on YAML file: yaml/testcase-01-address.yaml"); | ||
| } | ||
|
|
||
| private void isFileTestcase02SimpleK8sPodYamlPassed() { | ||
| String filePathRelative = "yaml/testcase-02-simple-k8s-pod.yaml"; | ||
| String filePathAbsolute = | ||
| Objects.requireNonNull(getClass().getClassLoader().getResource(filePathRelative)).getFile(); | ||
|
|
||
| Map<String, Object> yamlDataK8sPodExpected = new HashMap<String, Object>() { | ||
|
|
||
| { | ||
| put("apiVersion", "v1"); | ||
| put("kind", "Pod"); | ||
| put("metadata", new HashMap<String, Object>() { | ||
|
|
||
| { | ||
| put("name", "testcase-02-simple-k8s-pod-nginx"); | ||
| put("labels", new HashMap<String, Object>() { | ||
|
|
||
| { | ||
| put("app", "nginx"); | ||
| } | ||
| }); | ||
| } | ||
| }); | ||
| put("spec", new HashMap<String, Object>() { | ||
|
|
||
| { | ||
| put("containers", new ArrayList<Object>() { | ||
|
|
||
| { | ||
| add(new HashMap<String, Object>() { | ||
|
|
||
| { | ||
| put("name", "nginx-container"); | ||
| put("image", "nginx:1.10"); | ||
| put("ports", new ArrayList<Object>() { | ||
|
|
||
| { | ||
| add(new HashMap<String, Object>() { | ||
|
|
||
| { | ||
| put("containerPort", 80); | ||
| } | ||
| }); | ||
| } | ||
| }); | ||
| } | ||
| }); | ||
| } | ||
| }); | ||
| } | ||
| }); | ||
| } | ||
| }; | ||
| File file = new File(filePathAbsolute); | ||
| Map<String, Object> yamlDataK8sPodActual = YamlUtils.load(file, new TypeReference<Map<String, Object>>() { | ||
| }); | ||
| Assertions.assertEquals( | ||
| yamlDataK8sPodExpected, yamlDataK8sPodActual, | ||
| "[!] Test FAILED on YAML file: yaml/testcase-02-simple-k8s-pod.yaml"); | ||
| } | ||
| } |
24 changes: 24 additions & 0 deletions
24
dolphinscheduler-common/src/test/resources/yaml/testcase-01-address.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| # | ||
| # Licensed to the Apache Software Foundation (ASF) under one or more | ||
| # contributor license agreements. See the NOTICE file distributed with | ||
| # this work for additional information regarding copyright ownership. | ||
| # The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| # (the "License"); you may not use this file except in compliance with | ||
| # the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # | ||
|
|
||
| # testcase-01-address.yaml | ||
| name: Yaml Parser | ||
| age: 30 | ||
| address: | ||
| city: New York | ||
| state: NY | ||
| zipcode: 10001 |
30 changes: 30 additions & 0 deletions
30
dolphinscheduler-common/src/test/resources/yaml/testcase-02-simple-k8s-pod.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| # | ||
| # Licensed to the Apache Software Foundation (ASF) under one or more | ||
| # contributor license agreements. See the NOTICE file distributed with | ||
| # this work for additional information regarding copyright ownership. | ||
| # The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| # (the "License"); you may not use this file except in compliance with | ||
| # the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # | ||
|
|
||
| # testcase-02-simple-k8s-pod.yaml | ||
| apiVersion: v1 | ||
| kind: Pod | ||
| metadata: | ||
| name: testcase-02-simple-k8s-pod-nginx | ||
| labels: | ||
| app: nginx | ||
| spec: | ||
| containers: | ||
| - name: nginx-container | ||
| image: nginx:1.10 | ||
| ports: | ||
| - containerPort: 80 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.