Improve alias support with nested tasks and a common section (tomzo#32)

README.md

@@ -139,6 +139,7 @@ Feel free to improve it!
     * [pluggable scm](#pluggable)
     * [config repo](#configrepo)
 1. [Secure variables](#to-generate-an-encrypted-value)
+1. [YAML Aliases](#yaml-aliases)
 
 # Pipeline
 
@@ -447,11 +448,9 @@ p4Material1:
   auto_update: false
 ```
 
-Instead of `password` you can use `encrypted_password`.
-
 ### Tfs
 
-*TODO: - not supported by yaml plugin yet*
+*TODO*
 
 ### Pluggable
 
@@ -464,45 +463,6 @@ myPluggableGit:
     - dir2
 ```
 
-### Configrepo
-
-This is a convenience for shorter and more consistent material declaration.
-When configuration repository is the same as one of pipeline materials,
-then you usually need to repeat definitions in XML and in JSON, for example:
-
-```yaml
-materials:
-  foo:
-    git: "https://github.com/tomzo/gocd-json-config-example.git",
-    branch: ci
-```
-
-And in server XML:
-```xml
-<config-repos>
-   <config-repo pluginId="yaml.config.plugin" id="repo1">
-     <git url="https://github.com/tomzo/gocd-json-config-example.git" branch="ci" />
-   </config-repo>
-</config-repos>
-```
-
-Notice that url and branch is repeated. This is inconvenient in case when you move repository,
-because it requires 2 updates, in code and in server XML.
-
-Using  **`configrepo` material type**, above repetition can be avoided,
-last example can be refactored into:
-
-```yaml
-materials:
-  foo:
-    type: configrepo
-```
-
-Server interprets `configrepo` material in this way:
-
-> Clone the material configuration of the repository we are parsing **as is in XML** and replace **name, destination and filters (whitelist/blacklist)**,
-then use the modified clone in place of `configrepo` material.
-
 ### Dependency
 
 To add a dependency on another pipeline stage:
@@ -713,6 +673,55 @@ using any of the keywords below (as in [yaml specs](http://yaml.org/type/bool.ht
  * **true** - `y|Y|yes|Yes|YES|true|True|TRUE|on|On|ON`
  * **false** - `n|N|no|No|NO|false|False|FALSE|off|Off|OFF`
 
+ ## YAML Aliases
+
+YAML Aliases ([specification](http://www.yaml.org/spec/1.2/spec.html#id2786196)) are supported and provide a way to avoid duplication.
+
+Aliases can be defined anywhere in the configuration as long as they are valid configuration elements.
+
+```yaml
+- exec:
+  command: make
+  arguments:
+   - clean
+   - &verbose_arg "VERBOSE=true" # define the alias
+- exec:
+  command: make
+  arguments:
+   - world
+   - *verbose_arg # use the alias
+```
+
+There is also a dedicated top-level `common` section which allows you to have all aliases in one place and where you don't need to worry about correct placement within the configuration.
+
+```yaml
+common:
+  verbose_arg: &verbose_arg "VERBOSE=true"
+  build_tasks: &build_tasks
+    - exec:
+        command: make
+        arguments:
+         - clean
+    - exec:
+        command: make
+        arguments:
+         - world
+pipelines:
+  pipe1:
+    stages:
+      - build:
+          jobs:
+            build:
+              tasks: *build_tasks
+            test:
+              tasks:
+               - *build_tasks # task list aliases can also be mixed with additional tasks in the same job
+               - exec:
+                   command: make
+                   arguments:
+                    - test
+```
+
 # Development
 
 Run all tests and create a ready to use jar

src/main/java/cd/go/plugin/config/yaml/transforms/JobTransform.java

@@ -173,11 +173,21 @@ private void addTasks(JsonObject jobJson, Map<String, Object> jobMap) {
             throw new YamlConfigException("tasks are required in a job");
         JsonArray tasksJson = new JsonArray();
         List<Object> taskList = (List<Object>) tasksObj;
+        addTasks(taskList, tasksJson);
+        jobJson.add(JSON_JOB_TASKS_FIELD, tasksJson);
+    }
+
+    private void addTasks(List<Object> taskList, JsonArray tasksJson) {
         for (Object maybeTask : taskList) {
-            JsonObject task = taskTransform.transform(maybeTask);
-            tasksJson.add(task);
+            if (maybeTask instanceof List) {
+                List<Object> taskNestedList = (List<Object>) maybeTask;
+                addTasks(taskNestedList, tasksJson);
+            }
+            else {
+                JsonObject task = taskTransform.transform(maybeTask);
+                tasksJson.add(task);
+            }
         }
-        jobJson.add(JSON_JOB_TASKS_FIELD, tasksJson);
     }
 
 }

src/main/java/cd/go/plugin/config/yaml/transforms/RootTransform.java

@@ -55,8 +55,8 @@ public JsonConfigCollection transform(Object rootObj, String location) {
                                 String.format("Failed to parse environment %s; %s", env.getKey(), ex.getMessage()), location));
                     }
                 }
-            } else
-                throw new YamlConfigException(pe.getKey() + " is invalid, expected pipelines or environments");
+            } else if (!"common".equalsIgnoreCase(pe.getKey()))
+                throw new YamlConfigException(pe.getKey() + " is invalid, expected pipelines, environments, or common");
         }
         return partialConfig;
     }

src/test/java/cd/go/plugin/config/yaml/YamlConfigPluginIntegrationTest.java

@@ -320,4 +320,26 @@ public void shouldContainValidFieldsInResponseMessage() throws UnhandledRequestT
         assertTrue(obj.has("environments"));
         assertTrue(obj.has("target_version"));
     }
+
+    @Test
+    public void shouldRespondSuccessToParseDirectoryRequestWhenAliasesCaseFile() throws UnhandledRequestTypeException,
+            IOException {
+        setupCase("aliasesCase", "aliases");
+
+        DefaultGoPluginApiRequest parseDirectoryRequest = new DefaultGoPluginApiRequest("configrepo", "1.0", "parse-directory");
+        String requestBody = "{\n" +
+                "    \"directory\":\"aliasesCase\",\n" +
+                "    \"configurations\":[]\n" +
+                "}";
+        parseDirectoryRequest.setRequestBody(requestBody);
+
+        GoPluginApiResponse response = plugin.handle(parseDirectoryRequest);
+        assertThat(response.responseCode(), is(DefaultGoPluginApiResponse.SUCCESS_RESPONSE_CODE));
+        JsonObject responseJsonObject = getJsonObjectFromResponse(response);
+        assertThat(responseJsonObject.get("errors"), Is.<JsonElement>is(new JsonArray()));
+        JsonArray pipelines = responseJsonObject.get("pipelines").getAsJsonArray();
+        assertThat(pipelines.size(), is(1));
+        JsonObject expected = (JsonObject) readJsonObject("examples.out/aliases.gocd.json");
+        assertThat(responseJsonObject, is(new JsonObjectMatcher(expected)));
+    }
 }

src/test/java/cd/go/plugin/config/yaml/transforms/JobTransformTest.java

@@ -47,6 +47,15 @@ public void shouldTransformElasticProfileJob() throws IOException {
         testTransform("elastic_profile");
     }
 
+    @Test
+    public void shouldTransformJobWithListOfListsTasks() throws IOException {
+        environmentTransform = new EnvironmentVariablesTransform();
+        taskTransform = new TaskTransform();
+        parser = new JobTransform(environmentTransform, taskTransform);
+
+        JsonObject job = testTransform("list_of_lists_tasks");
+    }
+
     private JsonObject testTransform(String caseFile) throws IOException {
         return testTransform(caseFile, caseFile);
     }

src/test/java/cd/go/plugin/config/yaml/transforms/RootTransformTest.java

@@ -43,6 +43,13 @@ public void shouldTransformRootWhen2PipelinesAnd2Environments() throws IOExcepti
         assertThat(collection.getJsonObject().get("pipelines").getAsJsonArray().size(), is(2));
     }
 
+    @Test
+    public void shouldTransformRootWithCommonSection() throws IOException {
+        JsonConfigCollection collection = readRootYaml("common_section");
+        assertThat(collection.getJsonObject().get("environments").getAsJsonArray().size(), is(1));
+        assertThat(collection.getJsonObject().get("pipelines").getAsJsonArray().size(), is(1));
+    }
+
     @Test(expected = YamlReader.YamlReaderException.class)
     public void shouldNotTransformRootWhenYAMLHasDuplicateKeys() throws IOException {
         readRootYaml("duplicate.materials.pipe");

src/test/resources/examples.out/aliases.gocd.json

@@ -0,0 +1,59 @@
+{
+  "target_version": 1,
+  "environments": [],
+  "pipelines": [{
+    "name": "pipe1",
+    "group": "aliases",
+    "materials": [{
+      "name": "mygit",
+      "type": "git",
+      "url": "http://my.example.org/mygit.git",
+      "branch": "ci"
+    }],
+    "stages": [{
+      "name": "prepare",
+      "jobs": [{
+        "name": "prepare",
+        "tasks": [{
+          "type": "exec",
+          "arguments": ["hello world"],
+          "command": "prepare"
+        }]
+      }]
+    }, {
+      "name": "build",
+      "jobs": [{
+        "name": "build",
+        "tasks": [{
+          "type": "exec",
+          "command": "init"
+        }, {
+          "type": "exec",
+          "arguments": ["VERBOSE=true"],
+          "command": "make"
+        }]
+      }]
+    }, {
+      "name": "test",
+      "jobs": [{
+        "name": "test",
+        "tasks": [{
+          "type": "exec",
+          "command": "init"
+        }, {
+          "type": "exec",
+          "arguments": ["VERBOSE=true"],
+          "command": "make"
+        }, {
+          "type": "exec",
+          "command": "test_unit"
+        }, {
+          "type": "exec",
+          "command": "test_integration"
+        }]
+      }]
+    }],
+    "location": "aliases.gocd.yaml"
+  }],
+  "errors": []
+}

src/test/resources/examples/aliases.gocd.yaml

@@ -0,0 +1,39 @@
+# aliases.gocd.yaml
+common: # aliases section
+  scalar_alias: &scalar_alias "hello world"
+  task_list_alias: &task_list_alias
+    - exec:
+        command: init
+    - exec:
+        command: make
+        arguments:
+          - "VERBOSE=true"
+pipelines:
+  pipe1:
+    group: aliases
+    materials:
+      mygit:
+        git: http://my.example.org/mygit.git
+        branch: ci
+    stages:
+      - prepare:
+          jobs:
+            prepare:
+              tasks:
+                - exec:
+                    command: prepare
+                    arguments: # use of scalar alias
+                      - *scalar_alias
+      - build:
+          jobs:
+            build:
+              tasks: *task_list_alias # use of task list alias as the full list
+      - test:
+          jobs:
+            test:
+              tasks:
+               - *task_list_alias # use of task list alias as a partial list with more added
+               - exec:
+                   command: test_unit
+               - exec:
+                  command: test_integration

src/test/resources/parts/jobs/list_of_lists_tasks.json

@@ -0,0 +1,20 @@
+{
+  "name": "test",
+  "tasks": [
+    {
+    "type": "exec",
+    "arguments": ["clean"],
+    "command": "./gradlew"
+    },
+    {
+      "type": "exec",
+      "arguments": ["assemble"],
+      "command": "./gradlew"
+    },
+    {
+      "type": "exec",
+      "arguments": ["test"],
+      "command": "./gradlew"
+    }
+  ]
+}

src/test/resources/parts/jobs/list_of_lists_tasks.yaml

@@ -0,0 +1,15 @@
+test:
+  tasks:
+    -
+      - exec:
+          command: ./gradlew
+          arguments:
+            - clean
+      - exec:
+          command: ./gradlew
+          arguments:
+            - assemble
+    - exec:
+        command: ./gradlew
+        arguments:
+          - test

src/test/resources/parts/roots/common_section.yaml

@@ -0,0 +1,6 @@
+common:
+  some_string: &alias1 hello world
+environments:
+  first:
+pipelines:
+  pipe1: