feat(plugins): add local flag to exec module type

When local is set to true, the build command, tests and tasks are run in
the module directory, as opposed in the .garden/build directory.

Furthermore, the source code for local exec modules does not get synced
into the .garden/build directory.

Author: eysi09

docs/reference/module-types/exec.md

@@ -7,6 +7,14 @@ title: Exec
 A simple module for executing commands in your shell. This can be a useful escape hatch if no other module
 type fits your needs, and you just need to execute something (as opposed to deploy it, track its status etc.).
 
+By default, the `exec` module type executes the commands in the Garden build directory
+(under .garden/build/<module-name>). By setting `local: true`, the commands are executed in the module
+source directory instead.
+
+Note that Garden does not sync the source code for local exec modules into the Garden build directory.
+This means that include/exclude filters and ignore files are not applied to local exec modules, as the
+filtering is done during the sync.
+
 Below is the schema reference. For an introduction to configuring Garden modules, please look at our [Configuration
 guide](../../using-garden/configuration-files.md).
 The [first section](#configuration-keys) lists and describes the available
@@ -206,7 +214,10 @@ POSIX-style path or filename to copy the directory or file(s).
 
 [build](#build) > command
 
-The command to run inside the module's directory to perform the build.
+The command to run to perform the build.
+
+By default, the command is run inside the Garden build directory (under .garden/build/<module-name>).
+If the top level `local` directive is set to `true`, the command runs in the module source directory instead.
 
 | Type            | Required | Default |
 | --------------- | -------- | ------- |
@@ -223,6 +234,18 @@ build:
     - build
 ```
 
+### `local`
+
+If set to true, Garden will run the build command, tests, and tasks in the module source directory,
+instead of in the Garden build directory (under .garden/build/<module-name>).
+
+Garden will therefore not stage the build for local exec modules. This means that include/exclude filters
+and ignore files are not applied to local exec modules.
+
+| Type      | Required | Default |
+| --------- | -------- | ------- |
+| `boolean` | No       | `false` |
+
 ### `env`
 
 Key/value map of environment variables. Keys must be valid POSIX environment variable names (must not start with `GARDEN`) and values must be primitives.
@@ -283,11 +306,24 @@ Maximum duration (in seconds) of the task's execution.
 
 [tasks](#tasks) > command
 
-The command to run in the module build context.
+The command to run.
+
+By default, the command is run inside the Garden build directory (under .garden/build/<module-name>).
+If the top level `local` directive is set to `true`, the command runs in the module source directory instead.
 
 | Type            | Required |
 | --------------- | -------- |
-| `array[string]` | No       |
+| `array[string]` | Yes      |
+
+### `tasks[].env`
+
+[tasks](#tasks) > env
+
+Key/value map of environment variables. Keys must be valid POSIX environment variable names (must not start with `GARDEN`) and values must be primitives.
+
+| Type     | Required | Default |
+| -------- | -------- | ------- |
+| `object` | No       | `{}`    |
 
 ### `tests`
 
@@ -331,11 +367,14 @@ Maximum duration (in seconds) of the test run.
 
 [tests](#tests) > command
 
-The command to run in the module build context in order to test it.
+The command to run to test the module.
+
+By default, the command is run inside the Garden build directory (under .garden/build/<module-name>).
+If the top level `local` directive is set to `true`, the command runs in the module source directory instead.
 
 | Type            | Required |
 | --------------- | -------- |
-| `array[string]` | No       |
+| `array[string]` | Yes      |
 
 ### `tests[].env`
 
@@ -366,13 +405,15 @@ build:
         - source:
           target: <same as source path>
   command: []
+local: false
 env: {}
 tasks:
   - name:
     description:
     dependencies: []
     timeout: null
     command:
+    env: {}
 tests:
   - name:
     dependencies: []

docs/reference/module-types/openfaas.md

@@ -288,7 +288,7 @@ The command to run in the module build context in order to test it.
 
 | Type            | Required |
 | --------------- | -------- |
-| `array[string]` | No       |
+| `array[string]` | Yes      |
 
 ### `tests[].env`
 

examples/local-exec/.gitignore

@@ -0,0 +1 @@
+bin

examples/local-exec/README.md

@@ -0,0 +1,41 @@
+# Local Exec (Executing local commands with Garden)
+
+> Note: You need to have Go installed to run this project.
+
+This example project demonstrates how you can use the `exec` module type to run build commands, tasks and tests in the module directory, by setting `local: true` in the module config. By default the commands are executed in the `.garden/build` directory.
+
+The idea is to use a local `exec` module to run pre-build commands for a container module.
+
+## Project Structure
+
+The project consists of a `builder` module and a `backend` module. Both modules are in the same `garden.yml` file in the `backend` directory.
+
+The `backend` module is a simple `container` module that acts as a web server written in Go. The corresponding Dockerfile expects the web server binary to already be built before adding it to the image.
+
+To achieve this, we add a `go build` command to the `builder` module, set `local: true`, and then declare it as a build dependency in the `backend` module. We also tell Garden to copy the built binary to the `backend` build context since we're git ignoring it. This way, it's available with rest of the `backend` build context at `./garden/build/backend`. These are the relevant parts of the config
+
+```yaml
+# backend/garden.yml
+kind: Module
+type: exec
+local: true
+...
+build:
+  command: [go, build, -o, bin/backend]
+---
+kind: Module
+type: container
+build:
+  dependencies:
+    - name: builder
+      copy:
+        - source: bin
+          target: .
+...
+```
+
+This ensures that Garden runs `go build` in the module directory before it attempts to build the Docker image for the `backend` module.
+
+## Usage
+
+Run `garden deploy` to deploy the project. You'll notice that Garden first builds the Go binary, before it's added to the container image.

examples/local-exec/backend/Dockerfile

@@ -0,0 +1,8 @@
+FROM golang:1.8.3-alpine
+
+WORKDIR /server/
+COPY bin/backend .
+
+ENTRYPOINT ./backend
+
+EXPOSE 8080

examples/local-exec/backend/garden.yml

@@ -0,0 +1,33 @@
+kind: Module
+type: exec
+local: true
+name: builder
+# This runs in the module directory before the backend container that's specified below is built
+build:
+  command: [go, build, -o, bin/backend]
+env:
+  GOOS: linux
+  GOARCH: amd64
+---
+# This module's Dockerfile expects the Go binary to already be built
+kind: Module
+type: container
+name: backend
+# Setting the "builder" module as a dependency ensures that the Go binary is built
+# before the build step for this module is executed.
+build:
+  dependencies:
+    - name: builder
+      copy:
+        - source: bin
+          target: .
+services:
+  - name: backend
+    ports:
+      - name: http
+        containerPort: 8080
+        # Maps service:80 -> container:8080
+        servicePort: 80
+    ingresses:
+      - path: /hello-backend
+        port: http

examples/local-exec/backend/main.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+)
+
+func handler(w http.ResponseWriter, r *http.Request) {
+	fmt.Fprint(w, "Hello from Go!")
+}
+
+func main() {
+	http.HandleFunc("/hello-backend", handler)
+	fmt.Println("Server running...")
+
+	http.ListenAndServe(":8080", nil)
+}

examples/local-exec/garden.yml

@@ -0,0 +1,6 @@
+kind: Project
+name: local-exec
+environments:
+  - name: local
+    providers:
+      - name: local-kubernetes

examples/multiple-modules/README.md

@@ -2,7 +2,7 @@
 
 This project shows how you can configure several modules in a single directory.
 
-This is useful, for exmample, when you want to use more than one Dockerfile for the same code.
+This is useful, for example, when you want to use more than one Dockerfile for the same code.
 
 ```shell
 $ garden deploy

garden-service/src/build-dir.ts

@@ -19,10 +19,17 @@ import {
 import { emptyDir, ensureDir } from "fs-extra"
 import { ConfigurationError } from "./exceptions"
 import { FileCopySpec, Module, getModuleKey } from "./types/module"
-import { zip } from "lodash"
 import execa from "execa"
 import { normalizeLocalRsyncPath } from "./util/fs"
 import { LogEntry } from "./logger/log-entry"
+import { ModuleConfig } from "./config/module"
+import { ConfigGraph } from "./config-graph"
+
+// FIXME: We don't want to keep special casing this module type so we need to think
+// of a better way around this.
+function isLocalExecModule(moduleConfig: ModuleConfig) {
+  return moduleConfig.type === "exec" && moduleConfig.spec.local
+}
 
 // Lazily construct a directory of modules inside which all build steps are performed.
 
@@ -38,6 +45,12 @@ export class BuildDir {
   }
 
   async syncFromSrc(module: Module, log: LogEntry) {
+    // We don't sync local exec modules to the build dir
+    if (isLocalExecModule(module)) {
+      log.silly("Skipping syncing from source for local exec module")
+      return
+    }
+
     const files = module.version.files
       // Normalize to relative POSIX-style paths
       .map(f => normalize(isAbsolute(f) ? relative(module.path, f) : f))
@@ -52,20 +65,20 @@ export class BuildDir {
     })
   }
 
-  async syncDependencyProducts(module: Module, log: LogEntry) {
-    const buildPath = await this.buildPath(module.name)
-    const buildDependencies = await module.build.dependencies
-    const dependencyConfigs = module.build.dependencies || []
+  async syncDependencyProducts(module: Module, graph: ConfigGraph, log: LogEntry) {
+    const buildPath = await this.buildPath(module)
+    const buildDependencies = module.build.dependencies
 
-    await bluebirdMap(zip(buildDependencies, dependencyConfigs), async ([sourceModule, depConfig]) => {
-      if (!sourceModule || !depConfig || !depConfig.copy) {
+    await bluebirdMap(buildDependencies, async (buildDepConfig) => {
+      if (!buildDepConfig || !buildDepConfig.copy) {
         return
       }
 
-      const sourceBuildPath = await this.buildPath(getModuleKey(sourceModule.name, sourceModule.plugin))
+      const sourceModule = await graph.getModule(getModuleKey(buildDepConfig.name, buildDepConfig.plugin))
+      const sourceBuildPath = await this.buildPath(sourceModule)
 
       // Sync to the module's top-level dir by default.
-      await bluebirdMap(depConfig.copy, (copy: FileCopySpec) => {
+      await bluebirdMap(buildDepConfig.copy, (copy: FileCopySpec) => {
         if (isAbsolute(copy.source)) {
           throw new ConfigurationError(`Source path in build dependency copy spec must be a relative path`, {
             copySpec: copy,
@@ -89,9 +102,18 @@ export class BuildDir {
     await emptyDir(this.buildDirPath)
   }
 
-  async buildPath(moduleName: string): Promise<string> {
-    const path = resolve(this.buildDirPath, moduleName)
+  async buildPath(moduleOrConfig: Module | ModuleConfig): Promise<string> {
+    // We don't stage the build for local exec modules, so the module path is effectively the build path.
+    if (isLocalExecModule(moduleOrConfig)) {
+      return moduleOrConfig.path
+    }
+
+    // This returns the same result for modules and module configs
+    const moduleKey = getModuleKey(moduleOrConfig.name, moduleOrConfig.plugin)
+
+    const path = resolve(this.buildDirPath, moduleKey)
     await ensureDir(path)
+
     return path
   }
 

garden-service/src/config/config-context.ts

@@ -514,7 +514,7 @@ export class ModuleConfigContext extends ProviderConfigContext {
           stack: [...opts.stack || [], stackKey],
         })
         const version = await garden.resolveVersion(resolvedConfig.name, resolvedConfig.build.dependencies)
-        const buildPath = await garden.buildDir.buildPath(config.name)
+        const buildPath = await garden.buildDir.buildPath(config)
 
         return new ModuleContext(_this, resolvedConfig, buildPath, version)
       }],

garden-service/src/plugins/container/build.ts

@@ -11,9 +11,8 @@ import { ContainerModule } from "./config"
 import { ConfigurationError } from "../../exceptions"
 import { GetBuildStatusParams } from "../../types/plugin/module/getBuildStatus"
 import { BuildModuleParams } from "../../types/plugin/module/build"
-import chalk from "chalk"
 import { LogLevel } from "../../logger/log-node"
-import split2 = require("split2")
+import { createOutputStream } from "../../util/util"
 
 export async function getContainerBuildStatus({ module, log }: GetBuildStatusParams<ContainerModule>) {
   const identifier = await containerHelpers.imageExistsLocally(module)
@@ -65,14 +64,7 @@ export async function buildContainerModule({ module, log }: BuildModuleParams<Co
   }
 
   // Stream log to a status line
-  const outputStream = split2()
-  const statusLine = log.placeholder(LogLevel.debug)
-
-  outputStream.on("error", () => { })
-  outputStream.on("data", (line: Buffer) => {
-    statusLine.setState(chalk.gray("  → " + line.toString().slice(0, 80)))
-  })
-
+  const outputStream = createOutputStream(log.placeholder(LogLevel.debug))
   const timeout = module.spec.build.timeout
   const buildLog = await containerHelpers.dockerCli(module, [...cmdOpts, buildPath], { outputStream, timeout })