Merge pull request #1010 from hashicorp/f-ast-graph

core: formalize internals to do graph transformation steps and walking over an AST tree for execution

Makefile

@@ -40,6 +40,14 @@ updatedeps:
  | sort -u \
  | xargs go get -f -u -v
 
+cover:
+ @go tool cover 2>/dev/null; if [ $$? -eq 3 ]; then \
+ go get -u golang.org/x/tools/cmd/cover; \
+ fi
+ go test $(TEST) -coverprofile=coverage.out
+ go tool cover -html=coverage.out
+ rm coverage.out
+
 # vet runs the Go source code static analysis tool `vet` to find
 # any common errors.
 vet:

command/graph.go

@@ -58,11 +58,7 @@ func (c *GraphCommand) Run(args []string) int {
  return 1
  }
 
- opts := &terraform.GraphDotOpts{
- ModuleDepth: moduleDepth,
- }
-
- c.Ui.Output(terraform.GraphDot(g, opts))
+ c.Ui.Output(terraform.GraphDot(g, nil))
 
  return 0
 }

command/graph_test.go

@@ -26,7 +26,7 @@ func TestGraph(t *testing.T) {
  }
 
  output := ui.OutputWriter.String()
- if !strings.Contains(output, "digraph {") {
+ if !strings.Contains(output, "provider.test") {
  t.Fatalf("doesn't look like digraph: %s", output)
  }
 }
@@ -73,7 +73,7 @@ func TestGraph_noArgs(t *testing.T) {
  }
 
  output := ui.OutputWriter.String()
- if !strings.Contains(output, "digraph {") {
+ if !strings.Contains(output, "provider.test") {
  t.Fatalf("doesn't look like digraph: %s", output)
  }
 }
@@ -99,7 +99,7 @@ func TestGraph_plan(t *testing.T) {
  }
 
  output := ui.OutputWriter.String()
- if !strings.Contains(output, "digraph {") {
+ if !strings.Contains(output, "provider.test") {
  t.Fatalf("doesn't look like digraph: %s", output)
  }
 }

config/raw_config.go

@@ -92,6 +92,51 @@ func (r *RawConfig) Interpolate(vs map[string]ast.Variable) error {
  })
 }
 
+// Merge merges another RawConfig into this one (overriding any conflicting
+// values in this config) and returns a new config. The original config
+// is not modified.
+func (r *RawConfig) Merge(other *RawConfig) *RawConfig {
+ // Merge the raw configurations
+ raw := make(map[string]interface{})
+ for k, v := range r.Raw {
+ raw[k] = v
+ }
+ for k, v := range other.Raw {
+ raw[k] = v
+ }
+
+ // Create the result
+ result, err := NewRawConfig(raw)
+ if err != nil {
+ panic(err)
+ }
+
+ // Merge the interpolated results
+ result.config = make(map[string]interface{})
+ for k, v := range r.config {
+ result.config[k] = v
+ }
+ for k, v := range other.config {
+ result.config[k] = v
+ }
+
+ // Build the unknown keys
+ unknownKeys := make(map[string]struct{})
+ for _, k := range r.unknownKeys {
+ unknownKeys[k] = struct{}{}
+ }
+ for _, k := range other.unknownKeys {
+ unknownKeys[k] = struct{}{}
+ }
+
+ result.unknownKeys = make([]string, 0, len(unknownKeys))
+ for k, _ := range unknownKeys {
+ result.unknownKeys = append(result.unknownKeys, k)
+ }
+
+ return result
+}
+
 func (r *RawConfig) init() error {
  r.config = r.Raw
  r.Interpolations = nil

config/raw_config_test.go

@@ -114,6 +114,87 @@ func TestRawConfig_double(t *testing.T) {
  }
 }
 
+func TestRawConfig_merge(t *testing.T) {
+ raw1 := map[string]interface{}{
+ "foo": "${var.foo}",
+ "bar": "${var.bar}",
+ }
+
+ rc1, err := NewRawConfig(raw1)
+ if err != nil {
+ t.Fatalf("err: %s", err)
+ }
+
+ {
+ vars := map[string]ast.Variable{
+ "var.foo": ast.Variable{
+ Value: "foovalue",
+ Type: ast.TypeString,
+ },
+ "var.bar": ast.Variable{
+ Value: "nope",
+ Type: ast.TypeString,
+ },
+ }
+ if err := rc1.Interpolate(vars); err != nil {
+ t.Fatalf("err: %s", err)
+ }
+ }
+
+ raw2 := map[string]interface{}{
+ "bar": "${var.bar}",
+ "baz": "${var.baz}",
+ }
+
+ rc2, err := NewRawConfig(raw2)
+ if err != nil {
+ t.Fatalf("err: %s", err)
+ }
+
+ {
+ vars := map[string]ast.Variable{
+ "var.bar": ast.Variable{
+ Value: "barvalue",
+ Type: ast.TypeString,
+ },
+ "var.baz": ast.Variable{
+ Value: UnknownVariableValue,
+ Type: ast.TypeString,
+ },
+ }
+ if err := rc2.Interpolate(vars); err != nil {
+ t.Fatalf("err: %s", err)
+ }
+ }
+
+ // Merge the two
+ rc3 := rc1.Merge(rc2)
+
+ // Raw should be merged
+ raw3 := map[string]interface{}{
+ "foo": "${var.foo}",
+ "bar": "${var.bar}",
+ "baz": "${var.baz}",
+ }
+ if !reflect.DeepEqual(rc3.Raw, raw3) {
+ t.Fatalf("bad: %#v", rc3.Raw)
+ }
+
+ actual := rc3.Config()
+ expected := map[string]interface{}{
+ "foo": "foovalue",
+ "bar": "barvalue",
+ }
+ if !reflect.DeepEqual(actual, expected) {
+ t.Fatalf("bad: %#v", actual)
+ }
+
+ expectedKeys := []string{"baz"}
+ if !reflect.DeepEqual(rc3.UnknownKeys(), expectedKeys) {
+ t.Fatalf("bad: %#v", rc3.UnknownKeys())
+ }
+}
+
 func TestRawConfig_syntax(t *testing.T) {
  raw := map[string]interface{}{
  "foo": "${var",

dag/dag.go

@@ -0,0 +1,163 @@
+package dag
+
+import (
+ "fmt"
+ "strings"
+ "sync"
+
+ "github.com/hashicorp/go-multierror"
+)
+
+// AcyclicGraph is a specialization of Graph that cannot have cycles. With
+// this property, we get the property of sane graph traversal.
+type AcyclicGraph struct {
+ Graph
+}
+
+// WalkFunc is the callback used for walking the graph.
+type WalkFunc func(Vertex) error
+
+// Root returns the root of the DAG, or an error.
+//
+// Complexity: O(V)
+func (g *AcyclicGraph) Root() (Vertex, error) {
+ roots := make([]Vertex, 0, 1)
+ for _, v := range g.Vertices() {
+ if g.UpEdges(v).Len() == 0 {
+ roots = append(roots, v)
+ }
+ }
+
+ if len(roots) > 1 {
+ // TODO(mitchellh): make this error message a lot better
+ return nil, fmt.Errorf("multiple roots: %#v", roots)
+ }
+
+ if len(roots) == 0 {
+ return nil, fmt.Errorf("no roots found")
+ }
+
+ return roots[0], nil
+}
+
+// Validate validates the DAG. A DAG is valid if it has a single root
+// with no cycles.
+func (g *AcyclicGraph) Validate() error {
+ if _, err := g.Root(); err != nil {
+ return err
+ }
+
+ // Look for cycles of more than 1 component
+ var err error
+ var cycles [][]Vertex
+ for _, cycle := range StronglyConnected(&g.Graph) {
+ if len(cycle) > 1 {
+ cycles = append(cycles, cycle)
+ }
+ }
+ if len(cycles) > 0 {
+ for _, cycle := range cycles {
+ cycleStr := make([]string, len(cycle))
+ for j, vertex := range cycle {
+ cycleStr[j] = VertexName(vertex)
+ }
+
+ err = multierror.Append(err, fmt.Errorf(
+ "Cycle: %s", strings.Join(cycleStr, ", ")))
+ }
+ }
+
+ // Look for cycles to self
+ for _, e := range g.Edges() {
+ if e.Source() == e.Target() {
+ err = multierror.Append(err, fmt.Errorf(
+ "Self reference: %s", VertexName(e.Source())))
+ }
+ }
+
+ return err
+}
+
+// Walk walks the graph, calling your callback as each node is visited.
+// This will walk nodes in parallel if it can. Because the walk is done
+// in parallel, the error returned will be a multierror.
+func (g *AcyclicGraph) Walk(cb WalkFunc) error {
+ // Cache the vertices since we use it multiple times
+ vertices := g.Vertices()
+
+ // Build the waitgroup that signals when we're done
+ var wg sync.WaitGroup
+ wg.Add(len(vertices))
+ doneCh := make(chan struct{})
+ go func() {
+ defer close(doneCh)
+ wg.Wait()
+ }()
+
+ // The map of channels to watch to wait for vertices to finish
+ vertMap := make(map[Vertex]chan struct{})
+ for _, v := range vertices {
+ vertMap[v] = make(chan struct{})
+ }
+
+ // The map of whether a vertex errored or not during the walk
+ var errLock sync.Mutex
+ var errs error
+ errMap := make(map[Vertex]bool)
+ for _, v := range vertices {
+ // Build our list of dependencies and the list of channels to
+ // wait on until we start executing for this vertex.
+ depsRaw := g.DownEdges(v).List()
+ deps := make([]Vertex, len(depsRaw))
+ depChs := make([]<-chan struct{}, len(deps))
+ for i, raw := range depsRaw {
+ deps[i] = raw.(Vertex)
+ depChs[i] = vertMap[deps[i]]
+ }
+
+ // Get our channel so that we can close it when we're done
+ ourCh := vertMap[v]
+
+ // Start the goroutine to wait for our dependencies
+ readyCh := make(chan bool)
+ go func(deps []Vertex, chs []<-chan struct{}, readyCh chan<- bool) {
+ // First wait for all the dependencies
+ for _, ch := range chs {
+ <-ch
+ }
+
+ // Then, check the map to see if any of our dependencies failed
+ errLock.Lock()
+ defer errLock.Unlock()
+ for _, dep := range deps {
+ if errMap[dep] {
+ readyCh <- false
+ return
+ }
+ }
+
+ readyCh <- true
+ }(deps, depChs, readyCh)
+
+ // Start the goroutine that executes
+ go func(v Vertex, doneCh chan<- struct{}, readyCh <-chan bool) {
+ defer close(doneCh)
+ defer wg.Done()
+
+ var err error
+ if ready := <-readyCh; ready {
+ err = cb(v)
+ }
+
+ errLock.Lock()
+ defer errLock.Unlock()
+ if err != nil {
+ errMap[v] = true
+ errs = multierror.Append(errs, err)
+ }
+ }(v, ourCh, readyCh)
+ }
+
+ <-doneCh
+ return errs
+}