From 9a900a479ed793a82a14df702cc152e333ed228e Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Sun, 11 Dec 2016 13:57:45 -0800 Subject: [PATCH 1/3] Annotate various types for JSON serialization Making the plan, diff and configuration structures JSON-serializable creates the possibility for machine-readable Terraform output of various kinds. --- config/config.go | 88 ++++++++++++++++++++------------------ config/module/tree_json.go | 51 ++++++++++++++++++++++ config/raw_config.go | 19 ++++++-- config/resource_mode.go | 11 +++++ terraform/diff.go | 60 +++++++++++++++++++------- terraform/plan.go | 10 ++--- 6 files changed, 172 insertions(+), 67 deletions(-) create mode 100644 config/module/tree_json.go diff --git a/config/config.go b/config/config.go index c7e4f351dc4c..e26d7463fbbb 100644 --- a/config/config.go +++ b/config/config.go @@ -26,15 +26,15 @@ type Config struct { // Dir is the path to the directory where this configuration was // loaded from. If it is blank, this configuration wasn't loaded from // any meaningful directory. - Dir string + Dir string `json:"source_dir"` - Terraform *Terraform - Atlas *AtlasConfig - Modules []*Module - ProviderConfigs []*ProviderConfig - Resources []*Resource - Variables []*Variable - Outputs []*Output + Terraform *Terraform `json:"terraform"` + Atlas *AtlasConfig `json:"atlas"` + Modules []*Module `json:"modules"` + ProviderConfigs []*ProviderConfig `json:"provider_configs"` + Resources []*Resource `json:"resources"` + Variables []*Variable `json:"variables"` + Outputs []*Output `json:"outputs"` // The fields below can be filled in by loaders for validation // purposes. @@ -44,14 +44,15 @@ type Config struct { // Terraform is the Terraform meta-configuration that can be present // in configuration files for configuring Terraform itself. type Terraform struct { - RequiredVersion string `hcl:"required_version"` // Required Terraform version (constraint) + // Required Terraform version (constraint) + RequiredVersion string `hcl:"required_version" json:"required_version"` } // AtlasConfig is the configuration for building in HashiCorp's Atlas. type AtlasConfig struct { - Name string - Include []string - Exclude []string + Name string `json:"name"` + Include []string `json:"include"` + Exclude []string `json:"exclude"` } // Module is a module used within a configuration. @@ -59,9 +60,9 @@ type AtlasConfig struct { // This does not represent a module itself, this represents a module // call-site within an existing configuration. type Module struct { - Name string - Source string - RawConfig *RawConfig + Name string `json:"name"` + Source string `json:"source"` + RawConfig *RawConfig `json:"config"` } // ProviderConfig is the configuration for a resource provider. @@ -69,9 +70,9 @@ type Module struct { // For example, Terraform needs to set the AWS access keys for the AWS // resource provider. type ProviderConfig struct { - Name string - Alias string - RawConfig *RawConfig + Name string `json:"name"` + Alias string `json:"alias,omitempty"` + RawConfig *RawConfig `json:"config"` } // A resource represents a single Terraform resource in the configuration. @@ -79,15 +80,18 @@ type ProviderConfig struct { // usual "create, read, update, delete" operations, depending on // the given Mode. type Resource struct { - Mode ResourceMode // which operations the resource supports - Name string - Type string - RawCount *RawConfig - RawConfig *RawConfig - Provisioners []*Provisioner - Provider string - DependsOn []string - Lifecycle ResourceLifecycle + + // which operations the resource supports + Mode ResourceMode `json:"mode"` + + Name string `json:"name"` + Type string `json:"type"` + RawCount *RawConfig `json:"meta_config"` + RawConfig *RawConfig `json:"config"` + Provisioners []*Provisioner `json:"provisioners,omitempty"` + Provider string `json:"provider_name,omitempty"` + DependsOn []string `json:"depends_on,omitempty"` + Lifecycle ResourceLifecycle `json:"lifecycle"` } // Copy returns a copy of this Resource. Helpful for avoiding shared @@ -115,9 +119,9 @@ func (r *Resource) Copy() *Resource { // ResourceLifecycle is used to store the lifecycle tuning parameters // to allow customized behavior type ResourceLifecycle struct { - CreateBeforeDestroy bool `mapstructure:"create_before_destroy"` - PreventDestroy bool `mapstructure:"prevent_destroy"` - IgnoreChanges []string `mapstructure:"ignore_changes"` + CreateBeforeDestroy bool `mapstructure:"create_before_destroy" json:"create_before_destroy"` + PreventDestroy bool `mapstructure:"prevent_destroy" json:"prevent_destroy"` + IgnoreChanges []string `mapstructure:"ignore_changes" json:"ignore_changes"` } // Copy returns a copy of this ResourceLifecycle @@ -133,9 +137,9 @@ func (r *ResourceLifecycle) Copy() *ResourceLifecycle { // Provisioner is a configured provisioner step on a resource. type Provisioner struct { - Type string - RawConfig *RawConfig - ConnInfo *RawConfig + Type string `json:"type"` + RawConfig *RawConfig `json:"config"` + ConnInfo *RawConfig `json:"conn_info,omitempty"` } // Copy returns a copy of this Provisioner @@ -149,10 +153,10 @@ func (p *Provisioner) Copy() *Provisioner { // Variable is a variable defined within the configuration. type Variable struct { - Name string - DeclaredType string `mapstructure:"type"` - Default interface{} - Description string + Name string `json:"name"` + DeclaredType string `json:"type" mapstructure:"type"` + Default interface{} `json:"default,omitempty"` + Description string `json:"description,omitempty"` } // Output is an output defined within the configuration. An output is @@ -160,11 +164,11 @@ type Variable struct { // output marked Sensitive will be output in a masked form following // application, but will still be available in state. type Output struct { - Name string - DependsOn []string - Description string - Sensitive bool - RawConfig *RawConfig + Name string `json:"name"` + DependsOn []string `json:"depends_on,omitempty"` + Description string `json:"description,omitempty"` + Sensitive bool `json:"sensitive"` + RawConfig *RawConfig `json:"config"` } // VariableType is the type of value a variable is holding, and returned diff --git a/config/module/tree_json.go b/config/module/tree_json.go new file mode 100644 index 000000000000..08d9432c57d1 --- /dev/null +++ b/config/module/tree_json.go @@ -0,0 +1,51 @@ +package module + +import ( + "bytes" + "encoding/json" + + "github.com/hashicorp/terraform/config" +) + +func (t *Tree) UnmarshalJSON(bs []byte) error { + t.lock.Lock() + defer t.lock.Unlock() + + // Decode the gob data + var data treeJSON + dec := json.NewDecoder(bytes.NewReader(bs)) + if err := dec.Decode(&data); err != nil { + return err + } + + // Set the fields + t.name = data.Name + t.config = data.Config + t.children = data.Children + t.path = data.Path + + return nil +} + +func (t *Tree) MarshalJSON() ([]byte, error) { + data := &treeJSON{ + Config: t.config, + Children: t.children, + Name: t.name, + Path: t.path, + } + + return json.Marshal(data) +} + +// treeJSON is used as a structure to JSON encode a tree. +// +// This structure is private so it can't be referenced but the fields are +// public, allowing us to properly encode this. When we decode this, we are +// able to turn it into a Tree. +type treeJSON struct { + Config *config.Config `json:"config"` + Children map[string]*Tree `json:"children"` + Name string `json:"name"` + Path []string `json:"path"` +} diff --git a/config/raw_config.go b/config/raw_config.go index 3bcc6241285b..720986fd0a9b 100644 --- a/config/raw_config.go +++ b/config/raw_config.go @@ -3,6 +3,7 @@ package config import ( "bytes" "encoding/gob" + "encoding/json" "sync" "github.com/hashicorp/hil" @@ -27,10 +28,10 @@ const UnknownVariableValue = "74D93920-ED26-11E3-AC10-0800200C9A66" // RawConfig supports a query-like interface to request // information from deep within the structure. type RawConfig struct { - Key string - Raw map[string]interface{} - Interpolations []ast.Node - Variables map[string]InterpolatedVariable + Key string `json:"key"` + Raw map[string]interface{} `json:"raw"` + Interpolations []ast.Node `json:"interpolations,omitempty"` + Variables map[string]InterpolatedVariable `json:"variables,omitempty"` lock sync.Mutex config map[string]interface{} @@ -323,3 +324,13 @@ func langEvalConfig(vs map[string]ast.Variable) *hil.EvalConfig { }, } } + +// MarshalJSON produces a JSON serialization of just the raw configuration +// values, under the assumption that the remainder can be derived from it. +// +// The output here is intended by consumption of tools outside of Terraform +// rather than by Terraform itself, so there is no corresponding +// UnmarshalJSON method. +func (r *RawConfig) MarshalJSON() ([]byte, error) { + return json.Marshal(r.Raw) +} diff --git a/config/resource_mode.go b/config/resource_mode.go index 877c6e8485f7..fe4fe1151d5c 100644 --- a/config/resource_mode.go +++ b/config/resource_mode.go @@ -7,3 +7,14 @@ const ( ManagedResourceMode ResourceMode = iota DataResourceMode ) + +func (m ResourceMode) MarshalJSON() ([]byte, error) { + switch m { + case ManagedResourceMode: + return []byte(`"managed"`), nil + case DataResourceMode: + return []byte(`"data"`), nil + default: + return []byte(`"invalid"`), nil + } +} diff --git a/terraform/diff.go b/terraform/diff.go index c50d3cedb386..ff5888a618a8 100644 --- a/terraform/diff.go +++ b/terraform/diff.go @@ -29,7 +29,7 @@ const ( // to an existing infrastructure. type Diff struct { // Modules contains all the modules that have a diff - Modules []*ModuleDiff + Modules []*ModuleDiff `json:"modules"` } // Prune cleans out unused structures in the diff without affecting @@ -197,9 +197,9 @@ func (d *Diff) init() { // ModuleDiff tracks the differences between resources to apply within // a single module. type ModuleDiff struct { - Path []string - Resources map[string]*InstanceDiff - Destroy bool // Set only by the destroy plan + Path []string `json:"path"` + Resources map[string]*InstanceDiff `json:"resources"` + Destroy bool `json:"destroy"` // Set only by the destroy plan } func (d *ModuleDiff) init() { @@ -360,10 +360,10 @@ func (d *ModuleDiff) String() string { // InstanceDiff is the diff of a resource from some state to another. type InstanceDiff struct { mu sync.Mutex - Attributes map[string]*ResourceAttrDiff - Destroy bool - DestroyDeposed bool - DestroyTainted bool + Attributes map[string]*ResourceAttrDiff `json:"attributes"` + Destroy bool `json:"destroy"` + DestroyDeposed bool `json:"destroy_deposed"` + DestroyTainted bool `json:"destroy_tainted"` } func (d *InstanceDiff) Lock() { d.mu.Lock() } @@ -371,14 +371,14 @@ func (d *InstanceDiff) Unlock() { d.mu.Unlock() } // ResourceAttrDiff is the diff of a single attribute of a resource. type ResourceAttrDiff struct { - Old string // Old Value - New string // New Value - NewComputed bool // True if new value is computed (unknown currently) - NewRemoved bool // True if this attribute is being removed - NewExtra interface{} // Extra information for the provider - RequiresNew bool // True if change requires new resource - Sensitive bool // True if the data should not be displayed in UI output - Type DiffAttrType + Old string `json:"old_value"` // Old Value + New string `json:"new_value"` // New Value + NewComputed bool `json:"new_computed,omitempty"` // True if new value is computed (unknown currently) + NewRemoved bool `json:"new_removed,omitempty"` // True if this attribute is being removed + NewExtra interface{} `json:"new_extra,omitempty"` // Extra information for the provider + RequiresNew bool `json:"requires_new,omitempty"` // True if change requires new resource + Sensitive bool `json:"sensitive,omitempty"` // True if the data should not be displayed in UI output + Type DiffAttrType `json:"type"` } // Empty returns true if the diff for this attr is neutral @@ -403,6 +403,34 @@ const ( DiffAttrOutput ) +func (t DiffAttrType) MarshalJSON() ([]byte, error) { + switch t { + case DiffAttrUnknown: + return []byte(`"unknown"`), nil + case DiffAttrInput: + return []byte(`"input"`), nil + case DiffAttrOutput: + return []byte(`"output"`), nil + default: + return []byte(`"invalid"`), nil + } +} + +func (t *DiffAttrType) UnmarshalJSON(b []byte) error { + if len(b) < 3 || b[0] != '"' { + return fmt.Errorf("DiffAttrType must be string in JSON") + } + switch b[1] { + case 'i': + *t = DiffAttrInput + case 'o': + *t = DiffAttrOutput + default: + *t = DiffAttrUnknown + } + return nil +} + func (d *InstanceDiff) init() { if d.Attributes == nil { d.Attributes = make(map[string]*ResourceAttrDiff) diff --git a/terraform/plan.go b/terraform/plan.go index 75023a0c6fa3..11532a538cf6 100644 --- a/terraform/plan.go +++ b/terraform/plan.go @@ -21,11 +21,11 @@ func init() { // Plan represents a single Terraform execution plan, which contains // all the information necessary to make an infrastructure change. type Plan struct { - Diff *Diff - Module *module.Tree - State *State - Vars map[string]interface{} - Targets []string + Diff *Diff `json:"diff"` + Module *module.Tree `json:"module"` + State *State `json:"state"` + Vars map[string]interface{} `json:"variables"` + Targets []string `json:"targets,omitempty"` once sync.Once } From 3bfc6088d5502878daf5eda28e4ef93d90ce0a6e Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Sun, 11 Dec 2016 14:13:07 -0800 Subject: [PATCH 2/3] Factor out state write preparation steps There are some steps that get done before writing a state to disk. Here we factor out those operations so that they can be used when a state is included as part of another structure that is being serialized, such as a plan. --- terraform/state.go | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/terraform/state.go b/terraform/state.go index 472fac0d5e10..b0540cdb5f95 100644 --- a/terraform/state.go +++ b/terraform/state.go @@ -1935,8 +1935,16 @@ func ReadStateV3(jsonBytes []byte) (*State, error) { return state, nil } -// WriteState writes a state somewhere in a binary format. -func WriteState(d *State, dst io.Writer) error { +// PrepareForWrite makes changes to the receiving state to prepare it for +// being serialized to disk or elsewhere. These steps are executed +// automatically when a top-level state is written directly with WriteState, +// but this separate method is useful for writing out data structures that +// *contain* states, such as plans. +func (d *State) PrepareForWrite() error { + if d == nil { + return nil + } + // Make sure it is sorted d.sort() @@ -1959,6 +1967,16 @@ func WriteState(d *State, dst io.Writer) error { } } + return nil +} + +// WriteState writes a state somewhere in a binary format. +func WriteState(d *State, dst io.Writer) error { + err := d.PrepareForWrite() + if err != nil { + return err + } + // Encode the data in a human-friendly way data, err := json.MarshalIndent(d, "", " ") if err != nil { From 32bc2b37ab682428b04b7568cb4c88fda7867a07 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Sun, 11 Dec 2016 15:39:43 -0800 Subject: [PATCH 3/3] terraform show -format=json As a simple, non-intrusive way to get machine-readable plan output, add a -format=json argument to the "terraform show" command. This then allows external tools to interrogate plan files, e.g. to enforce additional restrictions on what can be changed that Terraform itself is unable to represent. For completeness this also supports JSON-formatted state output, but of course that is rather less interesting because the on-disk state file is already in this same format. --- command/format_plan.go | 19 +++++ command/format_state.go | 13 ++++ command/show.go | 52 ++++++++++--- command/show_test.go | 73 +++++++++++++++++++ .../source/docs/commands/show.html.markdown | 19 +++++ 5 files changed, 166 insertions(+), 10 deletions(-) diff --git a/command/format_plan.go b/command/format_plan.go index e05f5753600c..9d5f3ab621cc 100644 --- a/command/format_plan.go +++ b/command/format_plan.go @@ -2,6 +2,7 @@ package command import ( "bytes" + "encoding/json" "fmt" "sort" "strings" @@ -229,3 +230,21 @@ func formatPlanModuleSingle( len(m.Resources))) buf.WriteString(opts.Color.Color("[reset]\n")) } + +func FormatPlanJSON(plan *terraform.Plan) string { + err := plan.State.PrepareForWrite() + if err != nil { + // should never happen + panic(err) + } + + dst := bytes.NewBuffer(make([]byte, 0, 64)) + enc := json.NewEncoder(dst) + enc.SetIndent("", " ") + err = enc.Encode(plan) + if err != nil { + // should never happen + panic(err) + } + return dst.String() +} diff --git a/command/format_state.go b/command/format_state.go index d54a7648add3..b5b019732e2b 100644 --- a/command/format_state.go +++ b/command/format_state.go @@ -2,6 +2,7 @@ package command import ( "bytes" + "encoding/json" "fmt" "sort" "strings" @@ -150,3 +151,15 @@ func formatStateModuleSingle( // Now just write how many resources are in here. buf.WriteString(fmt.Sprintf(" %d resource(s)\n", len(m.Resources))) } + +func FormatStateJSON(state *terraform.State) string { + dst := bytes.NewBuffer(make([]byte, 0, 64)) + enc := json.NewEncoder(dst) + enc.SetIndent("", " ") + err := enc.Encode(state) + if err != nil { + // should never happen + panic(err) + } + return dst.String() +} diff --git a/command/show.go b/command/show.go index 8a32c4a8d589..a9ce7e0e4abe 100644 --- a/command/show.go +++ b/command/show.go @@ -20,8 +20,11 @@ func (c *ShowCommand) Run(args []string) int { args = c.Meta.process(args, false) + var format string + cmdFlags := flag.NewFlagSet("show", flag.ContinueOnError) c.addModuleDepthFlag(cmdFlags, &moduleDepth) + cmdFlags.StringVar(&format, "format", "ui", "format-mode") cmdFlags.Usage = func() { c.Ui.Error(c.Help()) } if err := cmdFlags.Parse(args); err != nil { return 1 @@ -91,21 +94,37 @@ func (c *ShowCommand) Run(args []string) int { return 1 } - if plan != nil { - c.Ui.Output(FormatPlan(&FormatPlanOpts{ - Plan: plan, + switch format { + case "ui": // the default, if no -format option is specified + if plan != nil { + c.Ui.Output(FormatPlan(&FormatPlanOpts{ + Plan: plan, + Color: c.Colorize(), + ModuleDepth: moduleDepth, + })) + return 0 + } + + c.Ui.Output(FormatState(&FormatStateOpts{ + State: state, Color: c.Colorize(), ModuleDepth: moduleDepth, })) return 0 - } - c.Ui.Output(FormatState(&FormatStateOpts{ - State: state, - Color: c.Colorize(), - ModuleDepth: moduleDepth, - })) - return 0 + case "json": + if plan != nil { + c.Ui.Output(FormatPlanJSON(plan)) + return 0 + } + + c.Ui.Output(FormatStateJSON(state)) + return 0 + + default: + c.Ui.Error(fmt.Sprintf("%q is not a supported output format", format)) + return 1 + } } func (c *ShowCommand) Help() string { @@ -117,11 +136,24 @@ Usage: terraform show [options] [path] Options: + -format=name Specifies the output format. By default, human-readable + output is produced. Set -format=json for a + machine-readable JSON data structure. The remaining + options are ignored for JSON output. + -module-depth=n Specifies the depth of modules to show in the output. By default this is -1, which will expand all. -no-color If specified, output won't contain any color. +WARNING: JSON output is provided as a convenience for lightweight integrations +with external tools, but the JSON format is *not* frozen and may change in +future versions of Terraform. + +JSON output is also more detailed than the standard human-readable output and +may contain sensitive information that is not normally included, including +the values of any outputs that are marked as sensitive. + ` return strings.TrimSpace(helpText) } diff --git a/command/show_test.go b/command/show_test.go index eb73ebe2f441..ceec87a2dc44 100644 --- a/command/show_test.go +++ b/command/show_test.go @@ -1,6 +1,7 @@ package command import ( + "encoding/json" "io/ioutil" "os" "path/filepath" @@ -183,3 +184,75 @@ func TestShow_state(t *testing.T) { t.Fatalf("bad: \n%s", ui.ErrorWriter.String()) } } + +func TestShow_planJSON(t *testing.T) { + planPath := testPlanFile(t, &terraform.Plan{ + Module: new(module.Tree), + State: testState(), + }) + + ui := new(cli.MockUi) + c := &ShowCommand{ + Meta: Meta{ + ContextOpts: testCtxConfig(testProvider()), + Ui: ui, + }, + } + + args := []string{ + "-format=json", + planPath, + } + if code := c.Run(args); code != 0 { + t.Fatalf("bad: \n%s", ui.ErrorWriter.String()) + } + + output := ui.OutputWriter.Bytes() + data := make(map[string]interface{}) + err := json.Unmarshal(output, &data) + if err != nil { + t.Fatalf("not valid JSON: %s", err) + } + // Sniff to make sure this looks like a plan rather than a state + if _, ok := data["module"]; !ok { + t.Fatalf("JSON does not contain 'module' key") + } + if _, ok := data["state"]; !ok { + t.Fatalf("JSON does not contain 'state' key") + } +} + +func TestShow_stateJSON(t *testing.T) { + originalState := testState() + statePath := testStateFile(t, originalState) + + ui := new(cli.MockUi) + c := &ShowCommand{ + Meta: Meta{ + ContextOpts: testCtxConfig(testProvider()), + Ui: ui, + }, + } + + args := []string{ + "-format=json", + statePath, + } + if code := c.Run(args); code != 0 { + t.Fatalf("bad: \n%s", ui.ErrorWriter.String()) + } + + output := ui.OutputWriter.Bytes() + data := make(map[string]interface{}) + err := json.Unmarshal(output, &data) + if err != nil { + t.Fatalf("not valid JSON: %s", err) + } + // Sniff to make sure this looks like a state rather than a plan + if _, ok := data["modules"]; !ok { + t.Fatalf("JSON does not contain 'modules' key") + } + if _, ok := data["lineage"]; !ok { + t.Fatalf("JSON does not contain 'lineage' key") + } +} diff --git a/website/source/docs/commands/show.html.markdown b/website/source/docs/commands/show.html.markdown index f2b9edc5ee83..5f7443e0ae89 100644 --- a/website/source/docs/commands/show.html.markdown +++ b/website/source/docs/commands/show.html.markdown @@ -22,8 +22,27 @@ file. If no path is specified, the current state will be shown. The command-line flags are all optional. The list of available flags are: +* `-format=name` - Produces output in an alternative format. Currently "json" + is the only supported alternative format. See below for more information + and caveats regarding the JSON output. + * `-module-depth=n` - Specifies the depth of modules to show in the output. By default this is -1, which will expand all. * `-no-color` - Disables output with coloring +## JSON Output + +As a convenience for intepreting Terraform data using external tools, Terraform +can produce detailed plan and state information in JSON format. + +However, since Terraform is still quickly evolving we are unable to guarantee +100% compatibility with the current JSON data structures in future versions, +and so the current data structures are not documented in detail. + +Please use this feature sparingly and with care, and be ready to update any +integrations when moving to newer versions of Terraform. + +~> **Warning** The JSON output is generally more detailed than the +human-readable output, and in particular can include *sensitive information*. +The JSON output must therefore be stored and transmitted with care.