From 4886b567b336802dea7485b846be585283f8e0c7 Mon Sep 17 00:00:00 2001 From: Dave Henderson Date: Sat, 6 Apr 2019 13:06:25 -0400 Subject: [PATCH 1/3] Handle multi-document YAML streams Signed-off-by: Dave Henderson --- data/data_test.go | 2 +- .../tests/integration/datasources_file_test.go | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/data/data_test.go b/data/data_test.go index ae2c31c7b..7ce108c05 100644 --- a/data/data_test.go +++ b/data/data_test.go @@ -65,7 +65,7 @@ func TestUnmarshalArray(t *testing.T) { assert.EqualValues(t, expected, actual) } test(JSONArray(`["foo","bar",{"baz":{"qux": true},"quux":{"42":18},"corge":{"false":"blah"}}]`)) - test(YAMLArray(` + test(YAMLArray(`--- - foo - bar - baz: diff --git a/internal/tests/integration/datasources_file_test.go b/internal/tests/integration/datasources_file_test.go index 8304bca77..ec756abaf 100644 --- a/internal/tests/integration/datasources_file_test.go +++ b/internal/tests/integration/datasources_file_test.go @@ -39,6 +39,14 @@ FOO.BAR = "values can be double-quoted, and shell\nescapes are supported" BAZ = "variable expansion: ${FOO}" QUX='single quotes ignore $variables' +`, + "multidoc.yaml": `--- +# empty document +--- +foo: bar +--- +foo: baz +... `, }), fs.WithDir("sortorder", fs.WithFiles(map[string]string{ @@ -191,4 +199,12 @@ bar`}) "FOO.BAR": "values can be double-quoted, and shell\nescapes are supported", "QUX": "single quotes ignore $variables" }`}) + + result = icmd.RunCmd(icmd.Command(GomplateBin, + "-c=multidoc.yaml", + "-i", `{{ .multidoc.foo }}`, + ), func(c *icmd.Cmd) { + c.Dir = s.tmpDir.Path() + }) + result.Assert(c, icmd.Expected{ExitCode: 0, Out: "bar"}) } From 64076963ed8633b39c201b8f3ff27e4adf42169d Mon Sep 17 00:00:00 2001 From: Dave Henderson Date: Thu, 14 Nov 2019 20:56:52 -0500 Subject: [PATCH 2/3] fixup! Handle multi-document YAML streams --- data/data.go | 19 +++++++++++++++++++ data/data_test.go | 19 +++++++++++++++++++ funcs/data.go | 5 +++++ .../integration/datasources_file_test.go | 6 +++--- 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/data/data.go b/data/data.go index beda7f038..8f0a8d967 100644 --- a/data/data.go +++ b/data/data.go @@ -123,6 +123,25 @@ func YAMLArray(in string) ([]interface{}, error) { return obj, nil } +// YAMLStream - parses a (potentially) multi-document YAML stream +func YAMLStream(in string) (ch chan interface{}, err error) { + ch = make(chan interface{}) + s := strings.NewReader(in) + d := yaml.NewDecoder(s) + go func() { + for { + var o interface{} + err := d.Decode(&o) + if err == io.EOF { + close(ch) + break + } + ch <- o + } + }() + return ch, nil +} + // TOML - Unmarshal a TOML Object func TOML(in string) (interface{}, error) { obj := make(map[string]interface{}) diff --git a/data/data_test.go b/data/data_test.go index 7ce108c05..a76b84a0e 100644 --- a/data/data_test.go +++ b/data/data_test.go @@ -523,3 +523,22 @@ QUX='single quotes ignore $variables' assert.NoError(t, err) assert.EqualValues(t, expected, out) } + +func TestMultiDocYAML(t *testing.T) { + in := `foo: bar +--- +baz: qux +--- +# empty document with a comment +` + expected := []interface{}{ + map[string]interface{}{"foo": "bar"}, + map[string]interface{}{"baz": "qux"}, + nil, + } + ch, err := YAMLArray(in) + for i, out := range ch { + assert.NoError(t, err) + assert.EqualValues(t, expected[i], out) + } +} diff --git a/funcs/data.go b/funcs/data.go index b13619e3d..3d1c32934 100644 --- a/funcs/data.go +++ b/funcs/data.go @@ -67,6 +67,11 @@ func (f *DataFuncs) YAMLArray(in interface{}) ([]interface{}, error) { return data.YAMLArray(conv.ToString(in)) } +// YAMLStream - +func (f *DataFuncs) YAMLStream(in interface{}) (chan interface{}, error) { + return data.YAMLStream(conv.ToString(in)) +} + // TOML - func (f *DataFuncs) TOML(in interface{}) (interface{}, error) { return data.TOML(conv.ToString(in)) diff --git a/internal/tests/integration/datasources_file_test.go b/internal/tests/integration/datasources_file_test.go index ec756abaf..9b953337c 100644 --- a/internal/tests/integration/datasources_file_test.go +++ b/internal/tests/integration/datasources_file_test.go @@ -201,10 +201,10 @@ bar`}) }`}) result = icmd.RunCmd(icmd.Command(GomplateBin, - "-c=multidoc.yaml", - "-i", `{{ .multidoc.foo }}`, + "-d=multidoc.yaml", + "-i", `{{ range $d := (include "multidoc" | data.YAMLStream) }}---{{"\n"}}{{ range $k, $v := $d }}{{ $k }}: {{ $v }}{{"\n"}}{{ end }}{{ end }}`, ), func(c *icmd.Cmd) { c.Dir = s.tmpDir.Path() }) - result.Assert(c, icmd.Expected{ExitCode: 0, Out: "bar"}) + result.Assert(c, icmd.Expected{ExitCode: 0, Out: "---\n---\nfoo: bar\n---\nfoo: baz\n"}) } From 2ac0dcea48c3d0061d85bee0828d2ca032213d80 Mon Sep 17 00:00:00 2001 From: Dave Henderson Date: Thu, 14 Nov 2019 21:45:50 -0500 Subject: [PATCH 3/3] fixup! fixup! Handle multi-document YAML streams --- data/data.go | 23 ++++++++++------------- funcs/data.go | 2 +- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/data/data.go b/data/data.go index 8f0a8d967..fd97c7afc 100644 --- a/data/data.go +++ b/data/data.go @@ -124,22 +124,19 @@ func YAMLArray(in string) ([]interface{}, error) { } // YAMLStream - parses a (potentially) multi-document YAML stream -func YAMLStream(in string) (ch chan interface{}, err error) { - ch = make(chan interface{}) +func YAMLStream(in string) ([]interface{}, error) { + obj := []interface{}{} s := strings.NewReader(in) d := yaml.NewDecoder(s) - go func() { - for { - var o interface{} - err := d.Decode(&o) - if err == io.EOF { - close(ch) - break - } - ch <- o + for { + var o interface{} + err := d.Decode(&o) + if err == io.EOF { + break } - }() - return ch, nil + obj = append(obj, o) + } + return obj, nil } // TOML - Unmarshal a TOML Object diff --git a/funcs/data.go b/funcs/data.go index 3d1c32934..96b4c3522 100644 --- a/funcs/data.go +++ b/funcs/data.go @@ -68,7 +68,7 @@ func (f *DataFuncs) YAMLArray(in interface{}) ([]interface{}, error) { } // YAMLStream - -func (f *DataFuncs) YAMLStream(in interface{}) (chan interface{}, error) { +func (f *DataFuncs) YAMLStream(in interface{}) ([]interface{}, error) { return data.YAMLStream(conv.ToString(in)) }