diff --git a/.gitignore b/.gitignore index 53a1f625b68..8aa98112d99 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,6 @@ yq.1 # debian pkg _build debian/files + +# intellij +/.idea \ No newline at end of file diff --git a/cmd/utils.go b/cmd/utils.go index faec5bed2cf..032d6519f17 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -93,7 +93,7 @@ func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder { case yqlib.JSONOutputFormat: return yqlib.NewJONEncoder(indent, colorsEnabled) case yqlib.PropsOutputFormat: - return yqlib.NewPropertiesEncoder() + return yqlib.NewPropertiesEncoder(unwrapScalar) case yqlib.CSVOutputFormat: return yqlib.NewCsvEncoder(',') case yqlib.TSVOutputFormat: diff --git a/pkg/yqlib/doc/usage/properties.md b/pkg/yqlib/doc/usage/properties.md index 97d05760906..1b0f39048ce 100644 --- a/pkg/yqlib/doc/usage/properties.md +++ b/pkg/yqlib/doc/usage/properties.md @@ -17,7 +17,7 @@ Given a sample.yml file of: ```yaml # block comments don't come through person: # neither do comments on maps - name: Mike # comments on values appear + name: Mike Wazowski # comments on values appear pets: - cat # comments on array values appear food: [pizza] # comments on arrays do not @@ -32,7 +32,36 @@ yq -o=props sample.yml will output ```properties # comments on values appear -person.name = Mike +person.name = Mike Wazowski + +# comments on array values appear +person.pets.0 = cat +person.food.0 = pizza +``` + +## Encode properties: scalar encapsulation +Note that string values with blank characters in them are encapsulated with double quotes + +Given a sample.yml file of: +```yaml +# block comments don't come through +person: # neither do comments on maps + name: Mike Wazowski # comments on values appear + pets: + - cat # comments on array values appear + food: [pizza] # comments on arrays do not +emptyArray: [] +emptyMap: [] + +``` +then +```bash +yq -o=props --unwrapScalar=false sample.yml +``` +will output +```properties +# comments on values appear +person.name = "Mike Wazowski" # comments on array values appear person.pets.0 = cat @@ -44,7 +73,7 @@ Given a sample.yml file of: ```yaml # block comments don't come through person: # neither do comments on maps - name: Mike # comments on values appear + name: Mike Wazowski # comments on values appear pets: - cat # comments on array values appear food: [pizza] # comments on arrays do not @@ -58,7 +87,7 @@ yq -o=props '... comments = ""' sample.yml ``` will output ```properties -person.name = Mike +person.name = Mike Wazowski person.pets.0 = cat person.food.0 = pizza ``` @@ -70,7 +99,7 @@ Given a sample.yml file of: ```yaml # block comments don't come through person: # neither do comments on maps - name: Mike # comments on values appear + name: Mike Wazowski # comments on values appear pets: - cat # comments on array values appear food: [pizza] # comments on arrays do not @@ -85,7 +114,7 @@ yq -o=props '(.. | select( (tag == "!!map" or tag =="!!seq") and length == 0)) = will output ```properties # comments on values appear -person.name = Mike +person.name = Mike Wazowski # comments on array values appear person.pets.0 = cat @@ -98,7 +127,7 @@ emptyMap = Given a sample.properties file of: ```properties # comments on values appear -person.name = Mike +person.name = Mike Wazowski # comments on array values appear person.pets.0 = cat @@ -112,7 +141,7 @@ yq -p=props sample.properties will output ```yaml person: - name: Mike # comments on values appear + name: Mike Wazowski # comments on values appear pets: - cat # comments on array values appear food: @@ -123,7 +152,7 @@ person: Given a sample.properties file of: ```properties # comments on values appear -person.name = Mike +person.name = Mike Wazowski # comments on array values appear person.pets.0 = cat @@ -137,7 +166,7 @@ yq -p=props -o=props '.person.pets.0 = "dog"' sample.properties will output ```properties # comments on values appear -person.name = Mike +person.name = Mike Wazowski # comments on array values appear person.pets.0 = dog diff --git a/pkg/yqlib/encoder_properties.go b/pkg/yqlib/encoder_properties.go index 287b707f64f..745f1279b81 100644 --- a/pkg/yqlib/encoder_properties.go +++ b/pkg/yqlib/encoder_properties.go @@ -12,10 +12,13 @@ import ( ) type propertiesEncoder struct { + unwrapScalar bool } -func NewPropertiesEncoder() Encoder { - return &propertiesEncoder{} +func NewPropertiesEncoder(unwrapScalar bool) Encoder { + return &propertiesEncoder{ + unwrapScalar: unwrapScalar, + } } func (pe *propertiesEncoder) CanHandleAliases() bool { @@ -75,7 +78,13 @@ func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node, p.SetComment(path, headAndLineComment(node)) switch node.Kind { case yaml.ScalarNode: - _, _, err := p.Set(path, node.Value) + var nodeValue string + if pe.unwrapScalar || !strings.Contains(node.Value, " ") { + nodeValue = node.Value + } else { + nodeValue = fmt.Sprintf("%q", node.Value) + } + _, _, err := p.Set(path, nodeValue) return err case yaml.DocumentNode: return pe.doEncode(p, node.Content[0], path) diff --git a/pkg/yqlib/encoder_properties_test.go b/pkg/yqlib/encoder_properties_test.go index 647010a6695..2c2b7f580a1 100644 --- a/pkg/yqlib/encoder_properties_test.go +++ b/pkg/yqlib/encoder_properties_test.go @@ -9,11 +9,11 @@ import ( "github.com/mikefarah/yq/v4/test" ) -func yamlToProps(sampleYaml string) string { +func yamlToProps(sampleYaml string, unwrapScalar bool) string { var output bytes.Buffer writer := bufio.NewWriter(&output) - var propsEncoder = NewPropertiesEncoder() + var propsEncoder = NewPropertiesEncoder(unwrapScalar) inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder()) if err != nil { panic(err) @@ -28,51 +28,100 @@ func yamlToProps(sampleYaml string) string { return strings.TrimSuffix(output.String(), "\n") } -func TestPropertiesEncoderSimple(t *testing.T) { +func TestPropertiesEncoderSimple_Unwrapped(t *testing.T) { var sampleYaml = `a: 'bob cool'` var expectedProps = `a = bob cool` - var actualProps = yamlToProps(sampleYaml) + var actualProps = yamlToProps(sampleYaml, true) test.AssertResult(t, expectedProps, actualProps) } -func TestPropertiesEncoderSimpleWithComments(t *testing.T) { +func TestPropertiesEncoderSimple_Wrapped(t *testing.T) { + var sampleYaml = `a: 'bob cool'` + + var expectedProps = `a = "bob cool"` + var actualProps = yamlToProps(sampleYaml, false) + test.AssertResult(t, expectedProps, actualProps) +} + +func TestPropertiesEncoderSimpleWithComments_Unwrapped(t *testing.T) { var sampleYaml = `a: 'bob cool' # line` var expectedProps = `# line a = bob cool` - var actualProps = yamlToProps(sampleYaml) + var actualProps = yamlToProps(sampleYaml, true) + test.AssertResult(t, expectedProps, actualProps) +} + +func TestPropertiesEncoderSimpleWithComments_Wrapped(t *testing.T) { + var sampleYaml = `a: 'bob cool' # line` + + var expectedProps = `# line +a = "bob cool"` + var actualProps = yamlToProps(sampleYaml, false) test.AssertResult(t, expectedProps, actualProps) } -func TestPropertiesEncoderDeep(t *testing.T) { +func TestPropertiesEncoderDeep_Unwrapped(t *testing.T) { var sampleYaml = `a: b: "bob cool" ` var expectedProps = `a.b = bob cool` - var actualProps = yamlToProps(sampleYaml) + var actualProps = yamlToProps(sampleYaml, true) + test.AssertResult(t, expectedProps, actualProps) +} + +func TestPropertiesEncoderDeep_Wrapped(t *testing.T) { + var sampleYaml = `a: + b: "bob cool" +` + + var expectedProps = `a.b = "bob cool"` + var actualProps = yamlToProps(sampleYaml, false) test.AssertResult(t, expectedProps, actualProps) } -func TestPropertiesEncoderDeepWithComments(t *testing.T) { +func TestPropertiesEncoderDeepWithComments_Unwrapped(t *testing.T) { var sampleYaml = `a: # a thing b: "bob cool" # b thing ` var expectedProps = `# b thing a.b = bob cool` - var actualProps = yamlToProps(sampleYaml) + var actualProps = yamlToProps(sampleYaml, true) + test.AssertResult(t, expectedProps, actualProps) +} + +func TestPropertiesEncoderDeepWithComments_Wrapped(t *testing.T) { + var sampleYaml = `a: # a thing + b: "bob cool" # b thing +` + + var expectedProps = `# b thing +a.b = "bob cool"` + var actualProps = yamlToProps(sampleYaml, false) test.AssertResult(t, expectedProps, actualProps) } -func TestPropertiesEncoderArray(t *testing.T) { +func TestPropertiesEncoderArray_Unwrapped(t *testing.T) { var sampleYaml = `a: b: [{c: dog}, {c: cat}] ` var expectedProps = `a.b.0.c = dog a.b.1.c = cat` - var actualProps = yamlToProps(sampleYaml) + var actualProps = yamlToProps(sampleYaml, true) + test.AssertResult(t, expectedProps, actualProps) +} + +func TestPropertiesEncoderArray_Wrapped(t *testing.T) { + var sampleYaml = `a: + b: [{c: dog named jim}, {c: cat named jam}] +` + + var expectedProps = `a.b.0.c = "dog named jim" +a.b.1.c = "cat named jam"` + var actualProps = yamlToProps(sampleYaml, false) test.AssertResult(t, expectedProps, actualProps) } diff --git a/pkg/yqlib/json_test.go b/pkg/yqlib/json_test.go index 0a400434e0c..06958f37961 100644 --- a/pkg/yqlib/json_test.go +++ b/pkg/yqlib/json_test.go @@ -9,7 +9,7 @@ import ( "github.com/mikefarah/yq/v4/test" ) -var complexExpectYaml = `D0, P[], (!!map)::a: Easy! as one two three +const complexExpectYaml = `D0, P[], (!!map)::a: Easy! as one two three b: c: 2 d: @@ -96,11 +96,15 @@ func decodeJSON(t *testing.T, jsonString string) *CandidateNode { } func testJSONScenario(t *testing.T, s formatScenario) { - if s.scenarioType == "encode" || s.scenarioType == "roundtrip" { + switch s.scenarioType { + case "encode", "decode": test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewJONEncoder(s.indent, false)), s.description) - } else { + case "": var actual = resultToString(t, decodeJSON(t, s.input)) test.AssertResultWithContext(t, s.expected, actual, s.description) + + default: + panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } @@ -135,14 +139,17 @@ func documentJSONDecodeScenario(t *testing.T, w *bufio.Writer, s formatScenario) func documentJSONScenario(t *testing.T, w *bufio.Writer, i interface{}) { s := i.(formatScenario) - if s.skipDoc { return } - if s.scenarioType == "encode" { - documentJSONEncodeScenario(w, s) - } else { + switch s.scenarioType { + case "": documentJSONDecodeScenario(t, w, s) + case "encode": + documentJSONEncodeScenario(w, s) + + default: + panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } diff --git a/pkg/yqlib/operator_encoder_decoder.go b/pkg/yqlib/operator_encoder_decoder.go index ac4fd73f83d..dabaf01a16b 100644 --- a/pkg/yqlib/operator_encoder_decoder.go +++ b/pkg/yqlib/operator_encoder_decoder.go @@ -15,7 +15,7 @@ func configureEncoder(format PrinterOutputFormat, indent int) Encoder { case JSONOutputFormat: return NewJONEncoder(indent, false) case PropsOutputFormat: - return NewPropertiesEncoder() + return NewPropertiesEncoder(true) case CSVOutputFormat: return NewCsvEncoder(',') case TSVOutputFormat: diff --git a/pkg/yqlib/properties_test.go b/pkg/yqlib/properties_test.go index 59e012026c5..3a59fcf3e15 100644 --- a/pkg/yqlib/properties_test.go +++ b/pkg/yqlib/properties_test.go @@ -8,9 +8,9 @@ import ( "github.com/mikefarah/yq/v4/test" ) -var samplePropertiesYaml = `# block comments don't come through +const samplePropertiesYaml = `# block comments don't come through person: # neither do comments on maps - name: Mike # comments on values appear + name: Mike Wazowski # comments on values appear pets: - cat # comments on array values appear food: [pizza] # comments on arrays do not @@ -18,37 +18,45 @@ emptyArray: [] emptyMap: [] ` -var expectedProperties = `# comments on values appear -person.name = Mike +const expectedPropertiesUnwrapped = `# comments on values appear +person.name = Mike Wazowski # comments on array values appear person.pets.0 = cat person.food.0 = pizza ` -var expectedUpdatedProperties = `# comments on values appear -person.name = Mike +const expectedPropertiesWrapped = `# comments on values appear +person.name = "Mike Wazowski" + +# comments on array values appear +person.pets.0 = cat +person.food.0 = pizza +` + +const expectedUpdatedProperties = `# comments on values appear +person.name = Mike Wazowski # comments on array values appear person.pets.0 = dog person.food.0 = pizza ` -var expectedDecodedYaml = `person: - name: Mike # comments on values appear +const expectedDecodedYaml = `person: + name: Mike Wazowski # comments on values appear pets: - cat # comments on array values appear food: - pizza ` -var expectedPropertiesNoComments = `person.name = Mike +const expectedPropertiesNoComments = `person.name = Mike Wazowski person.pets.0 = cat person.food.0 = pizza ` -var expectedPropertiesWithEmptyMapsAndArrays = `# comments on values appear -person.name = Mike +const expectedPropertiesWithEmptyMapsAndArrays = `# comments on values appear +person.name = Mike Wazowski # comments on array values appear person.pets.0 = cat @@ -62,7 +70,14 @@ var propertyScenarios = []formatScenario{ description: "Encode properties", subdescription: "Note that empty arrays and maps are not encoded by default.", input: samplePropertiesYaml, - expected: expectedProperties, + expected: expectedPropertiesUnwrapped, + }, + { + description: "Encode properties: scalar encapsulation", + subdescription: "Note that string values with blank characters in them are encapsulated with double quotes", + input: samplePropertiesYaml, + expected: expectedPropertiesWrapped, + scenarioType: "encode-wrapped", }, { description: "Encode properties: no comments", @@ -79,7 +94,7 @@ var propertyScenarios = []formatScenario{ }, { description: "Decode properties", - input: expectedProperties, + input: expectedPropertiesUnwrapped, expected: expectedDecodedYaml, scenarioType: "decode", }, @@ -92,7 +107,7 @@ var propertyScenarios = []formatScenario{ }, { description: "Roundtrip", - input: expectedProperties, + input: expectedPropertiesUnwrapped, expression: `.person.pets.0 = "dog"`, expected: expectedUpdatedProperties, scenarioType: "roundtrip", @@ -106,7 +121,7 @@ var propertyScenarios = []formatScenario{ }, } -func documentEncodePropertyScenario(w *bufio.Writer, s formatScenario) { +func documentUnwrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { @@ -128,7 +143,32 @@ func documentEncodePropertyScenario(w *bufio.Writer, s formatScenario) { } writeOrPanic(w, "will output\n") - writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder()))) + writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(true)))) +} + +func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) { + writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) + + if s.subdescription != "" { + writeOrPanic(w, s.subdescription) + writeOrPanic(w, "\n\n") + } + + writeOrPanic(w, "Given a sample.yml file of:\n") + writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) + + writeOrPanic(w, "then\n") + + expression := s.expression + + if expression != "" { + writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=props --unwrapScalar=false '%v' sample.yml\n```\n", expression)) + } else { + writeOrPanic(w, "```bash\nyq -o=props --unwrapScalar=false sample.yml\n```\n") + } + writeOrPanic(w, "will output\n") + + writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(false)))) } func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) { @@ -178,7 +218,7 @@ func documentRoundTripPropertyScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, "will output\n") - writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder()))) + writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true)))) } func documentPropertyScenario(t *testing.T, w *bufio.Writer, i interface{}) { @@ -186,24 +226,35 @@ func documentPropertyScenario(t *testing.T, w *bufio.Writer, i interface{}) { if s.skipDoc { return } - if s.scenarioType == "decode" { + switch s.scenarioType { + case "": + documentUnwrappedEncodePropertyScenario(w, s) + case "decode": documentDecodePropertyScenario(w, s) - } else if s.scenarioType == "roundtrip" { + case "encode-wrapped": + documentWrappedEncodePropertyScenario(w, s) + case "roundtrip": documentRoundTripPropertyScenario(w, s) - } else { - documentEncodePropertyScenario(w, s) - } + default: + panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) + } } func TestPropertyScenarios(t *testing.T) { for _, s := range propertyScenarios { - if s.scenarioType == "decode" { + switch s.scenarioType { + case "": + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(true)), s.description) + case "decode": test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(2, false, true, true)), s.description) - } else if s.scenarioType == "roundtrip" { - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder()), s.description) - } else { - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder()), s.description) + case "encode-wrapped": + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(false)), s.description) + case "roundtrip": + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true)), s.description) + + default: + panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } genericScenarios := make([]interface{}, len(propertyScenarios)) diff --git a/pkg/yqlib/xml_test.go b/pkg/yqlib/xml_test.go index 276f7006e2d..9058c084c50 100644 --- a/pkg/yqlib/xml_test.go +++ b/pkg/yqlib/xml_test.go @@ -8,7 +8,7 @@ import ( "github.com/mikefarah/yq/v4/test" ) -var inputXMLWithComments = ` +const inputXMLWithComments = ` @@ -26,7 +26,7 @@ for x --> ` -var inputXMLWithCommentsWithSubChild = ` +const inputXMLWithCommentsWithSubChild = ` @@ -45,7 +45,7 @@ for x --> ` -var expectedDecodeYamlWithSubChild = `# before cat +const expectedDecodeYamlWithSubChild = `# before cat cat: # in cat before x: "3" # multi @@ -65,7 +65,7 @@ cat: # after cat ` -var inputXMLWithCommentsWithArray = ` +const inputXMLWithCommentsWithArray = ` @@ -85,7 +85,7 @@ for x --> ` -var expectedDecodeYamlWithArray = `# before cat +const expectedDecodeYamlWithArray = `# before cat cat: # in cat before x: "3" # multi @@ -109,7 +109,7 @@ cat: # after cat ` -var expectedDecodeYamlWithComments = `# before cat +const expectedDecodeYamlWithComments = `# before cat cat: # in cat before x: "3" # multi @@ -126,7 +126,7 @@ cat: # after cat ` -var expectedRoundtripXMLWithComments = ` +const expectedRoundtripXMLWithComments = ` 3 @@ -137,7 +137,7 @@ in d before --> ` -var yamlWithComments = `# above_cat +const yamlWithComments = `# above_cat cat: # inline_cat # above_array array: # inline_array @@ -147,31 +147,31 @@ cat: # inline_cat # below_cat ` -var expectedXMLWithComments = ` +const expectedXMLWithComments = ` val1 val2 ` -var inputXMLWithNamespacedAttr = ` +const inputXMLWithNamespacedAttr = ` ` -var expectedYAMLWithNamespacedAttr = `map: +const expectedYAMLWithNamespacedAttr = `map: +xmlns: some-namespace +xmlns:xsi: some-instance +some-instance:schemaLocation: some-url ` -var expectedYAMLWithRawNamespacedAttr = `map: +const expectedYAMLWithRawNamespacedAttr = `map: +xmlns: some-namespace +xmlns:xsi: some-instance +xsi:schemaLocation: some-url ` -var xmlWithCustomDtd = ` +const xmlWithCustomDtd = ` @@ -181,7 +181,7 @@ var xmlWithCustomDtd = ` &writer;©right; ` -var expectedDtd = `root: +const expectedDtd = `root: item: '&writer;©right;' ` @@ -336,6 +336,8 @@ var xmlScenarios = []formatScenario{ func testXMLScenario(t *testing.T, s formatScenario) { switch s.scenarioType { + case "", "decode": + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false), NewYamlEncoder(4, false, true, true)), s.description) case "encode": test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewXMLEncoder(2, "+", "+content")), s.description) case "roundtrip": @@ -344,8 +346,9 @@ func testXMLScenario(t *testing.T, s formatScenario) { test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, false), NewYamlEncoder(2, false, true, true)), s.description) case "decode-raw-token": test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, true), NewYamlEncoder(2, false, true, true)), s.description) + default: - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false), NewYamlEncoder(4, false, true, true)), s.description) + panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } @@ -356,6 +359,8 @@ func documentXMLScenario(t *testing.T, w *bufio.Writer, i interface{}) { return } switch s.scenarioType { + case "", "decode": + documentXMLDecodeScenario(w, s) case "encode": documentXMLEncodeScenario(w, s) case "roundtrip": @@ -364,10 +369,10 @@ func documentXMLScenario(t *testing.T, w *bufio.Writer, i interface{}) { documentXMLDecodeKeepNsScenario(w, s) case "decode-raw-token": documentXMLDecodeKeepNsRawTokenScenario(w, s) + default: - documentXMLDecodeScenario(w, s) + panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } - } func documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) {