Skip to content

Commit

Permalink
Added XML decoder (#1044)
Browse files Browse the repository at this point in the history
  • Loading branch information
mikefarah authored Dec 21, 2021
1 parent 915e9de commit df32bae
Show file tree
Hide file tree
Showing 154 changed files with 725 additions and 113 deletions.
2 changes: 2 additions & 0 deletions LICENSE
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
The MIT License (MIT)

Copyright (c) 2017 Mike Farah

Permission is hereby granted, free of charge, to any person obtaining a copy
Expand Down
4 changes: 4 additions & 0 deletions cmd/constant.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ var unwrapScalar = true
var writeInplace = false
var outputToJSON = false
var outputFormat = "yaml"
var inputFormat = "yaml"

var xmlAttributePrefix = "+"
var xmlContentName = "+content"

var exitStatus = false
var forceColor = false
Expand Down
11 changes: 8 additions & 3 deletions cmd/evaluate_all_command.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,11 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
return err
}

decoder, err := configureDecoder()
if err != nil {
return err
}

printerWriter := configurePrinterWriter(format, out)

printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
Expand All @@ -99,7 +104,7 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
switch len(args) {
case 0:
if pipingStdIn {
err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing)
err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing, decoder)
} else {
cmd.Println(cmd.UsageString())
return nil
Expand All @@ -108,10 +113,10 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
if nullInput {
err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(args[0]), printer, "")
} else {
err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing)
err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing, decoder)
}
default:
err = allAtOnceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing)
err = allAtOnceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing, decoder)
}

completedSuccessfully = err == nil
Expand Down
11 changes: 8 additions & 3 deletions cmd/evalute_sequence_command.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,11 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {

printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)

decoder, err := configureDecoder()
if err != nil {
return err
}

streamEvaluator := yqlib.NewStreamEvaluator()

if frontMatter != "" {
Expand All @@ -113,7 +118,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
switch len(args) {
case 0:
if pipingStdIn {
err = streamEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing)
err = streamEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing, decoder)
} else {
cmd.Println(cmd.UsageString())
return nil
Expand All @@ -122,10 +127,10 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
if nullInput {
err = streamEvaluator.EvaluateNew(processExpression(args[0]), printer, "")
} else {
err = streamEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing)
err = streamEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing, decoder)
}
default:
err = streamEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing)
err = streamEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing, decoder)
}
completedSuccessfully = err == nil

Expand Down
5 changes: 5 additions & 0 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,11 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
}

rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p] output format type.")
rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "yaml", "[yaml|y|xml|x] input format type.")

rootCmd.PersistentFlags().StringVar(&xmlAttributePrefix, "xml-attribute-prefix", "+", "prefix for xml attributes")
rootCmd.PersistentFlags().StringVar(&xmlContentName, "xml-content-name", "+content", "name for xml content (if no attribute name is present).")

rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.")
rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")

Expand Down
12 changes: 12 additions & 0 deletions cmd/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,18 @@ func initCommand(cmd *cobra.Command, args []string) (firstFileIndex int, err err
return firstFileIndex, nil
}

func configureDecoder() (yqlib.Decoder, error) {
yqlibInputFormat, err := yqlib.InputFormatFromString(inputFormat)
if err != nil {
return nil, err
}
switch yqlibInputFormat {
case yqlib.XmlInputFormat:
return yqlib.NewXmlDecoder(xmlAttributePrefix, xmlContentName), nil
}
return yqlib.NewYamlDecoder(), nil
}

func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) yqlib.PrinterWriter {

var printerWriter yqlib.PrinterWriter
Expand Down
4 changes: 4 additions & 0 deletions examples/mike.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<cat>3f</cat>
<dog>meow:as</dog>
<dog3>true</dog3>
11 changes: 11 additions & 0 deletions examples/mike2.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- osm-->
<osm version="0.6" generator="CGImap 0.0.2">
<!-- bounds-->
<bounds minlat="54.0889580" minlon="12.2487570" maxlat="54.0913900" maxlon="12.2524800">
<!-- great -->
cool
</bounds>
<foo>ba2234r</foo>
<foo>bar2234233</foo>
</osm>
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ require (
github.com/magiconair/properties v1.8.5
github.com/spf13/cobra v1.3.0
github.com/timtadh/lexmachine v0.2.2
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
)
Expand All @@ -19,6 +20,7 @@ require (
github.com/spf13/pflag v1.0.5 // indirect
github.com/timtadh/data-structures v0.5.3 // indirect
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
)

Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,7 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c=
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand Down Expand Up @@ -553,6 +554,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down
6 changes: 3 additions & 3 deletions pkg/yqlib/all_at_once_evaluator.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (

// A yaml expression evaluator that runs the expression once against all files/nodes in memory.
type Evaluator interface {
EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error
EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error

// EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes
EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error)
Expand Down Expand Up @@ -46,7 +46,7 @@ func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCand
return context.MatchingNodes, nil
}

func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error {
func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error {
fileIndex := 0
firstFileLeadingContent := ""

Expand All @@ -61,7 +61,7 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
firstFileLeadingContent = leadingContent
}

fileDocuments, err := readDocuments(reader, filename, fileIndex)
fileDocuments, err := readDocuments(reader, filename, fileIndex, decoder)
if err != nil {
return err
}
Expand Down
110 changes: 110 additions & 0 deletions pkg/yqlib/decode_xml_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package yqlib

import (
"bufio"
"bytes"
"fmt"
"strings"
"testing"

"github.com/mikefarah/yq/v4/test"
yaml "gopkg.in/yaml.v3"
)

func decodeXml(t *testing.T, xml string) *CandidateNode {
decoder := NewXmlDecoder("+", "+content")

decoder.Init(strings.NewReader(xml))

node := &yaml.Node{}
err := decoder.Decode(node)
if err != nil {
t.Error(err, "fail to decode", xml)
}
return &CandidateNode{Node: node}
}

type xmlScenario struct {
inputXml string
expected string
description string
subdescription string
skipDoc bool
}

var xmlScenarios = []xmlScenario{
{
description: "Parse xml: simple",
inputXml: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>meow</cat>",
expected: "D0, P[], (doc)::cat: meow\n",
},
{
description: "Parse xml: array",
subdescription: "Consecutive nodes with identical xml names are assumed to be arrays.",
inputXml: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<animal>1</animal>\n<animal>2</animal>",
expected: "D0, P[], (doc)::animal:\n - \"1\"\n - \"2\"\n",
},
{
description: "Parse xml: attributes",
subdescription: "Attributes are converted to fields, with the attribute prefix.",
inputXml: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">\n <legs>7</legs>\n</cat>",
expected: "D0, P[], (doc)::cat:\n +legs: \"4\"\n legs: \"7\"\n",
},
{
description: "Parse xml: attributes with content",
subdescription: "Content is added as a field, using the content name",
inputXml: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">meow</cat>",
expected: "D0, P[], (doc)::cat:\n +content: meow\n +legs: \"4\"\n",
},
}

func testXmlScenario(t *testing.T, s *xmlScenario) {
var actual = resultToString(t, decodeXml(t, s.inputXml))
test.AssertResult(t, s.expected, actual)
}

func documentXmlScenario(t *testing.T, w *bufio.Writer, i interface{}) {
s := i.(xmlScenario)

if s.skipDoc {
return
}
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))

if s.subdescription != "" {
writeOrPanic(w, s.subdescription)
writeOrPanic(w, "\n\n")
}

writeOrPanic(w, "Given a sample.xml file of:\n")
writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.inputXml))

writeOrPanic(w, "then\n")
writeOrPanic(w, "```bash\nyq e sample.xml\n```\n")
writeOrPanic(w, "will output\n")

var output bytes.Buffer
printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)

node := decodeXml(t, s.inputXml)

err := printer.PrintResults(node.AsList())
if err != nil {
t.Error(err)
return
}

writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", output.String()))

}

func TestXmlScenarios(t *testing.T) {
for _, tt := range xmlScenarios {
testXmlScenario(t, &tt)
}
genericScenarios := make([]interface{}, len(xmlScenarios))
for i, s := range xmlScenarios {
genericScenarios[i] = s
}
documentScenarios(t, "usage", "xml", genericScenarios, documentXmlScenario)
}
Loading

0 comments on commit df32bae

Please sign in to comment.