From f5b2ca7247b14c0c2865c18d0f759c4fbfdd1076 Mon Sep 17 00:00:00 2001 From: Charlie Egan Date: Thu, 19 Jan 2023 14:23:24 +0000 Subject: [PATCH 1/8] Implement configurable marshalling for ast data Part of: https://github.com/open-policy-agent/opa/issues/3143 This is being done so that the json representation of an ast parse can be used more easily by automated tools which need the location information. This location information is verbose for human users of the data and has until now been excluded from JSON representation. This PR makes it possible to configure the JSON marshalling with some options. However, the functionality of the OPA parse command has been left mostly unchanged. The only with the json output is that the fields are in a different order: ``` diff --git a/new.json b/old.json index 5b9504fa..0fb9d4f2 100644 --- a/new.json +++ b/old.json @@ -13,15 +13,6 @@ }, "rules": [ { - "body": [ - { - "index": 0, - "terms": { - "type": "boolean", - "value": true - } - } - ], "head": { "name": "allow", "value": { @@ -34,17 +25,26 @@ "value": "allow" } ] - } + }, + "body": [ + { + "terms": { + "type": "boolean", + "value": true + }, + "index": 0 + } + ] } ], "comments": [ { + "Text": "IGNvbW1lbnQ=", "Location": { "file": "main.rego", "row": 3, "col": 1 - }, - "Text": "IGNvbW1lbnQ=" + } } ] } ``` It's also possible to now show locations for all terms with another flag option. ``` $ go run main.go parse main.rego --format=json --json-include=locations { "package": { "location": { "file": "main.rego", "row": 1, "col": 1 }, "path": [ { "location": { "file": "main.rego", "row": 1, "col": 9 }, "type": "var", "value": "data" }, { "location": { "file": "main.rego", "row": 1, "col": 9 }, "type": "string", "value": "foo" } ] }, "rules": [ { "body": [ { "index": 0, "terms": { "type": "boolean", "value": true } } ], "head": { "name": "allow", "value": { "location": { "file": "main.rego", "row": 5, "col": 9 }, "type": "boolean", "value": true }, "ref": [ { "type": "var", "value": "allow" } ] } } ], "comments": [ { "Location": { "file": "main.rego", "row": 3, "col": 1 }, "Text": "IGNvbW1lbnQ=" } ] } ``` Signed-off-by: Charlie Egan --- ast/annotations.go | 91 ++- ast/annotations_test.go | 8 +- ast/marshal.go | 8 + ast/marshal_test.go | 1329 +++++++++++++++++++++++++++++++++++++++ ast/parser.go | 19 +- ast/parser_ext.go | 1 + ast/policy.go | 233 ++++++- ast/policy_test.go | 17 +- ast/term.go | 84 ++- cmd/parse.go | 46 +- 10 files changed, 1789 insertions(+), 47 deletions(-) create mode 100644 ast/marshal.go create mode 100644 ast/marshal_test.go diff --git a/ast/annotations.go b/ast/annotations.go index 8ca96d5056..2eb712e139 100644 --- a/ast/annotations.go +++ b/ast/annotations.go @@ -26,7 +26,6 @@ const ( type ( // Annotations represents metadata attached to other AST nodes such as rules. Annotations struct { - Location *Location `json:"-"` Scope string `json:"scope"` Title string `json:"title,omitempty"` Entrypoint bool `json:"entrypoint,omitempty"` @@ -36,8 +35,11 @@ type ( Authors []*AuthorAnnotation `json:"authors,omitempty"` Schemas []*SchemaAnnotation `json:"schemas,omitempty"` Custom map[string]interface{} `json:"custom,omitempty"` - node Node - comments []*Comment + Location *Location `json:"location,omitempty"` + + comments []*Comment + node Node + jsonFields map[string]bool } // SchemaAnnotation contains a schema declaration for the document identified by the path. @@ -70,10 +72,13 @@ type ( } AnnotationsRef struct { - Location *Location `json:"location"` // The location of the node the annotations are applied to - Path Ref `json:"path"` // The path of the node the annotations are applied to + Path Ref `json:"path"` // The path of the node the annotations are applied to Annotations *Annotations `json:"annotations,omitempty"` - node Node // The node the annotations are applied to + Location *Location `json:"location,omitempty"` // The location of the node the annotations are applied to + + jsonFields map[string]bool + + node Node // The node the annotations are applied to } AnnotationsRefSet []*AnnotationsRef @@ -82,7 +87,7 @@ type ( ) func (a *Annotations) String() string { - bs, _ := json.Marshal(a) + bs, _ := a.MarshalJSON() return string(bs) } @@ -175,6 +180,60 @@ func (a *Annotations) GetTargetPath() Ref { } } +func (a *Annotations) exposeJSONFields(jsonFields map[string]bool) { + a.jsonFields = jsonFields +} + +func (a *Annotations) MarshalJSON() ([]byte, error) { + if a == nil { + return []byte(`{"scope":""}`), nil + } + + data := map[string]interface{}{ + "scope": a.Scope, + } + + if a.Title != "" { + data["title"] = a.Title + } + + if a.Description != "" { + data["description"] = a.Description + } + + if a.Entrypoint { + data["entrypoint"] = a.Entrypoint + } + + if len(a.Organizations) > 0 { + data["organizations"] = a.Organizations + } + + if len(a.RelatedResources) > 0 { + data["related_resources"] = a.RelatedResources + } + + if len(a.Authors) > 0 { + data["authors"] = a.Authors + } + + if len(a.Schemas) > 0 { + data["schemas"] = a.Schemas + } + + if len(a.Custom) > 0 { + data["custom"] = a.Custom + } + + if showLocation, ok := a.jsonFields["location"]; ok && showLocation { + if a.Location != nil { + data["location"] = a.Location + } + } + + return json.Marshal(data) +} + func NewAnnotationsRef(a *Annotations) *AnnotationsRef { var loc *Location if a.node != nil { @@ -209,6 +268,24 @@ func (ar *AnnotationsRef) GetRule() *Rule { } } +func (ar *AnnotationsRef) MarshalJSON() ([]byte, error) { + data := map[string]interface{}{ + "path": ar.Path, + } + + if ar.Annotations != nil { + data["annotations"] = ar.Annotations + } + + if showLocation, ok := ar.jsonFields["location"]; ok && showLocation { + if ar.Location != nil { + data["location"] = ar.Location + } + } + + return json.Marshal(data) +} + func scopeCompare(s1, s2 string) int { o1 := scopeOrder(s1) diff --git a/ast/annotations_test.go b/ast/annotations_test.go index 6627befa69..05563d5197 100644 --- a/ast/annotations_test.go +++ b/ast/annotations_test.go @@ -52,8 +52,8 @@ p := 7`}, } // Output: - // data.foo at foo.rego:5 has annotations {"scope":"subpackages","organizations":["Acme Corp."]} - // data.foo.bar at mod:3 has annotations {"scope":"package","description":"A couple of useful rules"} + // data.foo at foo.rego:5 has annotations {"organizations":["Acme Corp."],"scope":"subpackages"} + // data.foo.bar at mod:3 has annotations {"description":"A couple of useful rules","scope":"package"} // data.foo.bar.p at mod:7 has annotations {"scope":"rule","title":"My Rule P"} } @@ -102,8 +102,8 @@ p := 7`}, // Output: // data.foo.bar.p at mod:7 has annotations {"scope":"rule","title":"My Rule P"} - // data.foo.bar at mod:3 has annotations {"scope":"package","description":"A couple of useful rules"} - // data.foo at foo.rego:5 has annotations {"scope":"subpackages","organizations":["Acme Corp."]} + // data.foo.bar at mod:3 has annotations {"description":"A couple of useful rules","scope":"package"} + // data.foo at foo.rego:5 has annotations {"organizations":["Acme Corp."],"scope":"subpackages"} } func TestAnnotationSet_Flatten(t *testing.T) { diff --git a/ast/marshal.go b/ast/marshal.go new file mode 100644 index 0000000000..65b323154f --- /dev/null +++ b/ast/marshal.go @@ -0,0 +1,8 @@ +package ast + +// customJSON is an interface that can be implemented by AST nodes that +// allows the parser to set a list of fields and if the field is to be +// included in the JSON output. +type customJSON interface { + exposeJSONFields(map[string]bool) +} diff --git a/ast/marshal_test.go b/ast/marshal_test.go new file mode 100644 index 0000000000..7486ea3a98 --- /dev/null +++ b/ast/marshal_test.go @@ -0,0 +1,1329 @@ +package ast + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/open-policy-agent/opa/util" +) + +func TestTerm_MarshalJSON(t *testing.T) { + testCases := map[string]struct { + Term *Term + ExpectedJSON string + }{ + "base case": { + Term: func() *Term { + v, _ := InterfaceToValue("example") + return &Term{ + Value: v, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + } + }(), + ExpectedJSON: `{"type":"string","value":"example"}`, + }, + "location excluded": { + Term: func() *Term { + v, _ := InterfaceToValue("example") + return &Term{ + Value: v, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + jsonFields: map[string]bool{ + "location": false, + }, + } + }(), + ExpectedJSON: `{"type":"string","value":"example"}`, + }, + "location included": { + Term: func() *Term { + v, _ := InterfaceToValue("example") + return &Term{ + Value: v, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + jsonFields: map[string]bool{ + "location": true, + }, + } + }(), + ExpectedJSON: `{"location":{"file":"example.rego","row":1,"col":2},"type":"string","value":"example"}`, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + bs := util.MustMarshalJSON(data.Term) + got := string(bs) + exp := data.ExpectedJSON + + if got != exp { + t.Fatalf("expected:\n%s got\n%s", exp, got) + } + }) + } +} + +func TestTerm_UnmarshalJSON(t *testing.T) { + testCases := map[string]struct { + JSON string + ExpectedTerm *Term + }{ + "base case": { + JSON: `{"type":"string","value":"example"}`, + ExpectedTerm: func() *Term { + v, _ := InterfaceToValue("example") + return &Term{ + Value: v, + } + }(), + }, + "location case": { + JSON: `{"location":{"file":"example.rego","row":1,"col":2},"type":"string","value":"example"}`, + ExpectedTerm: func() *Term { + v, _ := InterfaceToValue("example") + return &Term{ + Value: v, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + } + }(), + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + var term Term + err := json.Unmarshal([]byte(data.JSON), &term) + if err != nil { + t.Fatal(err) + } + + if !term.Equal(data.ExpectedTerm) { + t.Fatalf("expected:\n%#v got\n%#v", data.ExpectedTerm, term) + } + if data.ExpectedTerm.Location != nil { + if !term.Location.Equal(data.ExpectedTerm.Location) { + t.Fatalf("expected location:\n%#v got\n%#v", data.ExpectedTerm, term) + } + } + }) + } +} + +func TestPackage_MarshalJSON(t *testing.T) { + testCases := map[string]struct { + Package *Package + ExpectedJSON string + }{ + "base case": { + Package: &Package{ + Path: EmptyRef(), + }, + ExpectedJSON: `{"path":[]}`, + }, + "location excluded": { + Package: &Package{ + Path: EmptyRef(), + Location: NewLocation([]byte{}, "example.rego", 1, 2), + jsonFields: map[string]bool{ + "location": false, + }, + }, + ExpectedJSON: `{"path":[]}`, + }, + "location included": { + Package: &Package{ + Path: EmptyRef(), + Location: NewLocation([]byte{}, "example.rego", 1, 2), + jsonFields: map[string]bool{ + "location": true, + }, + }, + ExpectedJSON: `{"location":{"file":"example.rego","row":1,"col":2},"path":[]}`, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + bs := util.MustMarshalJSON(data.Package) + got := string(bs) + exp := data.ExpectedJSON + + if got != exp { + t.Fatalf("expected:\n%s got\n%s", exp, got) + } + }) + } +} + +func TestPackage_UnmarshalJSON(t *testing.T) { + testCases := map[string]struct { + JSON string + ExpectedPackage *Package + }{ + "base case": { + JSON: `{"path":[]}`, + ExpectedPackage: &Package{ + Path: EmptyRef(), + }, + }, + "location case": { + JSON: `{"location":{"file":"example.rego","row":1,"col":2},"path":[]}`, + ExpectedPackage: &Package{ + Path: EmptyRef(), + Location: NewLocation([]byte{}, "example.rego", 1, 2), + }, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + var pkg Package + err := json.Unmarshal([]byte(data.JSON), &pkg) + if err != nil { + t.Fatal(err) + } + + if !pkg.Equal(data.ExpectedPackage) { + t.Fatalf("expected:\n%#v got\n%#v", data.ExpectedPackage, pkg) + } + if data.ExpectedPackage.Location != nil { + if !pkg.Location.Equal(data.ExpectedPackage.Location) { + t.Fatalf("expected location:\n%#v got\n%#v", data.ExpectedPackage, pkg) + } + } + }) + } +} + +// TODO: Comment has inconsistent JSON field names starting with an upper case letter. Comment Location is +// also always included for legacy reasons +func TestComment_MarshalJSON(t *testing.T) { + testCases := map[string]struct { + Comment *Comment + ExpectedJSON string + }{ + "base case": { + Comment: &Comment{ + Text: []byte("comment"), + }, + ExpectedJSON: `{"Text":"Y29tbWVudA=="}`, + }, + "location excluded, still included for legacy reasons": { + Comment: &Comment{ + Text: []byte("comment"), + Location: NewLocation([]byte{}, "example.rego", 1, 2), + jsonFields: map[string]bool{ + "location": false, // ignored + }, + }, + ExpectedJSON: `{"Location":{"file":"example.rego","row":1,"col":2},"Text":"Y29tbWVudA=="}`, + }, + "location included": { + Comment: &Comment{ + Text: []byte("comment"), + Location: NewLocation([]byte{}, "example.rego", 1, 2), + jsonFields: map[string]bool{ + "location": true, // ignored + }, + }, + ExpectedJSON: `{"Location":{"file":"example.rego","row":1,"col":2},"Text":"Y29tbWVudA=="}`, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + bs := util.MustMarshalJSON(data.Comment) + got := string(bs) + exp := data.ExpectedJSON + + if got != exp { + t.Fatalf("expected:\n%s got\n%s", exp, got) + } + }) + } +} + +// TODO: Comment has inconsistent JSON field names starting with an upper case letter. Comment Location is +// also always included for legacy reasons +func TestComment_UnmarshalJSON(t *testing.T) { + testCases := map[string]struct { + JSON string + ExpectedComment *Comment + }{ + "base case": { + JSON: `{"Text":"Y29tbWVudA=="}`, + ExpectedComment: &Comment{ + Text: []byte("comment"), + }, + }, + "location case": { + JSON: `{"Location":{"file":"example.rego","row":1,"col":2},"Text":"Y29tbWVudA=="}`, + ExpectedComment: &Comment{ + Text: []byte("comment"), + Location: NewLocation([]byte{}, "example.rego", 1, 2), + }, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + var comment Comment + err := json.Unmarshal([]byte(data.JSON), &comment) + if err != nil { + t.Fatal(err) + } + + equal := true + if data.ExpectedComment.Location != nil { + if comment.Location == nil { + t.Fatal("expected location to be non-nil") + } + + // comment.Equal will check the location too + if !comment.Equal(data.ExpectedComment) { + equal = false + } + } else { + if !bytes.Equal(comment.Text, data.ExpectedComment.Text) { + equal = false + } + } + if !equal { + t.Fatalf("expected:\n%#v got\n%#v", data.ExpectedComment, comment) + } + }) + } +} + +func TestImport_MarshalJSON(t *testing.T) { + testCases := map[string]struct { + Import *Import + ExpectedJSON string + }{ + "base case": { + Import: func() *Import { + v, _ := InterfaceToValue("example") + term := Term{ + Value: v, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + } + return &Import{Path: &term} + }(), + ExpectedJSON: `{"path":{"type":"string","value":"example"}}`, + }, + "location excluded": { + Import: func() *Import { + v, _ := InterfaceToValue("example") + term := Term{ + Value: v, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + } + return &Import{ + Path: &term, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + jsonFields: map[string]bool{ + "location": false, + }, + } + }(), + ExpectedJSON: `{"path":{"type":"string","value":"example"}}`, + }, + "location included": { + Import: func() *Import { + v, _ := InterfaceToValue("example") + term := Term{ + Value: v, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + } + return &Import{ + Path: &term, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + jsonFields: map[string]bool{ + "location": true, + }, + } + }(), + ExpectedJSON: `{"location":{"file":"example.rego","row":1,"col":2},"path":{"type":"string","value":"example"}}`, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + bs := util.MustMarshalJSON(data.Import) + got := string(bs) + exp := data.ExpectedJSON + + if got != exp { + t.Fatalf("expected:\n%s got\n%s", exp, got) + } + }) + } +} + +func TestImport_UnmarshalJSON(t *testing.T) { + testCases := map[string]struct { + JSON string + ExpectedImport *Import + }{ + "base case": { + JSON: `{"path":{"type":"string","value":"example"}}`, + ExpectedImport: func() *Import { + v, _ := InterfaceToValue("example") + term := Term{ + Value: v, + } + return &Import{Path: &term} + }(), + }, + "location case": { + JSON: `{"location":{"file":"example.rego","row":1,"col":2},"path":{"type":"string","value":"example"}}`, + ExpectedImport: func() *Import { + v, _ := InterfaceToValue("example") + term := Term{ + Value: v, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + } + return &Import{ + Path: &term, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + jsonFields: map[string]bool{ + "location": false, + }, + } + }(), + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + var imp Import + err := json.Unmarshal([]byte(data.JSON), &imp) + if err != nil { + t.Fatal(err) + } + + if !imp.Equal(data.ExpectedImport) { + t.Fatalf("expected:\n%#v got\n%#v", data.ExpectedImport, imp) + } + if data.ExpectedImport.Location != nil { + if !imp.Location.Equal(data.ExpectedImport.Location) { + t.Fatalf("expected location:\n%#v got\n%#v", data.ExpectedImport, imp) + } + } + }) + } +} + +func TestRule_MarshalJSON(t *testing.T) { + rawModule := ` + package foo + + # comment + + allow { true } + ` + + module, err := ParseModuleWithOpts("example.rego", rawModule, ParserOptions{}) + if err != nil { + t.Fatal(err) + } + + rule := module.Rules[0] + + testCases := map[string]struct { + Rule *Rule + ExpectedJSON string + }{ + "base case": { + Rule: rule.Copy(), + ExpectedJSON: `{"body":[{"index":0,"terms":{"type":"boolean","value":true}}],"head":{"name":"allow","value":{"type":"boolean","value":true},"ref":[{"type":"var","value":"allow"}]}}`, + }, + "location excluded": { + Rule: func() *Rule { + r := rule.Copy() + r.jsonFields = map[string]bool{ + "location": false, + } + return r + }(), + ExpectedJSON: `{"body":[{"index":0,"terms":{"type":"boolean","value":true}}],"head":{"name":"allow","value":{"type":"boolean","value":true},"ref":[{"type":"var","value":"allow"}]}}`, + }, + "location included": { + Rule: func() *Rule { + r := rule.Copy() + r.jsonFields = map[string]bool{ + "location": true, + } + return r + }(), + ExpectedJSON: `{"body":[{"index":0,"terms":{"type":"boolean","value":true}}],"head":{"name":"allow","value":{"type":"boolean","value":true},"ref":[{"type":"var","value":"allow"}]},"location":{"file":"example.rego","row":6,"col":2}}`, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + bs := util.MustMarshalJSON(data.Rule) + got := string(bs) + exp := data.ExpectedJSON + + if got != exp { + t.Fatalf("expected:\n%s got\n%s", exp, got) + } + }) + } +} + +func TestRule_UnmarshalJSON(t *testing.T) { + rawModule := ` + package foo + + # comment + + allow { true } + ` + + module, err := ParseModuleWithOpts("example.rego", rawModule, ParserOptions{}) + if err != nil { + t.Fatal(err) + } + + rule := module.Rules[0] + // text is not marshalled to JSON so we just drop it in our examples + rule.Location.Text = nil + + testCases := map[string]struct { + JSON string + ExpectedRule *Rule + }{ + "base case": { + JSON: `{"body":[{"terms":{"type":"boolean","value":true},"location":{"file":"example.rego","row":6,"col":10},"index":0}],"head":{"name":"allow","value":{"type":"boolean","value":true},"location":{"file":"example.rego","row":6,"col":2},"ref":[{"type":"var","value":"allow"}]}}`, + ExpectedRule: func() *Rule { + r := rule.Copy() + r.Location = nil + return r + }(), + }, + "location case": { + JSON: `{"body":[{"terms":{"type":"boolean","value":true},"location":{"file":"example.rego","row":6,"col":10},"index":0}],"head":{"name":"allow","value":{"type":"boolean","value":true},"location":{"file":"example.rego","row":6,"col":2},"ref":[{"type":"var","value":"allow"}]},"location":{"file":"example.rego","row":6,"col":2}}`, + ExpectedRule: rule, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + var rule Rule + err := json.Unmarshal([]byte(data.JSON), &rule) + if err != nil { + t.Fatal(err) + } + + if !rule.Equal(data.ExpectedRule) { + t.Fatalf("expected:\n%#v got\n%#v", data.ExpectedRule, rule) + } + if data.ExpectedRule.Location != nil { + if !rule.Location.Equal(data.ExpectedRule.Location) { + t.Fatalf("expected location:\n%#v got\n%#v", data.ExpectedRule.Location, rule.Location) + } + } + }) + } +} + +func TestHead_MarshalJSON(t *testing.T) { + rawModule := ` + package foo + + # comment + + allow { true } + ` + + module, err := ParseModuleWithOpts("example.rego", rawModule, ParserOptions{}) + if err != nil { + t.Fatal(err) + } + + head := module.Rules[0].Head + + testCases := map[string]struct { + Head *Head + ExpectedJSON string + }{ + "base case": { + Head: head.Copy(), + ExpectedJSON: `{"name":"allow","value":{"type":"boolean","value":true},"ref":[{"type":"var","value":"allow"}]}`, + }, + "location excluded": { + Head: func() *Head { + h := head.Copy() + h.jsonFields = map[string]bool{ + "location": false, + } + return h + }(), + ExpectedJSON: `{"name":"allow","value":{"type":"boolean","value":true},"ref":[{"type":"var","value":"allow"}]}`, + }, + "location included": { + Head: func() *Head { + h := head.Copy() + h.jsonFields = map[string]bool{ + "location": true, + } + return h + }(), + ExpectedJSON: `{"name":"allow","value":{"type":"boolean","value":true},"ref":[{"type":"var","value":"allow"}],"location":{"file":"example.rego","row":6,"col":2}}`, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + bs := util.MustMarshalJSON(data.Head) + got := string(bs) + exp := data.ExpectedJSON + + if got != exp { + t.Fatalf("expected:\n%s got\n%s", exp, got) + } + }) + } +} + +func TestHead_UnmarshalJSON(t *testing.T) { + rawModule := ` + package foo + + # comment + + allow { true } + ` + + module, err := ParseModuleWithOpts("example.rego", rawModule, ParserOptions{}) + if err != nil { + t.Fatal(err) + } + + head := module.Rules[0].Head + // text is not marshalled to JSON so we just drop it in our examples + head.Location.Text = nil + + testCases := map[string]struct { + JSON string + ExpectedHead *Head + }{ + "base case": { + JSON: `{"name":"allow","value":{"type":"boolean","value":true},"ref":[{"type":"var","value":"allow"}]}`, + ExpectedHead: func() *Head { + h := head.Copy() + h.Location = nil + return h + }(), + }, + "location case": { + JSON: `{"name":"allow","value":{"type":"boolean","value":true},"ref":[{"type":"var","value":"allow"}],"location":{"file":"example.rego","row":6,"col":2}}`, + ExpectedHead: head, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + var head Head + err := json.Unmarshal([]byte(data.JSON), &head) + if err != nil { + t.Fatal(err) + } + + if !head.Equal(data.ExpectedHead) { + t.Fatalf("expected:\n%#v got\n%#v", data.ExpectedHead, head) + } + if data.ExpectedHead.Location != nil { + if !head.Location.Equal(data.ExpectedHead.Location) { + t.Fatalf("expected location:\n%#v got\n%#v", data.ExpectedHead.Location, head.Location) + } + } + }) + } +} + +func TestExpr_MarshalJSON(t *testing.T) { + rawModule := ` + package foo + + # comment + + allow { true } + ` + + module, err := ParseModuleWithOpts("example.rego", rawModule, ParserOptions{}) + if err != nil { + t.Fatal(err) + } + + expr := module.Rules[0].Body[0] + + testCases := map[string]struct { + Expr *Expr + ExpectedJSON string + }{ + "base case": { + Expr: expr.Copy(), + ExpectedJSON: `{"index":0,"terms":{"type":"boolean","value":true}}`, + }, + "location excluded": { + Expr: func() *Expr { + e := expr.Copy() + e.jsonFields = map[string]bool{ + "location": false, + } + return e + }(), + ExpectedJSON: `{"index":0,"terms":{"type":"boolean","value":true}}`, + }, + "location included": { + Expr: func() *Expr { + e := expr.Copy() + e.jsonFields = map[string]bool{ + "location": true, + } + return e + }(), + ExpectedJSON: `{"index":0,"location":{"file":"example.rego","row":6,"col":10},"terms":{"type":"boolean","value":true}}`, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + bs := util.MustMarshalJSON(data.Expr) + got := string(bs) + exp := data.ExpectedJSON + + if got != exp { + t.Fatalf("expected:\n%s got\n%s", exp, got) + } + }) + } +} + +func TestExpr_UnmarshalJSON(t *testing.T) { + rawModule := ` + package foo + + # comment + + allow { true } + ` + + module, err := ParseModuleWithOpts("example.rego", rawModule, ParserOptions{}) + if err != nil { + t.Fatal(err) + } + + expr := module.Rules[0].Body[0] + // text is not marshalled to JSON so we just drop it in our examples + expr.Location.Text = nil + + testCases := map[string]struct { + JSON string + ExpectedExpr *Expr + }{ + "base case": { + JSON: `{"index":0,"terms":{"type":"boolean","value":true}}`, + ExpectedExpr: func() *Expr { + e := expr.Copy() + e.Location = nil + return e + }(), + }, + "location case": { + JSON: `{"index":0,"location":{"file":"example.rego","row":6,"col":10},"terms":{"type":"boolean","value":true}}`, + ExpectedExpr: expr, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + var expr Expr + err := json.Unmarshal([]byte(data.JSON), &expr) + if err != nil { + t.Fatal(err) + } + + if !expr.Equal(data.ExpectedExpr) { + t.Fatalf("expected:\n%#v got\n%#v", data.ExpectedExpr, expr) + } + if data.ExpectedExpr.Location != nil { + if !expr.Location.Equal(data.ExpectedExpr.Location) { + t.Fatalf("expected location:\n%#v got\n%#v", data.ExpectedExpr.Location, expr.Location) + } + } + }) + } +} + +func TestSomeDecl_MarshalJSON(t *testing.T) { + v, _ := InterfaceToValue("example") + term := &Term{ + Value: v, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + } + + testCases := map[string]struct { + SomeDecl *SomeDecl + ExpectedJSON string + }{ + "base case": { + SomeDecl: &SomeDecl{ + Symbols: []*Term{term}, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + }, + ExpectedJSON: `{"symbols":[{"type":"string","value":"example"}]}`, + }, + "location excluded": { + SomeDecl: &SomeDecl{ + Symbols: []*Term{term}, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + jsonFields: map[string]bool{"location": false}, + }, + ExpectedJSON: `{"symbols":[{"type":"string","value":"example"}]}`, + }, + "location included": { + SomeDecl: &SomeDecl{ + Symbols: []*Term{term}, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + jsonFields: map[string]bool{"location": true}, + }, + ExpectedJSON: `{"location":{"file":"example.rego","row":1,"col":2},"symbols":[{"type":"string","value":"example"}]}`, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + bs := util.MustMarshalJSON(data.SomeDecl) + got := string(bs) + exp := data.ExpectedJSON + + if got != exp { + t.Fatalf("expected:\n%s got\n%s", exp, got) + } + }) + } +} + +func TestSomeDecl_UnmarshalJSON(t *testing.T) { + v, _ := InterfaceToValue("example") + term := &Term{ + Value: v, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + } + + testCases := map[string]struct { + JSON string + ExpectedSomeDecl *SomeDecl + }{ + "base case": { + JSON: `{"symbols":[{"type":"string","value":"example"}]}`, + ExpectedSomeDecl: &SomeDecl{ + Symbols: []*Term{term}, + }, + }, + "location case": { + JSON: `{"location":{"file":"example.rego","row":1,"col":2},"symbols":[{"type":"string","value":"example"}]}`, + ExpectedSomeDecl: &SomeDecl{ + Symbols: []*Term{term}, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + }, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + var d SomeDecl + err := json.Unmarshal([]byte(data.JSON), &d) + if err != nil { + t.Fatal(err) + } + + if len(d.Symbols) != len(data.ExpectedSomeDecl.Symbols) { + t.Fatalf("expected:\n%#v got\n%#v", data.ExpectedSomeDecl.Symbols, d.Symbols) + } + + if data.ExpectedSomeDecl.Location != nil { + if !d.Location.Equal(data.ExpectedSomeDecl.Location) { + t.Fatalf("expected location:\n%#v got\n%#v", data.ExpectedSomeDecl.Location, d.Location) + } + } + }) + } +} + +func TestEvery_MarshalJSON(t *testing.T) { + + rawModule := ` +package foo + +import future.keywords.every + +allow { + every e in [1,2,3] { + e == 1 + } +} +` + + module, err := ParseModuleWithOpts("example.rego", rawModule, ParserOptions{}) + if err != nil { + t.Fatal(err) + } + + every, ok := module.Rules[0].Body[0].Terms.(*Every) + if !ok { + t.Fatal("expected every term") + } + + testCases := map[string]struct { + Every *Every + ExpectedJSON string + }{ + "base case": { + Every: every, + ExpectedJSON: `{"body":[{"index":0,"terms":[{"type":"ref","value":[{"type":"var","value":"equal"}]},{"type":"var","value":"e"},{"type":"number","value":1}]}],"domain":{"type":"array","value":[{"type":"number","value":1},{"type":"number","value":2},{"type":"number","value":3}]},"key":null,"value":{"type":"var","value":"e"}}`, + }, + "location excluded": { + Every: func() *Every { + e := every.Copy() + e.jsonFields = map[string]bool{"location": false} + return e + }(), + ExpectedJSON: `{"body":[{"index":0,"terms":[{"type":"ref","value":[{"type":"var","value":"equal"}]},{"type":"var","value":"e"},{"type":"number","value":1}]}],"domain":{"type":"array","value":[{"type":"number","value":1},{"type":"number","value":2},{"type":"number","value":3}]},"key":null,"value":{"type":"var","value":"e"}}`, + }, + "location included": { + Every: func() *Every { + e := every.Copy() + e.jsonFields = map[string]bool{"location": true} + return e + }(), + ExpectedJSON: `{"body":[{"index":0,"terms":[{"type":"ref","value":[{"type":"var","value":"equal"}]},{"type":"var","value":"e"},{"type":"number","value":1}]}],"domain":{"type":"array","value":[{"type":"number","value":1},{"type":"number","value":2},{"type":"number","value":3}]},"key":null,"location":{"file":"example.rego","row":7,"col":2},"value":{"type":"var","value":"e"}}`, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + bs := util.MustMarshalJSON(data.Every) + got := string(bs) + exp := data.ExpectedJSON + + if got != exp { + t.Fatalf("expected:\n%s got\n%s", exp, got) + } + }) + } +} + +func TestEvery_UnmarshalJSON(t *testing.T) { + rawModule := ` +package foo + +import future.keywords.every + +allow { + every e in [1,2,3] { + e == 1 + } +} +` + + module, err := ParseModuleWithOpts("example.rego", rawModule, ParserOptions{}) + if err != nil { + t.Fatal(err) + } + + every, ok := module.Rules[0].Body[0].Terms.(*Every) + if !ok { + t.Fatal("expected every term") + } + + testCases := map[string]struct { + JSON string + ExpectedEvery *Every + }{ + "base case": { + JSON: `{"body":[{"index":0,"terms":[{"type":"ref","value":[{"type":"var","value":"equal"}]},{"type":"var","value":"e"},{"type":"number","value":1}]}],"domain":{"type":"array","value":[{"type":"number","value":1},{"type":"number","value":2},{"type":"number","value":3}]},"key":null,"value":{"type":"var","value":"e"}}`, + ExpectedEvery: func() *Every { + e := every.Copy() + e.Location = nil + return e + }(), + }, + "location case": { + JSON: `{"body":[{"index":0,"terms":[{"type":"ref","value":[{"type":"var","value":"equal"}]},{"type":"var","value":"e"},{"type":"number","value":1}]}],"domain":{"type":"array","value":[{"type":"number","value":1},{"type":"number","value":2},{"type":"number","value":3}]},"key":null,"location":{"file":"example.rego","row":7,"col":2},"value":{"type":"var","value":"e"}}`, + ExpectedEvery: func() *Every { + e := every.Copy() + // text is not marshalled to JSON so we just drop it in our examples + e.Location.Text = []byte{} + return e + }(), + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + var e Every + err := json.Unmarshal([]byte(data.JSON), &e) + if err != nil { + t.Fatal(err) + } + + if e.String() != data.ExpectedEvery.String() { + t.Fatalf("expected:\n%#v got\n%#v", data.ExpectedEvery.String(), e.String()) + } + + if data.ExpectedEvery.Location != nil { + if !e.Location.Equal(data.ExpectedEvery.Location) { + t.Fatalf("expected location:\n%#v got\n%#v", data.ExpectedEvery.Location, e.Location) + } + } + }) + } +} + +func TestWith_MarshalJSON(t *testing.T) { + + rawModule := ` +package foo + +a {input} + +b { + a with input as 1 +} +` + + module, err := ParseModuleWithOpts("example.rego", rawModule, ParserOptions{}) + if err != nil { + t.Fatal(err) + } + + with := module.Rules[1].Body[0].With[0] + + testCases := map[string]struct { + With *With + ExpectedJSON string + }{ + "base case": { + With: with, + ExpectedJSON: `{"target":{"type":"ref","value":[{"type":"var","value":"input"}]},"value":{"type":"number","value":1}}`, + }, + "location excluded": { + With: func() *With { + w := with.Copy() + w.jsonFields = map[string]bool{"location": false} + return w + }(), + ExpectedJSON: `{"target":{"type":"ref","value":[{"type":"var","value":"input"}]},"value":{"type":"number","value":1}}`, + }, + "location included": { + With: func() *With { + w := with.Copy() + w.jsonFields = map[string]bool{"location": true} + return w + }(), + ExpectedJSON: `{"location":{"file":"example.rego","row":7,"col":4},"target":{"type":"ref","value":[{"type":"var","value":"input"}]},"value":{"type":"number","value":1}}`, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + bs := util.MustMarshalJSON(data.With) + got := string(bs) + exp := data.ExpectedJSON + + if got != exp { + t.Fatalf("expected:\n%s got\n%s", exp, got) + } + }) + } +} + +func TestWith_UnmarshalJSON(t *testing.T) { + + rawModule := ` +package foo + +a {input} + +b { + a with input as 1 +} +` + + module, err := ParseModuleWithOpts("example.rego", rawModule, ParserOptions{}) + if err != nil { + t.Fatal(err) + } + + with := module.Rules[1].Body[0].With[0] + + testCases := map[string]struct { + JSON string + ExpectedWith *With + }{ + "base case": { + JSON: `{"target":{"type":"ref","value":[{"type":"var","value":"input"}]},"value":{"type":"number","value":1}}`, + ExpectedWith: func() *With { + w := with.Copy() + w.Location = nil + return w + }(), + }, + "location case": { + JSON: `{"location":{"file":"example.rego","row":7,"col":4},"target":{"type":"ref","value":[{"type":"var","value":"input"}]},"value":{"type":"number","value":1}}`, + ExpectedWith: func() *With { + e := with.Copy() + // text is not marshalled to JSON so we just drop it in our examples + e.Location.Text = []byte{} + return e + }(), + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + var w With + err := json.Unmarshal([]byte(data.JSON), &w) + if err != nil { + t.Fatal(err) + } + + if w.String() != data.ExpectedWith.String() { + t.Fatalf("expected:\n%#v got\n%#v", data.ExpectedWith.String(), w.String()) + } + + if data.ExpectedWith.Location != nil { + if !w.Location.Equal(data.ExpectedWith.Location) { + t.Fatalf("expected location:\n%#v got\n%#v", data.ExpectedWith.Location, w.Location) + } + } + }) + } +} + +func TestAnnotations_MarshalJSON(t *testing.T) { + + testCases := map[string]struct { + Annotations *Annotations + ExpectedJSON string + }{ + "base case": { + Annotations: &Annotations{ + Scope: "rule", + Title: "My rule", + Entrypoint: true, + Organizations: []string{"org1"}, + Description: "My desc", + Custom: map[string]interface{}{ + "foo": "bar", + }, + Location: NewLocation([]byte{}, "example.rego", 1, 4), + }, + ExpectedJSON: `{"custom":{"foo":"bar"},"description":"My desc","entrypoint":true,"organizations":["org1"],"scope":"rule","title":"My rule"}`, + }, + "location excluded": { + Annotations: &Annotations{ + Scope: "rule", + Title: "My rule", + Entrypoint: true, + Organizations: []string{"org1"}, + Description: "My desc", + Custom: map[string]interface{}{ + "foo": "bar", + }, + Location: NewLocation([]byte{}, "example.rego", 1, 4), + + jsonFields: map[string]bool{"location": false}, + }, + ExpectedJSON: `{"custom":{"foo":"bar"},"description":"My desc","entrypoint":true,"organizations":["org1"],"scope":"rule","title":"My rule"}`, + }, + "location included": { + Annotations: &Annotations{ + Scope: "rule", + Title: "My rule", + Entrypoint: true, + Organizations: []string{"org1"}, + Description: "My desc", + Custom: map[string]interface{}{ + "foo": "bar", + }, + Location: NewLocation([]byte{}, "example.rego", 1, 4), + + jsonFields: map[string]bool{"location": true}, + }, + ExpectedJSON: `{"custom":{"foo":"bar"},"description":"My desc","entrypoint":true,"location":{"file":"example.rego","row":1,"col":4},"organizations":["org1"],"scope":"rule","title":"My rule"}`, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + bs := util.MustMarshalJSON(data.Annotations) + got := string(bs) + exp := data.ExpectedJSON + + if got != exp { + t.Fatalf("expected:\n%s got\n%s", exp, got) + } + }) + } +} + +func TestAnnotations_UnmarshalJSON(t *testing.T) { + + testCases := map[string]struct { + JSON string + ExpectedAnnotations *Annotations + }{ + "base case": { + JSON: `{"custom":{"foo":"bar"},"description":"My desc","entrypoint":true,"organizations":["org1"],"scope":"rule","title":"My rule"}`, + ExpectedAnnotations: &Annotations{ + Scope: "rule", + Title: "My rule", + Entrypoint: true, + Organizations: []string{"org1"}, + Description: "My desc", + Custom: map[string]interface{}{ + "foo": "bar", + }, + }, + }, + "location case": { + JSON: `{"custom":{"foo":"bar"},"description":"My desc","entrypoint":true,"location":{"file":"example.rego","row":1,"col":4},"organizations":["org1"],"scope":"rule","title":"My rule"}`, + ExpectedAnnotations: &Annotations{ + Scope: "rule", + Title: "My rule", + Entrypoint: true, + Organizations: []string{"org1"}, + Description: "My desc", + Custom: map[string]interface{}{ + "foo": "bar", + }, + Location: NewLocation([]byte{}, "example.rego", 1, 4), + }, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + var a Annotations + err := json.Unmarshal([]byte(data.JSON), &a) + if err != nil { + t.Fatal(err) + } + + if a.String() != data.ExpectedAnnotations.String() { + t.Fatalf("expected:\n%#v got\n%#v", data.ExpectedAnnotations.String(), a.String()) + } + + if data.ExpectedAnnotations.Location != nil { + if !a.Location.Equal(data.ExpectedAnnotations.Location) { + t.Fatalf("expected location:\n%#v got\n%#v", data.ExpectedAnnotations.Location, a.Location) + } + } + }) + } +} + +func TestAnnotationsRef_MarshalJSON(t *testing.T) { + + testCases := map[string]struct { + AnnotationsRef *AnnotationsRef + ExpectedJSON string + }{ + "base case": { + AnnotationsRef: &AnnotationsRef{ + Path: []*Term{}, + // using an empty annotations object here since Annotations marshalling is tested separately + Annotations: &Annotations{}, + Location: NewLocation([]byte{}, "example.rego", 1, 4), + }, + ExpectedJSON: `{"annotations":{"scope":""},"path":[]}`, + }, + "location excluded": { + AnnotationsRef: &AnnotationsRef{ + Path: []*Term{}, + Annotations: &Annotations{}, + Location: NewLocation([]byte{}, "example.rego", 1, 4), + + jsonFields: map[string]bool{"location": false}, + }, + ExpectedJSON: `{"annotations":{"scope":""},"path":[]}`, + }, + "location included": { + AnnotationsRef: &AnnotationsRef{ + Path: []*Term{}, + Annotations: &Annotations{}, + Location: NewLocation([]byte{}, "example.rego", 1, 4), + + jsonFields: map[string]bool{"location": true}, + }, + ExpectedJSON: `{"annotations":{"scope":""},"location":{"file":"example.rego","row":1,"col":4},"path":[]}`, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + bs := util.MustMarshalJSON(data.AnnotationsRef) + got := string(bs) + exp := data.ExpectedJSON + + if got != exp { + t.Fatalf("expected:\n%s got\n%s", exp, got) + } + }) + } +} + +func TestAnnotationsRef_UnmarshalJSON(t *testing.T) { + + testCases := map[string]struct { + JSON string + ExpectedAnnotationsRef *AnnotationsRef + }{ + "base case": { + JSON: `{"annotations":{"scope":""},"path":[]}`, + ExpectedAnnotationsRef: &AnnotationsRef{ + Path: []*Term{}, + Annotations: &Annotations{}, + }, + }, + "location case": { + JSON: `{"custom":{"foo":"bar"},"description":"My desc","entrypoint":true,"location":{"file":"example.rego","row":1,"col":4},"organizations":["org1"],"scope":"rule","title":"My rule"}`, + ExpectedAnnotationsRef: &AnnotationsRef{ + Path: []*Term{}, + Annotations: &Annotations{}, + Location: NewLocation([]byte{}, "example.rego", 1, 4), + }, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + var a AnnotationsRef + err := json.Unmarshal([]byte(data.JSON), &a) + if err != nil { + t.Fatal(err) + } + + if got, exp := len(a.Path), len(data.ExpectedAnnotationsRef.Path); exp != got { + t.Fatalf("expected:\n%#v got\n%#v", exp, got) + } + + if got, exp := a.Annotations.String(), data.ExpectedAnnotationsRef.Annotations.String(); exp != got { + t.Fatalf("expected:\n%#v got\n%#v", exp, got) + } + + if data.ExpectedAnnotationsRef.Location != nil { + if !a.Location.Equal(data.ExpectedAnnotationsRef.Location) { + t.Fatalf("expected location:\n%#v got\n%#v", data.ExpectedAnnotationsRef.Location, a.Location) + } + } + }) + } +} diff --git a/ast/parser.go b/ast/parser.go index 1d700825de..b3b03837c8 100644 --- a/ast/parser.go +++ b/ast/parser.go @@ -101,6 +101,7 @@ type ParserOptions struct { AllFutureKeywords bool FutureKeywords []string SkipRules bool + JSONFields map[string]bool unreleasedKeywords bool // TODO(sr): cleanup } @@ -177,6 +178,12 @@ func (p *Parser) WithSkipRules(skip bool) *Parser { return p } +// WithJSONFields sets the JSON fields that should be exposed in the JSON +func (p *Parser) WithJSONFields(jsonFields map[string]bool) *Parser { + p.po.JSONFields = jsonFields + return p +} + func (p *Parser) parsedTermCacheLookup() (*Term, *state) { l := p.s.loc.Offset // stop comparing once the cached offsets are lower than l @@ -357,6 +364,17 @@ func (p *Parser) Parse() ([]Statement, []*Comment, Errors) { stmts = p.parseAnnotations(stmts) } + for i := range stmts { + vis := NewGenericVisitor(func(x interface{}) bool { + if x, ok := x.(customJSON); ok { + x.exposeJSONFields(p.po.JSONFields) + } + return false + }) + + vis.Walk(stmts[i]) + } + return stmts, p.s.comments, p.s.errors } @@ -891,7 +909,6 @@ func (p *Parser) parseQuery(requireSemi bool, end tokens.Token) Body { } for { - expr := p.parseLiteral() if expr == nil { return nil diff --git a/ast/parser_ext.go b/ast/parser_ext.go index ce25c3a91b..15e9fb497d 100644 --- a/ast/parser_ext.go +++ b/ast/parser_ext.go @@ -594,6 +594,7 @@ func ParseStatementsWithOpts(filename, input string, popts ParserOptions) ([]Sta WithAllFutureKeywords(popts.AllFutureKeywords). WithCapabilities(popts.Capabilities). WithSkipRules(popts.SkipRules). + WithJSONFields(popts.JSONFields). withUnreleasedKeywords(popts.unreleasedKeywords) stmts, comments, errs := parser.Parse() diff --git a/ast/policy.go b/ast/policy.go index 4f56aa7edb..a29eb49297 100644 --- a/ast/policy.go +++ b/ast/policy.go @@ -149,50 +149,61 @@ type ( // Comment contains the raw text from the comment in the definition. Comment struct { + // TODO: these fields have inconsistent JSON keys with other structs in this package. Text []byte Location *Location + + jsonFields map[string]bool } // Package represents the namespace of the documents produced // by rules inside the module. Package struct { - Location *Location `json:"-"` Path Ref `json:"path"` + Location *Location `json:"location,omitempty"` + + jsonFields map[string]bool } // Import represents a dependency on a document outside of the policy // namespace. Imports are optional. Import struct { - Location *Location `json:"-"` Path *Term `json:"path"` Alias Var `json:"alias,omitempty"` + Location *Location `json:"location,omitempty"` + + jsonFields map[string]bool } // Rule represents a rule as defined in the language. Rules define the // content of documents that represent policy decisions. Rule struct { - Location *Location `json:"-"` Default bool `json:"default,omitempty"` Head *Head `json:"head"` Body Body `json:"body"` Else *Rule `json:"else,omitempty"` + Location *Location `json:"location,omitempty"` // Module is a pointer to the module containing this rule. If the rule // was NOT created while parsing/constructing a module, this should be // left unset. The pointer is not included in any standard operations // on the rule (e.g., printing, comparison, visiting, etc.) Module *Module `json:"-"` + + jsonFields map[string]bool } // Head represents the head of a rule. Head struct { - Location *Location `json:"-"` Name Var `json:"name,omitempty"` Reference Ref `json:"ref,omitempty"` Args Args `json:"args,omitempty"` Key *Term `json:"key,omitempty"` Value *Term `json:"value,omitempty"` Assign bool `json:"assign,omitempty"` + Location *Location `json:"location,omitempty"` + + jsonFields map[string]bool } // Args represents zero or more arguments to a rule. @@ -206,31 +217,39 @@ type ( Expr struct { With []*With `json:"with,omitempty"` Terms interface{} `json:"terms"` - Location *Location `json:"-"` Index int `json:"index"` Generated bool `json:"generated,omitempty"` Negated bool `json:"negated,omitempty"` + Location *Location `json:"location,omitempty"` + + jsonFields map[string]bool } // SomeDecl represents a variable declaration statement. The symbols are variables. SomeDecl struct { - Location *Location `json:"-"` Symbols []*Term `json:"symbols"` + Location *Location `json:"location,omitempty"` + + jsonFields map[string]bool } Every struct { - Location *Location `json:"-"` Key *Term `json:"key"` Value *Term `json:"value"` Domain *Term `json:"domain"` Body Body `json:"body"` + Location *Location `json:"location,omitempty"` + + jsonFields map[string]bool } // With represents a modifier on an expression. With struct { - Location *Location `json:"-"` Target *Term `json:"target"` Value *Term `json:"value"` + Location *Location `json:"location,omitempty"` + + jsonFields map[string]bool } ) @@ -409,6 +428,25 @@ func (c *Comment) Equal(other *Comment) bool { return c.Location.Equal(other.Location) && bytes.Equal(c.Text, other.Text) } +func (c *Comment) exposeJSONFields(fields map[string]bool) { + // Note: this is not used for location since Comments have legacy JSON marshaling behavior + c.jsonFields = fields +} + +func (c *Comment) MarshalJSON() ([]byte, error) { + // TODO: Comment has inconsistent JSON field names starting with an upper case letter. + data := map[string]interface{}{ + "Text": c.Text, + } + + // TODO: Comment Location is also always included for legacy reasons. jsonFields data is ignored. + if c.Location != nil { + data["Location"] = c.Location + } + + return json.Marshal(data) +} + // Compare returns an integer indicating whether pkg is less than, equal to, // or greater than other. func (pkg *Package) Compare(other *Package) int { @@ -453,6 +491,24 @@ func (pkg *Package) String() string { return fmt.Sprintf("package %v", path) } +func (pkg *Package) exposeJSONFields(fields map[string]bool) { + pkg.jsonFields = fields +} + +func (pkg *Package) MarshalJSON() ([]byte, error) { + data := map[string]interface{}{ + "path": pkg.Path, + } + + if showLocation, ok := pkg.jsonFields["location"]; ok && showLocation { + if pkg.Location != nil { + data["location"] = pkg.Location + } + } + + return json.Marshal(data) +} + // IsValidImportPath returns an error indicating if the import path is invalid. // If the import path is invalid, err is nil. func IsValidImportPath(v Value) (err error) { @@ -545,6 +601,28 @@ func (imp *Import) String() string { return strings.Join(buf, " ") } +func (imp *Import) exposeJSONFields(fields map[string]bool) { + imp.jsonFields = fields +} + +func (imp *Import) MarshalJSON() ([]byte, error) { + data := map[string]interface{}{ + "path": imp.Path, + } + + if len(imp.Alias) != 0 { + data["alias"] = imp.Alias + } + + if showLocation, ok := imp.jsonFields["location"]; ok && showLocation { + if imp.Location != nil { + data["location"] = imp.Location + } + } + + return json.Marshal(data) +} + // Compare returns an integer indicating whether rule is less than, equal to, // or greater than other. func (rule *Rule) Compare(other *Rule) int { @@ -634,6 +712,33 @@ func (rule *Rule) String() string { return strings.Join(buf, " ") } +func (rule *Rule) exposeJSONFields(fields map[string]bool) { + rule.jsonFields = fields +} + +func (rule *Rule) MarshalJSON() ([]byte, error) { + data := map[string]interface{}{ + "head": rule.Head, + "body": rule.Body, + } + + if rule.Default { + data["default"] = true + } + + if rule.Else != nil { + data["else"] = rule.Else + } + + if showLocation, ok := rule.jsonFields["location"]; ok && showLocation { + if rule.Location != nil { + data["location"] = rule.Location + } + } + + return json.Marshal(data) +} + func (rule *Rule) elseString() string { var buf []string @@ -823,17 +928,31 @@ func (head *Head) String() string { return buf.String() } +func (head *Head) exposeJSONFields(fields map[string]bool) { + head.jsonFields = fields +} + func (head *Head) MarshalJSON() ([]byte, error) { + var loc *Location + if showLocation, ok := head.jsonFields["location"]; ok && showLocation { + if head.Location != nil { + loc = head.Location + } + } + // NOTE(sr): we do this to override the rendering of `head.Reference`. // It's still what'll be used via the default means of encoding/json // for unmarshaling a json object into a Head struct! + // NOTE(charlieegan3): we also need to optionally include the location type h Head return json.Marshal(struct { h - Ref Ref `json:"ref"` + Ref Ref `json:"ref"` + Location *Location `json:"location,omitempty"` }{ - h: h(*head), - Ref: head.Ref(), + h: h(*head), + Ref: head.Ref(), + Location: loc, }) } @@ -924,7 +1043,8 @@ func (body Body) MarshalJSON() ([]byte, error) { if len(body) == 0 { return []byte(`[]`), nil } - return json.Marshal([]*Expr(body)) + ret, err := json.Marshal([]*Expr(body)) + return ret, err } // Append adds the expr to the body and updates the expr's index accordingly. @@ -1352,6 +1472,37 @@ func (expr *Expr) String() string { return strings.Join(buf, " ") } +func (expr *Expr) exposeJSONFields(fields map[string]bool) { + expr.jsonFields = fields +} + +func (expr *Expr) MarshalJSON() ([]byte, error) { + data := map[string]interface{}{ + "terms": expr.Terms, + "index": expr.Index, + } + + if len(expr.With) > 0 { + data["with"] = expr.With + } + + if expr.Generated { + data["generated"] = true + } + + if expr.Negated { + data["negated"] = true + } + + if showLocation, ok := expr.jsonFields["location"]; ok && showLocation { + if expr.Location != nil { + data["location"] = expr.Location + } + } + + return json.Marshal(data) +} + // UnmarshalJSON parses the byte array and stores the result in expr. func (expr *Expr) UnmarshalJSON(bs []byte) error { v := map[string]interface{}{} @@ -1417,6 +1568,24 @@ func (d *SomeDecl) Hash() int { return termSliceHash(d.Symbols) } +func (d *SomeDecl) exposeJSONFields(fields map[string]bool) { + d.jsonFields = fields +} + +func (d *SomeDecl) MarshalJSON() ([]byte, error) { + data := map[string]interface{}{ + "symbols": d.Symbols, + } + + if showLocation, ok := d.jsonFields["location"]; ok && showLocation { + if d.Location != nil { + data["location"] = d.Location + } + } + + return json.Marshal(data) +} + func (q *Every) String() string { if q.Key != nil { return fmt.Sprintf("every %s, %s in %s { %s }", @@ -1473,6 +1642,27 @@ func (q *Every) KeyValueVars() VarSet { return vis.vars } +func (q *Every) exposeJSONFields(fields map[string]bool) { + q.jsonFields = fields +} + +func (q *Every) MarshalJSON() ([]byte, error) { + data := map[string]interface{}{ + "key": q.Key, + "value": q.Value, + "domain": q.Domain, + "body": q.Body, + } + + if showLocation, ok := q.jsonFields["location"]; ok && showLocation { + if q.Location != nil { + data["location"] = q.Location + } + } + + return json.Marshal(data) +} + func (w *With) String() string { return "with " + w.Target.String() + " as " + w.Value.String() } @@ -1531,6 +1721,25 @@ func (w *With) SetLoc(loc *Location) { w.Location = loc } +func (w *With) exposeJSONFields(fields map[string]bool) { + w.jsonFields = fields +} + +func (w *With) MarshalJSON() ([]byte, error) { + data := map[string]interface{}{ + "target": w.Target, + "value": w.Value, + } + + if showLocation, ok := w.jsonFields["location"]; ok && showLocation { + if w.Location != nil { + data["location"] = w.Location + } + } + + return json.Marshal(data) +} + // Copy returns a deep copy of the AST node x. If x is not an AST node, x is returned unmodified. func Copy(x interface{}) interface{} { switch x := x.(type) { diff --git a/ast/policy_test.go b/ast/policy_test.go index 2645a72d86..118fc5b99f 100644 --- a/ast/policy_test.go +++ b/ast/policy_test.go @@ -408,7 +408,7 @@ func TestRuleHeadJSON(t *testing.T) { if err != nil { t.Fatal(err) } - if exp, act := `{"head":{"name":"allow","ref":[{"type":"var","value":"allow"}]},"body":[]}`, string(bs); act != exp { + if exp, act := `{"body":[],"head":{"name":"allow","ref":[{"type":"var","value":"allow"}]}}`, string(bs); act != exp { t.Errorf("expected %q, got %q", exp, act) } @@ -805,17 +805,10 @@ func TestAnnotationsString(t *testing.T) { // NOTE(tsandall): for now, annotations are represented as JSON objects // which are a subset of YAML. We could improve this in the future. - exp := `{"scope":"foo",` + - `"title":"bar",` + - `"description":"baz",` + - `"organizations":["mi","fa"],` + - `"related_resources":[{"ref":"https://example.com"},{"description":"Some resource","ref":"https://example.com/2"}],` + - `"authors":[{"name":"John Doe","email":"john@example.com"},{"name":"Jane Doe"}],` + - `"schemas":[{"path":[{"type":"var","value":"data"},{"type":"string","value":"bar"}],"schema":[{"type":"var","value":"schema"},{"type":"string","value":"baz"}]}],` + - `"custom":{"flag":true,"list":[1,2,3],"map":{"one":1,"two":{"3":"three"}}}}` - - if exp != a.String() { - t.Fatalf("expected %q but got %q", exp, a.String()) + exp := `{"authors":[{"name":"John Doe","email":"john@example.com"},{"name":"Jane Doe"}],"custom":{"flag":true,"list":[1,2,3],"map":{"one":1,"two":{"3":"three"}}},"description":"baz","organizations":["mi","fa"],"related_resources":[{"ref":"https://example.com"},{"description":"Some resource","ref":"https://example.com/2"}],"schemas":[{"path":[{"type":"var","value":"data"},{"type":"string","value":"bar"}],"schema":[{"type":"var","value":"schema"},{"type":"string","value":"baz"}]}],"scope":"foo","title":"bar"}` + + if got := a.String(); exp != got { + t.Fatalf("expected\n%s\nbut got\n%s", exp, got) } } diff --git a/ast/term.go b/ast/term.go index b720d59d90..26826b4506 100644 --- a/ast/term.go +++ b/ast/term.go @@ -291,8 +291,10 @@ func MustInterfaceToValue(x interface{}) Value { // Term is an argument to a function. type Term struct { - Value Value `json:"value"` // the value of the Term as represented in Go - Location *Location `json:"-"` // the location of the Term in the source + Value Value `json:"value"` // the value of the Term as represented in Go + Location *Location `json:"location,omitempty"` // the location of the Term in the source + + jsonFields map[string]bool } // NewTerm returns a new Term object. @@ -417,6 +419,10 @@ func (term *Term) IsGround() bool { return term.Value.IsGround() } +func (term *Term) exposeJSONFields(fields map[string]bool) { + term.jsonFields = fields +} + // MarshalJSON returns the JSON encoding of the term. // // Specialized marshalling logic is required to include a type hint for Value. @@ -425,6 +431,11 @@ func (term *Term) MarshalJSON() ([]byte, error) { "type": TypeName(term.Value), "value": term.Value, } + if showLocation, ok := term.jsonFields["location"]; ok && showLocation { + if term.Location != nil { + d["location"] = term.Location + } + } return json.Marshal(d) } @@ -444,6 +455,14 @@ func (term *Term) UnmarshalJSON(bs []byte) error { return err } term.Value = val + + if loc, ok := v["location"].(map[string]interface{}); ok { + term.Location = &Location{} + err := unmarshalLocation(term.Location, loc) + if err != nil { + return err + } + } return nil } @@ -2897,6 +2916,14 @@ func unmarshalExpr(expr *Expr, v map[string]interface{}) error { return fmt.Errorf("ast: unable to unmarshal negated field with type: %T (expected true or false)", v["negated"]) } } + if generatedRaw, ok := v["generated"]; ok { + if b, ok := generatedRaw.(bool); ok { + expr.Generated = b + } else { + return fmt.Errorf("ast: unable to unmarshal generated field with type: %T (expected true or false)", v["generated"]) + } + } + if err := unmarshalExprIndex(expr, v); err != nil { return err } @@ -2929,6 +2956,46 @@ func unmarshalExpr(expr *Expr, v map[string]interface{}) error { expr.With = ws } } + if loc, ok := v["location"].(map[string]interface{}); ok { + expr.Location = &Location{} + if err := unmarshalLocation(expr.Location, loc); err != nil { + return err + } + } + return nil +} + +func unmarshalLocation(loc *Location, v map[string]interface{}) error { + if x, ok := v["file"]; ok { + if s, ok := x.(string); ok { + loc.File = s + } else { + return fmt.Errorf("ast: unable to unmarshal file field with type: %T (expected string)", v["file"]) + } + } + if x, ok := v["row"]; ok { + if n, ok := x.(json.Number); ok { + i64, err := n.Int64() + if err != nil { + return err + } + loc.Row = int(i64) + } else { + return fmt.Errorf("ast: unable to unmarshal row field with type: %T (expected number)", v["row"]) + } + } + if x, ok := v["col"]; ok { + if n, ok := x.(json.Number); ok { + i64, err := n.Int64() + if err != nil { + return err + } + loc.Col = int(i64) + } else { + return fmt.Errorf("ast: unable to unmarshal col field with type: %T (expected number)", v["col"]) + } + } + return nil } @@ -2946,11 +3013,22 @@ func unmarshalExprIndex(expr *Expr, v map[string]interface{}) error { } func unmarshalTerm(m map[string]interface{}) (*Term, error) { + var term Term + v, err := unmarshalValue(m) if err != nil { return nil, err } - return &Term{Value: v}, nil + term.Value = v + + if loc, ok := m["location"].(map[string]interface{}); ok { + term.Location = &Location{} + if err := unmarshalLocation(term.Location, loc); err != nil { + return nil, err + } + } + + return &term, nil } func unmarshalTermSlice(s []interface{}) ([]*Term, error) { diff --git a/cmd/parse.go b/cmd/parse.go index 28c37ae343..e01c6024ee 100644 --- a/cmd/parse.go +++ b/cmd/parse.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "os" + "strings" "github.com/spf13/cobra" @@ -24,9 +25,11 @@ const ( ) var parseParams = struct { - format *util.EnumFlag + format *util.EnumFlag + jsonInclude string }{ - format: util.NewEnumFlag(parseFormatPretty, []string{parseFormatPretty, parseFormatJSON}), + format: util.NewEnumFlag(parseFormatPretty, []string{parseFormatPretty, parseFormatJSON}), + jsonInclude: "", } var parseCommand = &cobra.Command{ @@ -49,20 +52,45 @@ func parse(args []string, stdout io.Writer, stderr io.Writer) int { return 0 } - result, err := loader.RegoWithOpts(args[0], ast.ParserOptions{ProcessAnnotation: true}) + exposeLocation := false + exposeComments := true + for _, opt := range strings.Split(parseParams.jsonInclude, ",") { + value := true + if strings.HasPrefix(opt, "-") { + value = false + } + + if strings.HasSuffix(opt, "locations") { + exposeLocation = value + } + if strings.HasSuffix(opt, "comments") { + exposeComments = value + } + } + + result, err := loader.RegoWithOpts(args[0], ast.ParserOptions{ + ProcessAnnotation: true, + JSONFields: map[string]bool{ + "location": exposeLocation, + }, + }) + if err != nil { + _ = pr.JSON(stderr, pr.Output{Errors: pr.NewOutputErrors(err)}) + return 1 + } + + if !exposeComments { + result.Parsed.Comments = nil + } switch parseParams.format.String() { case parseFormatJSON: + bs, err := json.MarshalIndent(result.Parsed, "", " ") if err != nil { _ = pr.JSON(stderr, pr.Output{Errors: pr.NewOutputErrors(err)}) return 1 } - bs, err := json.MarshalIndent(result.Parsed, "", " ") - if err != nil { - fmt.Fprintln(stderr, err) - return 1 - } fmt.Println(string(bs)) default: if err != nil { @@ -77,5 +105,7 @@ func parse(args []string, stdout io.Writer, stderr io.Writer) int { func init() { parseCommand.Flags().VarP(parseParams.format, "format", "f", "set output format") + parseCommand.Flags().StringVarP(&parseParams.jsonInclude, "json-include", "", "", "select optional elements, current options: locations, comments. E.g. --json-include locations,-comments will include locations and exclude comments.") + RootCommand.AddCommand(parseCommand) } From 66a0e20b00b4afa48a34c222e9a89fbb75e862f4 Mon Sep 17 00:00:00 2001 From: Charlie Egan Date: Tue, 24 Jan 2023 09:35:33 +0000 Subject: [PATCH 2/8] Explain comment location marshalling behaviour Signed-off-by: Charlie Egan --- ast/marshal_test.go | 6 +++--- ast/policy.go | 14 -------------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/ast/marshal_test.go b/ast/marshal_test.go index 7486ea3a98..9a021a5551 100644 --- a/ast/marshal_test.go +++ b/ast/marshal_test.go @@ -207,7 +207,7 @@ func TestComment_MarshalJSON(t *testing.T) { Comment: &Comment{ Text: []byte("comment"), }, - ExpectedJSON: `{"Text":"Y29tbWVudA=="}`, + ExpectedJSON: `{"Text":"Y29tbWVudA==","Location":null}`, }, "location excluded, still included for legacy reasons": { Comment: &Comment{ @@ -217,7 +217,7 @@ func TestComment_MarshalJSON(t *testing.T) { "location": false, // ignored }, }, - ExpectedJSON: `{"Location":{"file":"example.rego","row":1,"col":2},"Text":"Y29tbWVudA=="}`, + ExpectedJSON: `{"Text":"Y29tbWVudA==","Location":{"file":"example.rego","row":1,"col":2}}`, }, "location included": { Comment: &Comment{ @@ -227,7 +227,7 @@ func TestComment_MarshalJSON(t *testing.T) { "location": true, // ignored }, }, - ExpectedJSON: `{"Location":{"file":"example.rego","row":1,"col":2},"Text":"Y29tbWVudA=="}`, + ExpectedJSON: `{"Text":"Y29tbWVudA==","Location":{"file":"example.rego","row":1,"col":2}}`, }, } diff --git a/ast/policy.go b/ast/policy.go index a29eb49297..ae508a419e 100644 --- a/ast/policy.go +++ b/ast/policy.go @@ -433,20 +433,6 @@ func (c *Comment) exposeJSONFields(fields map[string]bool) { c.jsonFields = fields } -func (c *Comment) MarshalJSON() ([]byte, error) { - // TODO: Comment has inconsistent JSON field names starting with an upper case letter. - data := map[string]interface{}{ - "Text": c.Text, - } - - // TODO: Comment Location is also always included for legacy reasons. jsonFields data is ignored. - if c.Location != nil { - data["Location"] = c.Location - } - - return json.Marshal(data) -} - // Compare returns an integer indicating whether pkg is less than, equal to, // or greater than other. func (pkg *Package) Compare(other *Package) int { From 59b5ee78d5646e408ae340532dd3194851f30321 Mon Sep 17 00:00:00 2001 From: Charlie Egan Date: Tue, 24 Jan 2023 09:47:03 +0000 Subject: [PATCH 3/8] Remove surplus jsonFields data Signed-off-by: Charlie Egan --- ast/marshal_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/ast/marshal_test.go b/ast/marshal_test.go index 9a021a5551..f54d4d09f9 100644 --- a/ast/marshal_test.go +++ b/ast/marshal_test.go @@ -387,9 +387,6 @@ func TestImport_UnmarshalJSON(t *testing.T) { return &Import{ Path: &term, Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonFields: map[string]bool{ - "location": false, - }, } }(), }, From 4a090b1f85c031975396750855612d7679c0f39a Mon Sep 17 00:00:00 2001 From: Charlie Egan Date: Thu, 23 Feb 2023 11:33:40 +0000 Subject: [PATCH 4/8] Use custom options struct for future-proofing This change adds a new struct JSONOptions which allows the JSON marshalling and unmarshalling of ast nodes to be further customized in the future. It's my hope that this will be more extendable than the simple map we were using before. Signed-off-by: Charlie Egan --- ast/annotations.go | 16 ++--- ast/marshal.go | 5 +- ast/marshal_test.go | 159 ++++++++++++++++++++++++++++++++------------ ast/parser.go | 56 ++++++++++++---- ast/parser_ext.go | 2 +- ast/policy.go | 73 ++++++++++---------- ast/term.go | 8 +-- cmd/parse.go | 29 ++++++-- 8 files changed, 235 insertions(+), 113 deletions(-) diff --git a/ast/annotations.go b/ast/annotations.go index 2eb712e139..b547f766fa 100644 --- a/ast/annotations.go +++ b/ast/annotations.go @@ -37,9 +37,9 @@ type ( Custom map[string]interface{} `json:"custom,omitempty"` Location *Location `json:"location,omitempty"` - comments []*Comment - node Node - jsonFields map[string]bool + comments []*Comment + node Node + jsonOptions JSONOptions } // SchemaAnnotation contains a schema declaration for the document identified by the path. @@ -76,7 +76,7 @@ type ( Annotations *Annotations `json:"annotations,omitempty"` Location *Location `json:"location,omitempty"` // The location of the node the annotations are applied to - jsonFields map[string]bool + jsonOptions JSONOptions node Node // The node the annotations are applied to } @@ -180,8 +180,8 @@ func (a *Annotations) GetTargetPath() Ref { } } -func (a *Annotations) exposeJSONFields(jsonFields map[string]bool) { - a.jsonFields = jsonFields +func (a *Annotations) setJSONOptions(opts JSONOptions) { + a.jsonOptions = opts } func (a *Annotations) MarshalJSON() ([]byte, error) { @@ -225,7 +225,7 @@ func (a *Annotations) MarshalJSON() ([]byte, error) { data["custom"] = a.Custom } - if showLocation, ok := a.jsonFields["location"]; ok && showLocation { + if a.jsonOptions.MarshalOptions.IncludeLocation.Annotations { if a.Location != nil { data["location"] = a.Location } @@ -277,7 +277,7 @@ func (ar *AnnotationsRef) MarshalJSON() ([]byte, error) { data["annotations"] = ar.Annotations } - if showLocation, ok := ar.jsonFields["location"]; ok && showLocation { + if ar.jsonOptions.MarshalOptions.IncludeLocation.AnnotationsRef { if ar.Location != nil { data["location"] = ar.Location } diff --git a/ast/marshal.go b/ast/marshal.go index 65b323154f..891945db8b 100644 --- a/ast/marshal.go +++ b/ast/marshal.go @@ -1,8 +1,7 @@ package ast // customJSON is an interface that can be implemented by AST nodes that -// allows the parser to set a list of fields and if the field is to be -// included in the JSON output. +// allows the parser to set options for JSON operations on that node. type customJSON interface { - exposeJSONFields(map[string]bool) + setJSONOptions(JSONOptions) } diff --git a/ast/marshal_test.go b/ast/marshal_test.go index f54d4d09f9..3d2e85f291 100644 --- a/ast/marshal_test.go +++ b/ast/marshal_test.go @@ -29,8 +29,12 @@ func TestTerm_MarshalJSON(t *testing.T) { return &Term{ Value: v, Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonFields: map[string]bool{ - "location": false, + jsonOptions: JSONOptions{ + MarshalOptions: JSONMarshalOptions{ + IncludeLocation: NodeToggle{ + Term: false, + }, + }, }, } }(), @@ -42,8 +46,12 @@ func TestTerm_MarshalJSON(t *testing.T) { return &Term{ Value: v, Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonFields: map[string]bool{ - "location": true, + jsonOptions: JSONOptions{ + MarshalOptions: JSONMarshalOptions{ + IncludeLocation: NodeToggle{ + Term: true, + }, + }, }, } }(), @@ -125,8 +133,12 @@ func TestPackage_MarshalJSON(t *testing.T) { Package: &Package{ Path: EmptyRef(), Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonFields: map[string]bool{ - "location": false, + jsonOptions: JSONOptions{ + MarshalOptions: JSONMarshalOptions{ + IncludeLocation: NodeToggle{ + Package: false, + }, + }, }, }, ExpectedJSON: `{"path":[]}`, @@ -135,8 +147,12 @@ func TestPackage_MarshalJSON(t *testing.T) { Package: &Package{ Path: EmptyRef(), Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonFields: map[string]bool{ - "location": true, + jsonOptions: JSONOptions{ + MarshalOptions: JSONMarshalOptions{ + IncludeLocation: NodeToggle{ + Package: true, + }, + }, }, }, ExpectedJSON: `{"location":{"file":"example.rego","row":1,"col":2},"path":[]}`, @@ -213,8 +229,12 @@ func TestComment_MarshalJSON(t *testing.T) { Comment: &Comment{ Text: []byte("comment"), Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonFields: map[string]bool{ - "location": false, // ignored + jsonOptions: JSONOptions{ + MarshalOptions: JSONMarshalOptions{ + IncludeLocation: NodeToggle{ + Comment: false, // ignored + }, + }, }, }, ExpectedJSON: `{"Text":"Y29tbWVudA==","Location":{"file":"example.rego","row":1,"col":2}}`, @@ -223,8 +243,12 @@ func TestComment_MarshalJSON(t *testing.T) { Comment: &Comment{ Text: []byte("comment"), Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonFields: map[string]bool{ - "location": true, // ignored + jsonOptions: JSONOptions{ + MarshalOptions: JSONMarshalOptions{ + IncludeLocation: NodeToggle{ + Comment: true, // ignored + }, + }, }, }, ExpectedJSON: `{"Text":"Y29tbWVudA==","Location":{"file":"example.rego","row":1,"col":2}}`, @@ -322,8 +346,12 @@ func TestImport_MarshalJSON(t *testing.T) { return &Import{ Path: &term, Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonFields: map[string]bool{ - "location": false, + jsonOptions: JSONOptions{ + MarshalOptions: JSONMarshalOptions{ + IncludeLocation: NodeToggle{ + Import: false, + }, + }, }, } }(), @@ -339,8 +367,12 @@ func TestImport_MarshalJSON(t *testing.T) { return &Import{ Path: &term, Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonFields: map[string]bool{ - "location": true, + jsonOptions: JSONOptions{ + MarshalOptions: JSONMarshalOptions{ + IncludeLocation: NodeToggle{ + Import: true, + }, + }, }, } }(), @@ -439,8 +471,12 @@ func TestRule_MarshalJSON(t *testing.T) { "location excluded": { Rule: func() *Rule { r := rule.Copy() - r.jsonFields = map[string]bool{ - "location": false, + r.jsonOptions = JSONOptions{ + MarshalOptions: JSONMarshalOptions{ + IncludeLocation: NodeToggle{ + Rule: false, + }, + }, } return r }(), @@ -449,8 +485,12 @@ func TestRule_MarshalJSON(t *testing.T) { "location included": { Rule: func() *Rule { r := rule.Copy() - r.jsonFields = map[string]bool{ - "location": true, + r.jsonOptions = JSONOptions{ + MarshalOptions: JSONMarshalOptions{ + IncludeLocation: NodeToggle{ + Rule: true, + }, + }, } return r }(), @@ -554,9 +594,14 @@ func TestHead_MarshalJSON(t *testing.T) { "location excluded": { Head: func() *Head { h := head.Copy() - h.jsonFields = map[string]bool{ - "location": false, + h.jsonOptions = JSONOptions{ + MarshalOptions: JSONMarshalOptions{ + IncludeLocation: NodeToggle{ + Head: false, + }, + }, } + return h }(), ExpectedJSON: `{"name":"allow","value":{"type":"boolean","value":true},"ref":[{"type":"var","value":"allow"}]}`, @@ -564,8 +609,12 @@ func TestHead_MarshalJSON(t *testing.T) { "location included": { Head: func() *Head { h := head.Copy() - h.jsonFields = map[string]bool{ - "location": true, + h.jsonOptions = JSONOptions{ + MarshalOptions: JSONMarshalOptions{ + IncludeLocation: NodeToggle{ + Head: true, + }, + }, } return h }(), @@ -669,9 +718,14 @@ func TestExpr_MarshalJSON(t *testing.T) { "location excluded": { Expr: func() *Expr { e := expr.Copy() - e.jsonFields = map[string]bool{ - "location": false, + e.jsonOptions = JSONOptions{ + MarshalOptions: JSONMarshalOptions{ + IncludeLocation: NodeToggle{ + Expr: false, + }, + }, } + return e }(), ExpectedJSON: `{"index":0,"terms":{"type":"boolean","value":true}}`, @@ -679,8 +733,12 @@ func TestExpr_MarshalJSON(t *testing.T) { "location included": { Expr: func() *Expr { e := expr.Copy() - e.jsonFields = map[string]bool{ - "location": true, + e.jsonOptions = JSONOptions{ + MarshalOptions: JSONMarshalOptions{ + IncludeLocation: NodeToggle{ + Expr: true, + }, + }, } return e }(), @@ -777,17 +835,17 @@ func TestSomeDecl_MarshalJSON(t *testing.T) { }, "location excluded": { SomeDecl: &SomeDecl{ - Symbols: []*Term{term}, - Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonFields: map[string]bool{"location": false}, + Symbols: []*Term{term}, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + jsonOptions: JSONOptions{MarshalOptions: JSONMarshalOptions{IncludeLocation: NodeToggle{SomeDecl: false}}}, }, ExpectedJSON: `{"symbols":[{"type":"string","value":"example"}]}`, }, "location included": { SomeDecl: &SomeDecl{ - Symbols: []*Term{term}, - Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonFields: map[string]bool{"location": true}, + Symbols: []*Term{term}, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + jsonOptions: JSONOptions{MarshalOptions: JSONMarshalOptions{IncludeLocation: NodeToggle{SomeDecl: true}}}, }, ExpectedJSON: `{"location":{"file":"example.rego","row":1,"col":2},"symbols":[{"type":"string","value":"example"}]}`, }, @@ -888,7 +946,7 @@ allow { "location excluded": { Every: func() *Every { e := every.Copy() - e.jsonFields = map[string]bool{"location": false} + e.jsonOptions = JSONOptions{MarshalOptions: JSONMarshalOptions{IncludeLocation: NodeToggle{Every: false}}} return e }(), ExpectedJSON: `{"body":[{"index":0,"terms":[{"type":"ref","value":[{"type":"var","value":"equal"}]},{"type":"var","value":"e"},{"type":"number","value":1}]}],"domain":{"type":"array","value":[{"type":"number","value":1},{"type":"number","value":2},{"type":"number","value":3}]},"key":null,"value":{"type":"var","value":"e"}}`, @@ -896,7 +954,7 @@ allow { "location included": { Every: func() *Every { e := every.Copy() - e.jsonFields = map[string]bool{"location": true} + e.jsonOptions = JSONOptions{MarshalOptions: JSONMarshalOptions{IncludeLocation: NodeToggle{Every: true}}} return e }(), ExpectedJSON: `{"body":[{"index":0,"terms":[{"type":"ref","value":[{"type":"var","value":"equal"}]},{"type":"var","value":"e"},{"type":"number","value":1}]}],"domain":{"type":"array","value":[{"type":"number","value":1},{"type":"number","value":2},{"type":"number","value":3}]},"key":null,"location":{"file":"example.rego","row":7,"col":2},"value":{"type":"var","value":"e"}}`, @@ -1013,7 +1071,7 @@ b { "location excluded": { With: func() *With { w := with.Copy() - w.jsonFields = map[string]bool{"location": false} + w.jsonOptions = JSONOptions{MarshalOptions: JSONMarshalOptions{IncludeLocation: NodeToggle{With: false}}} return w }(), ExpectedJSON: `{"target":{"type":"ref","value":[{"type":"var","value":"input"}]},"value":{"type":"number","value":1}}`, @@ -1021,7 +1079,7 @@ b { "location included": { With: func() *With { w := with.Copy() - w.jsonFields = map[string]bool{"location": true} + w.jsonOptions = JSONOptions{MarshalOptions: JSONMarshalOptions{IncludeLocation: NodeToggle{With: true}}} return w }(), ExpectedJSON: `{"location":{"file":"example.rego","row":7,"col":4},"target":{"type":"ref","value":[{"type":"var","value":"input"}]},"value":{"type":"number","value":1}}`, @@ -1136,7 +1194,11 @@ func TestAnnotations_MarshalJSON(t *testing.T) { }, Location: NewLocation([]byte{}, "example.rego", 1, 4), - jsonFields: map[string]bool{"location": false}, + jsonOptions: JSONOptions{ + MarshalOptions: JSONMarshalOptions{ + IncludeLocation: NodeToggle{Annotations: false}, + }, + }, }, ExpectedJSON: `{"custom":{"foo":"bar"},"description":"My desc","entrypoint":true,"organizations":["org1"],"scope":"rule","title":"My rule"}`, }, @@ -1152,7 +1214,11 @@ func TestAnnotations_MarshalJSON(t *testing.T) { }, Location: NewLocation([]byte{}, "example.rego", 1, 4), - jsonFields: map[string]bool{"location": true}, + jsonOptions: JSONOptions{ + MarshalOptions: JSONMarshalOptions{ + IncludeLocation: NodeToggle{Annotations: true}, + }, + }, }, ExpectedJSON: `{"custom":{"foo":"bar"},"description":"My desc","entrypoint":true,"location":{"file":"example.rego","row":1,"col":4},"organizations":["org1"],"scope":"rule","title":"My rule"}`, }, @@ -1247,8 +1313,11 @@ func TestAnnotationsRef_MarshalJSON(t *testing.T) { Path: []*Term{}, Annotations: &Annotations{}, Location: NewLocation([]byte{}, "example.rego", 1, 4), - - jsonFields: map[string]bool{"location": false}, + jsonOptions: JSONOptions{ + MarshalOptions: JSONMarshalOptions{ + IncludeLocation: NodeToggle{AnnotationsRef: false}, + }, + }, }, ExpectedJSON: `{"annotations":{"scope":""},"path":[]}`, }, @@ -1258,7 +1327,11 @@ func TestAnnotationsRef_MarshalJSON(t *testing.T) { Annotations: &Annotations{}, Location: NewLocation([]byte{}, "example.rego", 1, 4), - jsonFields: map[string]bool{"location": true}, + jsonOptions: JSONOptions{ + MarshalOptions: JSONMarshalOptions{ + IncludeLocation: NodeToggle{AnnotationsRef: true}, + }, + }, }, ExpectedJSON: `{"annotations":{"scope":""},"location":{"file":"example.rego","row":1,"col":4},"path":[]}`, }, diff --git a/ast/parser.go b/ast/parser.go index b3b03837c8..58e9e73c8a 100644 --- a/ast/parser.go +++ b/ast/parser.go @@ -101,10 +101,39 @@ type ParserOptions struct { AllFutureKeywords bool FutureKeywords []string SkipRules bool - JSONFields map[string]bool + JSONOptions *JSONOptions unreleasedKeywords bool // TODO(sr): cleanup } +// JSONOptions defines the options for JSON operations, +// currently only marshaling can be configured +type JSONOptions struct { + MarshalOptions JSONMarshalOptions +} + +// JSONMarshalOptions defines the options for JSON marshaling, +// currently only toggling the marshaling of location information is supported +type JSONMarshalOptions struct { + IncludeLocation NodeToggle +} + +// NodeToggle is a generic struct to allow the toggling of +// settings for different ast node types +type NodeToggle struct { + Term bool + Package bool + Comment bool + Import bool + Rule bool + Head bool + Expr bool + SomeDecl bool + Every bool + With bool + Annotations bool + AnnotationsRef bool +} + // NewParser creates and initializes a Parser. func NewParser() *Parser { p := &Parser{ @@ -178,9 +207,10 @@ func (p *Parser) WithSkipRules(skip bool) *Parser { return p } -// WithJSONFields sets the JSON fields that should be exposed in the JSON -func (p *Parser) WithJSONFields(jsonFields map[string]bool) *Parser { - p.po.JSONFields = jsonFields +// WithJSONOptions sets the JSONOptions which will be set on nodes to configure +// their JSON marshaling behavior. +func (p *Parser) WithJSONOptions(jsonOptions *JSONOptions) *Parser { + p.po.JSONOptions = jsonOptions return p } @@ -364,15 +394,17 @@ func (p *Parser) Parse() ([]Statement, []*Comment, Errors) { stmts = p.parseAnnotations(stmts) } - for i := range stmts { - vis := NewGenericVisitor(func(x interface{}) bool { - if x, ok := x.(customJSON); ok { - x.exposeJSONFields(p.po.JSONFields) - } - return false - }) + if p.po.JSONOptions != nil { + for i := range stmts { + vis := NewGenericVisitor(func(x interface{}) bool { + if x, ok := x.(customJSON); ok { + x.setJSONOptions(*p.po.JSONOptions) + } + return false + }) - vis.Walk(stmts[i]) + vis.Walk(stmts[i]) + } } return stmts, p.s.comments, p.s.errors diff --git a/ast/parser_ext.go b/ast/parser_ext.go index 15e9fb497d..edc1b63a8e 100644 --- a/ast/parser_ext.go +++ b/ast/parser_ext.go @@ -594,7 +594,7 @@ func ParseStatementsWithOpts(filename, input string, popts ParserOptions) ([]Sta WithAllFutureKeywords(popts.AllFutureKeywords). WithCapabilities(popts.Capabilities). WithSkipRules(popts.SkipRules). - WithJSONFields(popts.JSONFields). + WithJSONOptions(popts.JSONOptions). withUnreleasedKeywords(popts.unreleasedKeywords) stmts, comments, errs := parser.Parse() diff --git a/ast/policy.go b/ast/policy.go index ae508a419e..caf756f6aa 100644 --- a/ast/policy.go +++ b/ast/policy.go @@ -153,7 +153,7 @@ type ( Text []byte Location *Location - jsonFields map[string]bool + jsonOptions JSONOptions } // Package represents the namespace of the documents produced @@ -162,7 +162,7 @@ type ( Path Ref `json:"path"` Location *Location `json:"location,omitempty"` - jsonFields map[string]bool + jsonOptions JSONOptions } // Import represents a dependency on a document outside of the policy @@ -172,7 +172,7 @@ type ( Alias Var `json:"alias,omitempty"` Location *Location `json:"location,omitempty"` - jsonFields map[string]bool + jsonOptions JSONOptions } // Rule represents a rule as defined in the language. Rules define the @@ -190,7 +190,7 @@ type ( // on the rule (e.g., printing, comparison, visiting, etc.) Module *Module `json:"-"` - jsonFields map[string]bool + jsonOptions JSONOptions } // Head represents the head of a rule. @@ -203,7 +203,7 @@ type ( Assign bool `json:"assign,omitempty"` Location *Location `json:"location,omitempty"` - jsonFields map[string]bool + jsonOptions JSONOptions } // Args represents zero or more arguments to a rule. @@ -222,7 +222,7 @@ type ( Negated bool `json:"negated,omitempty"` Location *Location `json:"location,omitempty"` - jsonFields map[string]bool + jsonOptions JSONOptions } // SomeDecl represents a variable declaration statement. The symbols are variables. @@ -230,7 +230,7 @@ type ( Symbols []*Term `json:"symbols"` Location *Location `json:"location,omitempty"` - jsonFields map[string]bool + jsonOptions JSONOptions } Every struct { @@ -240,7 +240,7 @@ type ( Body Body `json:"body"` Location *Location `json:"location,omitempty"` - jsonFields map[string]bool + jsonOptions JSONOptions } // With represents a modifier on an expression. @@ -249,7 +249,7 @@ type ( Value *Term `json:"value"` Location *Location `json:"location,omitempty"` - jsonFields map[string]bool + jsonOptions JSONOptions } ) @@ -428,9 +428,10 @@ func (c *Comment) Equal(other *Comment) bool { return c.Location.Equal(other.Location) && bytes.Equal(c.Text, other.Text) } -func (c *Comment) exposeJSONFields(fields map[string]bool) { - // Note: this is not used for location since Comments have legacy JSON marshaling behavior - c.jsonFields = fields +func (c *Comment) setJSONOptions(opts JSONOptions) { + // Note: this is not used for location since Comments use default JSON marshaling + // behavior with struct field names in JSON. + c.jsonOptions = opts } // Compare returns an integer indicating whether pkg is less than, equal to, @@ -477,8 +478,8 @@ func (pkg *Package) String() string { return fmt.Sprintf("package %v", path) } -func (pkg *Package) exposeJSONFields(fields map[string]bool) { - pkg.jsonFields = fields +func (pkg *Package) setJSONOptions(opts JSONOptions) { + pkg.jsonOptions = opts } func (pkg *Package) MarshalJSON() ([]byte, error) { @@ -486,7 +487,7 @@ func (pkg *Package) MarshalJSON() ([]byte, error) { "path": pkg.Path, } - if showLocation, ok := pkg.jsonFields["location"]; ok && showLocation { + if pkg.jsonOptions.MarshalOptions.IncludeLocation.Package { if pkg.Location != nil { data["location"] = pkg.Location } @@ -587,8 +588,8 @@ func (imp *Import) String() string { return strings.Join(buf, " ") } -func (imp *Import) exposeJSONFields(fields map[string]bool) { - imp.jsonFields = fields +func (imp *Import) setJSONOptions(opts JSONOptions) { + imp.jsonOptions = opts } func (imp *Import) MarshalJSON() ([]byte, error) { @@ -600,7 +601,7 @@ func (imp *Import) MarshalJSON() ([]byte, error) { data["alias"] = imp.Alias } - if showLocation, ok := imp.jsonFields["location"]; ok && showLocation { + if imp.jsonOptions.MarshalOptions.IncludeLocation.Import { if imp.Location != nil { data["location"] = imp.Location } @@ -698,8 +699,8 @@ func (rule *Rule) String() string { return strings.Join(buf, " ") } -func (rule *Rule) exposeJSONFields(fields map[string]bool) { - rule.jsonFields = fields +func (rule *Rule) setJSONOptions(opts JSONOptions) { + rule.jsonOptions = opts } func (rule *Rule) MarshalJSON() ([]byte, error) { @@ -716,7 +717,7 @@ func (rule *Rule) MarshalJSON() ([]byte, error) { data["else"] = rule.Else } - if showLocation, ok := rule.jsonFields["location"]; ok && showLocation { + if rule.jsonOptions.MarshalOptions.IncludeLocation.Rule { if rule.Location != nil { data["location"] = rule.Location } @@ -914,13 +915,13 @@ func (head *Head) String() string { return buf.String() } -func (head *Head) exposeJSONFields(fields map[string]bool) { - head.jsonFields = fields +func (head *Head) setJSONOptions(opts JSONOptions) { + head.jsonOptions = opts } func (head *Head) MarshalJSON() ([]byte, error) { var loc *Location - if showLocation, ok := head.jsonFields["location"]; ok && showLocation { + if head.jsonOptions.MarshalOptions.IncludeLocation.Head { if head.Location != nil { loc = head.Location } @@ -1458,8 +1459,8 @@ func (expr *Expr) String() string { return strings.Join(buf, " ") } -func (expr *Expr) exposeJSONFields(fields map[string]bool) { - expr.jsonFields = fields +func (expr *Expr) setJSONOptions(opts JSONOptions) { + expr.jsonOptions = opts } func (expr *Expr) MarshalJSON() ([]byte, error) { @@ -1480,7 +1481,7 @@ func (expr *Expr) MarshalJSON() ([]byte, error) { data["negated"] = true } - if showLocation, ok := expr.jsonFields["location"]; ok && showLocation { + if expr.jsonOptions.MarshalOptions.IncludeLocation.Expr { if expr.Location != nil { data["location"] = expr.Location } @@ -1554,8 +1555,8 @@ func (d *SomeDecl) Hash() int { return termSliceHash(d.Symbols) } -func (d *SomeDecl) exposeJSONFields(fields map[string]bool) { - d.jsonFields = fields +func (d *SomeDecl) setJSONOptions(opts JSONOptions) { + d.jsonOptions = opts } func (d *SomeDecl) MarshalJSON() ([]byte, error) { @@ -1563,7 +1564,7 @@ func (d *SomeDecl) MarshalJSON() ([]byte, error) { "symbols": d.Symbols, } - if showLocation, ok := d.jsonFields["location"]; ok && showLocation { + if d.jsonOptions.MarshalOptions.IncludeLocation.SomeDecl { if d.Location != nil { data["location"] = d.Location } @@ -1628,8 +1629,8 @@ func (q *Every) KeyValueVars() VarSet { return vis.vars } -func (q *Every) exposeJSONFields(fields map[string]bool) { - q.jsonFields = fields +func (q *Every) setJSONOptions(opts JSONOptions) { + q.jsonOptions = opts } func (q *Every) MarshalJSON() ([]byte, error) { @@ -1640,7 +1641,7 @@ func (q *Every) MarshalJSON() ([]byte, error) { "body": q.Body, } - if showLocation, ok := q.jsonFields["location"]; ok && showLocation { + if q.jsonOptions.MarshalOptions.IncludeLocation.Every { if q.Location != nil { data["location"] = q.Location } @@ -1707,8 +1708,8 @@ func (w *With) SetLoc(loc *Location) { w.Location = loc } -func (w *With) exposeJSONFields(fields map[string]bool) { - w.jsonFields = fields +func (w *With) setJSONOptions(opts JSONOptions) { + w.jsonOptions = opts } func (w *With) MarshalJSON() ([]byte, error) { @@ -1717,7 +1718,7 @@ func (w *With) MarshalJSON() ([]byte, error) { "value": w.Value, } - if showLocation, ok := w.jsonFields["location"]; ok && showLocation { + if w.jsonOptions.MarshalOptions.IncludeLocation.With { if w.Location != nil { data["location"] = w.Location } diff --git a/ast/term.go b/ast/term.go index 26826b4506..e9bb2f1207 100644 --- a/ast/term.go +++ b/ast/term.go @@ -294,7 +294,7 @@ type Term struct { Value Value `json:"value"` // the value of the Term as represented in Go Location *Location `json:"location,omitempty"` // the location of the Term in the source - jsonFields map[string]bool + jsonOptions JSONOptions } // NewTerm returns a new Term object. @@ -419,8 +419,8 @@ func (term *Term) IsGround() bool { return term.Value.IsGround() } -func (term *Term) exposeJSONFields(fields map[string]bool) { - term.jsonFields = fields +func (term *Term) setJSONOptions(opts JSONOptions) { + term.jsonOptions = opts } // MarshalJSON returns the JSON encoding of the term. @@ -431,7 +431,7 @@ func (term *Term) MarshalJSON() ([]byte, error) { "type": TypeName(term.Value), "value": term.Value, } - if showLocation, ok := term.jsonFields["location"]; ok && showLocation { + if term.jsonOptions.MarshalOptions.IncludeLocation.Term { if term.Location != nil { d["location"] = term.Location } diff --git a/cmd/parse.go b/cmd/parse.go index e01c6024ee..43acf743bb 100644 --- a/cmd/parse.go +++ b/cmd/parse.go @@ -68,12 +68,29 @@ func parse(args []string, stdout io.Writer, stderr io.Writer) int { } } - result, err := loader.RegoWithOpts(args[0], ast.ParserOptions{ - ProcessAnnotation: true, - JSONFields: map[string]bool{ - "location": exposeLocation, - }, - }) + parserOpts := ast.ParserOptions{ProcessAnnotation: true} + if exposeLocation { + parserOpts.JSONOptions = &ast.JSONOptions{ + MarshalOptions: ast.JSONMarshalOptions{ + IncludeLocation: ast.NodeToggle{ + Term: true, + Package: true, + Comment: true, + Import: true, + Rule: true, + Head: true, + Expr: true, + SomeDecl: true, + Every: true, + With: true, + Annotations: true, + AnnotationsRef: true, + }, + }, + } + } + + result, err := loader.RegoWithOpts(args[0], parserOpts) if err != nil { _ = pr.JSON(stderr, pr.Output{Errors: pr.NewOutputErrors(err)}) return 1 From 6a1de087195aed3fad00b81e614f97fa19b1e378 Mon Sep 17 00:00:00 2001 From: Charlie Egan Date: Thu, 23 Feb 2023 11:53:09 +0000 Subject: [PATCH 5/8] Remove surplus unmarshal function tests Before this PR, a number of types didn't support the unmarshalling of location data found in their JSON representations. As the PR evolved, this become only relevant for the Term and Expr types. Other unmarshal tests are removed in this commit. Signed-off-by: Charlie Egan --- ast/marshal_test.go | 537 -------------------------------------------- 1 file changed, 537 deletions(-) diff --git a/ast/marshal_test.go b/ast/marshal_test.go index 3d2e85f291..1ab2c52bd1 100644 --- a/ast/marshal_test.go +++ b/ast/marshal_test.go @@ -1,7 +1,6 @@ package ast import ( - "bytes" "encoding/json" "testing" @@ -172,46 +171,6 @@ func TestPackage_MarshalJSON(t *testing.T) { } } -func TestPackage_UnmarshalJSON(t *testing.T) { - testCases := map[string]struct { - JSON string - ExpectedPackage *Package - }{ - "base case": { - JSON: `{"path":[]}`, - ExpectedPackage: &Package{ - Path: EmptyRef(), - }, - }, - "location case": { - JSON: `{"location":{"file":"example.rego","row":1,"col":2},"path":[]}`, - ExpectedPackage: &Package{ - Path: EmptyRef(), - Location: NewLocation([]byte{}, "example.rego", 1, 2), - }, - }, - } - - for name, data := range testCases { - t.Run(name, func(t *testing.T) { - var pkg Package - err := json.Unmarshal([]byte(data.JSON), &pkg) - if err != nil { - t.Fatal(err) - } - - if !pkg.Equal(data.ExpectedPackage) { - t.Fatalf("expected:\n%#v got\n%#v", data.ExpectedPackage, pkg) - } - if data.ExpectedPackage.Location != nil { - if !pkg.Location.Equal(data.ExpectedPackage.Location) { - t.Fatalf("expected location:\n%#v got\n%#v", data.ExpectedPackage, pkg) - } - } - }) - } -} - // TODO: Comment has inconsistent JSON field names starting with an upper case letter. Comment Location is // also always included for legacy reasons func TestComment_MarshalJSON(t *testing.T) { @@ -268,58 +227,6 @@ func TestComment_MarshalJSON(t *testing.T) { } } -// TODO: Comment has inconsistent JSON field names starting with an upper case letter. Comment Location is -// also always included for legacy reasons -func TestComment_UnmarshalJSON(t *testing.T) { - testCases := map[string]struct { - JSON string - ExpectedComment *Comment - }{ - "base case": { - JSON: `{"Text":"Y29tbWVudA=="}`, - ExpectedComment: &Comment{ - Text: []byte("comment"), - }, - }, - "location case": { - JSON: `{"Location":{"file":"example.rego","row":1,"col":2},"Text":"Y29tbWVudA=="}`, - ExpectedComment: &Comment{ - Text: []byte("comment"), - Location: NewLocation([]byte{}, "example.rego", 1, 2), - }, - }, - } - - for name, data := range testCases { - t.Run(name, func(t *testing.T) { - var comment Comment - err := json.Unmarshal([]byte(data.JSON), &comment) - if err != nil { - t.Fatal(err) - } - - equal := true - if data.ExpectedComment.Location != nil { - if comment.Location == nil { - t.Fatal("expected location to be non-nil") - } - - // comment.Equal will check the location too - if !comment.Equal(data.ExpectedComment) { - equal = false - } - } else { - if !bytes.Equal(comment.Text, data.ExpectedComment.Text) { - equal = false - } - } - if !equal { - t.Fatalf("expected:\n%#v got\n%#v", data.ExpectedComment, comment) - } - }) - } -} - func TestImport_MarshalJSON(t *testing.T) { testCases := map[string]struct { Import *Import @@ -393,57 +300,6 @@ func TestImport_MarshalJSON(t *testing.T) { } } -func TestImport_UnmarshalJSON(t *testing.T) { - testCases := map[string]struct { - JSON string - ExpectedImport *Import - }{ - "base case": { - JSON: `{"path":{"type":"string","value":"example"}}`, - ExpectedImport: func() *Import { - v, _ := InterfaceToValue("example") - term := Term{ - Value: v, - } - return &Import{Path: &term} - }(), - }, - "location case": { - JSON: `{"location":{"file":"example.rego","row":1,"col":2},"path":{"type":"string","value":"example"}}`, - ExpectedImport: func() *Import { - v, _ := InterfaceToValue("example") - term := Term{ - Value: v, - Location: NewLocation([]byte{}, "example.rego", 1, 2), - } - return &Import{ - Path: &term, - Location: NewLocation([]byte{}, "example.rego", 1, 2), - } - }(), - }, - } - - for name, data := range testCases { - t.Run(name, func(t *testing.T) { - var imp Import - err := json.Unmarshal([]byte(data.JSON), &imp) - if err != nil { - t.Fatal(err) - } - - if !imp.Equal(data.ExpectedImport) { - t.Fatalf("expected:\n%#v got\n%#v", data.ExpectedImport, imp) - } - if data.ExpectedImport.Location != nil { - if !imp.Location.Equal(data.ExpectedImport.Location) { - t.Fatalf("expected location:\n%#v got\n%#v", data.ExpectedImport, imp) - } - } - }) - } -} - func TestRule_MarshalJSON(t *testing.T) { rawModule := ` package foo @@ -511,62 +367,6 @@ func TestRule_MarshalJSON(t *testing.T) { } } -func TestRule_UnmarshalJSON(t *testing.T) { - rawModule := ` - package foo - - # comment - - allow { true } - ` - - module, err := ParseModuleWithOpts("example.rego", rawModule, ParserOptions{}) - if err != nil { - t.Fatal(err) - } - - rule := module.Rules[0] - // text is not marshalled to JSON so we just drop it in our examples - rule.Location.Text = nil - - testCases := map[string]struct { - JSON string - ExpectedRule *Rule - }{ - "base case": { - JSON: `{"body":[{"terms":{"type":"boolean","value":true},"location":{"file":"example.rego","row":6,"col":10},"index":0}],"head":{"name":"allow","value":{"type":"boolean","value":true},"location":{"file":"example.rego","row":6,"col":2},"ref":[{"type":"var","value":"allow"}]}}`, - ExpectedRule: func() *Rule { - r := rule.Copy() - r.Location = nil - return r - }(), - }, - "location case": { - JSON: `{"body":[{"terms":{"type":"boolean","value":true},"location":{"file":"example.rego","row":6,"col":10},"index":0}],"head":{"name":"allow","value":{"type":"boolean","value":true},"location":{"file":"example.rego","row":6,"col":2},"ref":[{"type":"var","value":"allow"}]},"location":{"file":"example.rego","row":6,"col":2}}`, - ExpectedRule: rule, - }, - } - - for name, data := range testCases { - t.Run(name, func(t *testing.T) { - var rule Rule - err := json.Unmarshal([]byte(data.JSON), &rule) - if err != nil { - t.Fatal(err) - } - - if !rule.Equal(data.ExpectedRule) { - t.Fatalf("expected:\n%#v got\n%#v", data.ExpectedRule, rule) - } - if data.ExpectedRule.Location != nil { - if !rule.Location.Equal(data.ExpectedRule.Location) { - t.Fatalf("expected location:\n%#v got\n%#v", data.ExpectedRule.Location, rule.Location) - } - } - }) - } -} - func TestHead_MarshalJSON(t *testing.T) { rawModule := ` package foo @@ -635,62 +435,6 @@ func TestHead_MarshalJSON(t *testing.T) { } } -func TestHead_UnmarshalJSON(t *testing.T) { - rawModule := ` - package foo - - # comment - - allow { true } - ` - - module, err := ParseModuleWithOpts("example.rego", rawModule, ParserOptions{}) - if err != nil { - t.Fatal(err) - } - - head := module.Rules[0].Head - // text is not marshalled to JSON so we just drop it in our examples - head.Location.Text = nil - - testCases := map[string]struct { - JSON string - ExpectedHead *Head - }{ - "base case": { - JSON: `{"name":"allow","value":{"type":"boolean","value":true},"ref":[{"type":"var","value":"allow"}]}`, - ExpectedHead: func() *Head { - h := head.Copy() - h.Location = nil - return h - }(), - }, - "location case": { - JSON: `{"name":"allow","value":{"type":"boolean","value":true},"ref":[{"type":"var","value":"allow"}],"location":{"file":"example.rego","row":6,"col":2}}`, - ExpectedHead: head, - }, - } - - for name, data := range testCases { - t.Run(name, func(t *testing.T) { - var head Head - err := json.Unmarshal([]byte(data.JSON), &head) - if err != nil { - t.Fatal(err) - } - - if !head.Equal(data.ExpectedHead) { - t.Fatalf("expected:\n%#v got\n%#v", data.ExpectedHead, head) - } - if data.ExpectedHead.Location != nil { - if !head.Location.Equal(data.ExpectedHead.Location) { - t.Fatalf("expected location:\n%#v got\n%#v", data.ExpectedHead.Location, head.Location) - } - } - }) - } -} - func TestExpr_MarshalJSON(t *testing.T) { rawModule := ` package foo @@ -864,53 +608,6 @@ func TestSomeDecl_MarshalJSON(t *testing.T) { } } -func TestSomeDecl_UnmarshalJSON(t *testing.T) { - v, _ := InterfaceToValue("example") - term := &Term{ - Value: v, - Location: NewLocation([]byte{}, "example.rego", 1, 2), - } - - testCases := map[string]struct { - JSON string - ExpectedSomeDecl *SomeDecl - }{ - "base case": { - JSON: `{"symbols":[{"type":"string","value":"example"}]}`, - ExpectedSomeDecl: &SomeDecl{ - Symbols: []*Term{term}, - }, - }, - "location case": { - JSON: `{"location":{"file":"example.rego","row":1,"col":2},"symbols":[{"type":"string","value":"example"}]}`, - ExpectedSomeDecl: &SomeDecl{ - Symbols: []*Term{term}, - Location: NewLocation([]byte{}, "example.rego", 1, 2), - }, - }, - } - - for name, data := range testCases { - t.Run(name, func(t *testing.T) { - var d SomeDecl - err := json.Unmarshal([]byte(data.JSON), &d) - if err != nil { - t.Fatal(err) - } - - if len(d.Symbols) != len(data.ExpectedSomeDecl.Symbols) { - t.Fatalf("expected:\n%#v got\n%#v", data.ExpectedSomeDecl.Symbols, d.Symbols) - } - - if data.ExpectedSomeDecl.Location != nil { - if !d.Location.Equal(data.ExpectedSomeDecl.Location) { - t.Fatalf("expected location:\n%#v got\n%#v", data.ExpectedSomeDecl.Location, d.Location) - } - } - }) - } -} - func TestEvery_MarshalJSON(t *testing.T) { rawModule := ` @@ -974,73 +671,6 @@ allow { } } -func TestEvery_UnmarshalJSON(t *testing.T) { - rawModule := ` -package foo - -import future.keywords.every - -allow { - every e in [1,2,3] { - e == 1 - } -} -` - - module, err := ParseModuleWithOpts("example.rego", rawModule, ParserOptions{}) - if err != nil { - t.Fatal(err) - } - - every, ok := module.Rules[0].Body[0].Terms.(*Every) - if !ok { - t.Fatal("expected every term") - } - - testCases := map[string]struct { - JSON string - ExpectedEvery *Every - }{ - "base case": { - JSON: `{"body":[{"index":0,"terms":[{"type":"ref","value":[{"type":"var","value":"equal"}]},{"type":"var","value":"e"},{"type":"number","value":1}]}],"domain":{"type":"array","value":[{"type":"number","value":1},{"type":"number","value":2},{"type":"number","value":3}]},"key":null,"value":{"type":"var","value":"e"}}`, - ExpectedEvery: func() *Every { - e := every.Copy() - e.Location = nil - return e - }(), - }, - "location case": { - JSON: `{"body":[{"index":0,"terms":[{"type":"ref","value":[{"type":"var","value":"equal"}]},{"type":"var","value":"e"},{"type":"number","value":1}]}],"domain":{"type":"array","value":[{"type":"number","value":1},{"type":"number","value":2},{"type":"number","value":3}]},"key":null,"location":{"file":"example.rego","row":7,"col":2},"value":{"type":"var","value":"e"}}`, - ExpectedEvery: func() *Every { - e := every.Copy() - // text is not marshalled to JSON so we just drop it in our examples - e.Location.Text = []byte{} - return e - }(), - }, - } - - for name, data := range testCases { - t.Run(name, func(t *testing.T) { - var e Every - err := json.Unmarshal([]byte(data.JSON), &e) - if err != nil { - t.Fatal(err) - } - - if e.String() != data.ExpectedEvery.String() { - t.Fatalf("expected:\n%#v got\n%#v", data.ExpectedEvery.String(), e.String()) - } - - if data.ExpectedEvery.Location != nil { - if !e.Location.Equal(data.ExpectedEvery.Location) { - t.Fatalf("expected location:\n%#v got\n%#v", data.ExpectedEvery.Location, e.Location) - } - } - }) - } -} - func TestWith_MarshalJSON(t *testing.T) { rawModule := ` @@ -1099,69 +729,6 @@ b { } } -func TestWith_UnmarshalJSON(t *testing.T) { - - rawModule := ` -package foo - -a {input} - -b { - a with input as 1 -} -` - - module, err := ParseModuleWithOpts("example.rego", rawModule, ParserOptions{}) - if err != nil { - t.Fatal(err) - } - - with := module.Rules[1].Body[0].With[0] - - testCases := map[string]struct { - JSON string - ExpectedWith *With - }{ - "base case": { - JSON: `{"target":{"type":"ref","value":[{"type":"var","value":"input"}]},"value":{"type":"number","value":1}}`, - ExpectedWith: func() *With { - w := with.Copy() - w.Location = nil - return w - }(), - }, - "location case": { - JSON: `{"location":{"file":"example.rego","row":7,"col":4},"target":{"type":"ref","value":[{"type":"var","value":"input"}]},"value":{"type":"number","value":1}}`, - ExpectedWith: func() *With { - e := with.Copy() - // text is not marshalled to JSON so we just drop it in our examples - e.Location.Text = []byte{} - return e - }(), - }, - } - - for name, data := range testCases { - t.Run(name, func(t *testing.T) { - var w With - err := json.Unmarshal([]byte(data.JSON), &w) - if err != nil { - t.Fatal(err) - } - - if w.String() != data.ExpectedWith.String() { - t.Fatalf("expected:\n%#v got\n%#v", data.ExpectedWith.String(), w.String()) - } - - if data.ExpectedWith.Location != nil { - if !w.Location.Equal(data.ExpectedWith.Location) { - t.Fatalf("expected location:\n%#v got\n%#v", data.ExpectedWith.Location, w.Location) - } - } - }) - } -} - func TestAnnotations_MarshalJSON(t *testing.T) { testCases := map[string]struct { @@ -1237,62 +804,6 @@ func TestAnnotations_MarshalJSON(t *testing.T) { } } -func TestAnnotations_UnmarshalJSON(t *testing.T) { - - testCases := map[string]struct { - JSON string - ExpectedAnnotations *Annotations - }{ - "base case": { - JSON: `{"custom":{"foo":"bar"},"description":"My desc","entrypoint":true,"organizations":["org1"],"scope":"rule","title":"My rule"}`, - ExpectedAnnotations: &Annotations{ - Scope: "rule", - Title: "My rule", - Entrypoint: true, - Organizations: []string{"org1"}, - Description: "My desc", - Custom: map[string]interface{}{ - "foo": "bar", - }, - }, - }, - "location case": { - JSON: `{"custom":{"foo":"bar"},"description":"My desc","entrypoint":true,"location":{"file":"example.rego","row":1,"col":4},"organizations":["org1"],"scope":"rule","title":"My rule"}`, - ExpectedAnnotations: &Annotations{ - Scope: "rule", - Title: "My rule", - Entrypoint: true, - Organizations: []string{"org1"}, - Description: "My desc", - Custom: map[string]interface{}{ - "foo": "bar", - }, - Location: NewLocation([]byte{}, "example.rego", 1, 4), - }, - }, - } - - for name, data := range testCases { - t.Run(name, func(t *testing.T) { - var a Annotations - err := json.Unmarshal([]byte(data.JSON), &a) - if err != nil { - t.Fatal(err) - } - - if a.String() != data.ExpectedAnnotations.String() { - t.Fatalf("expected:\n%#v got\n%#v", data.ExpectedAnnotations.String(), a.String()) - } - - if data.ExpectedAnnotations.Location != nil { - if !a.Location.Equal(data.ExpectedAnnotations.Location) { - t.Fatalf("expected location:\n%#v got\n%#v", data.ExpectedAnnotations.Location, a.Location) - } - } - }) - } -} - func TestAnnotationsRef_MarshalJSON(t *testing.T) { testCases := map[string]struct { @@ -1349,51 +860,3 @@ func TestAnnotationsRef_MarshalJSON(t *testing.T) { }) } } - -func TestAnnotationsRef_UnmarshalJSON(t *testing.T) { - - testCases := map[string]struct { - JSON string - ExpectedAnnotationsRef *AnnotationsRef - }{ - "base case": { - JSON: `{"annotations":{"scope":""},"path":[]}`, - ExpectedAnnotationsRef: &AnnotationsRef{ - Path: []*Term{}, - Annotations: &Annotations{}, - }, - }, - "location case": { - JSON: `{"custom":{"foo":"bar"},"description":"My desc","entrypoint":true,"location":{"file":"example.rego","row":1,"col":4},"organizations":["org1"],"scope":"rule","title":"My rule"}`, - ExpectedAnnotationsRef: &AnnotationsRef{ - Path: []*Term{}, - Annotations: &Annotations{}, - Location: NewLocation([]byte{}, "example.rego", 1, 4), - }, - }, - } - - for name, data := range testCases { - t.Run(name, func(t *testing.T) { - var a AnnotationsRef - err := json.Unmarshal([]byte(data.JSON), &a) - if err != nil { - t.Fatal(err) - } - - if got, exp := len(a.Path), len(data.ExpectedAnnotationsRef.Path); exp != got { - t.Fatalf("expected:\n%#v got\n%#v", exp, got) - } - - if got, exp := a.Annotations.String(), data.ExpectedAnnotationsRef.Annotations.String(); exp != got { - t.Fatalf("expected:\n%#v got\n%#v", exp, got) - } - - if data.ExpectedAnnotationsRef.Location != nil { - if !a.Location.Equal(data.ExpectedAnnotationsRef.Location) { - t.Fatalf("expected location:\n%#v got\n%#v", data.ExpectedAnnotationsRef.Location, a.Location) - } - } - }) - } -} From c7bea96fd32a391c54637420caf5b8ced4292780 Mon Sep 17 00:00:00 2001 From: Charlie Egan Date: Thu, 2 Mar 2023 11:44:23 +0000 Subject: [PATCH 6/8] Improve parse command tests This adds tests for: * pretty output * json output * json location and comment flags This required the parse function to be adjusted to make the options configurable in test cases. Signed-off-by: Charlie Egan --- cmd/parse.go | 20 +++-- cmd/parse_test.go | 212 ++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 218 insertions(+), 14 deletions(-) diff --git a/cmd/parse.go b/cmd/parse.go index 43acf743bb..13bb569b2e 100644 --- a/cmd/parse.go +++ b/cmd/parse.go @@ -24,10 +24,12 @@ const ( parseFormatJSON = "json" ) -var parseParams = struct { +type parseParams struct { format *util.EnumFlag jsonInclude string -}{ +} + +var configuredParseParams = parseParams{ format: util.NewEnumFlag(parseFormatPretty, []string{parseFormatPretty, parseFormatJSON}), jsonInclude: "", } @@ -43,18 +45,18 @@ var parseCommand = &cobra.Command{ return nil }, Run: func(_ *cobra.Command, args []string) { - os.Exit(parse(args, os.Stdout, os.Stderr)) + os.Exit(parse(args, &configuredParseParams, os.Stdout, os.Stderr)) }, } -func parse(args []string, stdout io.Writer, stderr io.Writer) int { +func parse(args []string, params *parseParams, stdout io.Writer, stderr io.Writer) int { if len(args) == 0 { return 0 } exposeLocation := false exposeComments := true - for _, opt := range strings.Split(parseParams.jsonInclude, ",") { + for _, opt := range strings.Split(params.jsonInclude, ",") { value := true if strings.HasPrefix(opt, "-") { value = false @@ -100,7 +102,7 @@ func parse(args []string, stdout io.Writer, stderr io.Writer) int { result.Parsed.Comments = nil } - switch parseParams.format.String() { + switch params.format.String() { case parseFormatJSON: bs, err := json.MarshalIndent(result.Parsed, "", " ") if err != nil { @@ -108,7 +110,7 @@ func parse(args []string, stdout io.Writer, stderr io.Writer) int { return 1 } - fmt.Println(string(bs)) + fmt.Fprint(stdout, string(bs)+"\n") default: if err != nil { fmt.Fprintln(stderr, err) @@ -121,8 +123,8 @@ func parse(args []string, stdout io.Writer, stderr io.Writer) int { } func init() { - parseCommand.Flags().VarP(parseParams.format, "format", "f", "set output format") - parseCommand.Flags().StringVarP(&parseParams.jsonInclude, "json-include", "", "", "select optional elements, current options: locations, comments. E.g. --json-include locations,-comments will include locations and exclude comments.") + parseCommand.Flags().VarP(configuredParseParams.format, "format", "f", "set output format") + parseCommand.Flags().StringVarP(&configuredParseParams.jsonInclude, "json-include", "", "", "select optional elements, current options: locations, comments. E.g. --json-include locations,-comments will include locations and exclude comments.") RootCommand.AddCommand(parseCommand) } diff --git a/cmd/parse_test.go b/cmd/parse_test.go index 40652b78df..3ef99e58c8 100644 --- a/cmd/parse_test.go +++ b/cmd/parse_test.go @@ -3,8 +3,10 @@ package cmd import ( "bytes" "path/filepath" + "strings" "testing" + "github.com/open-policy-agent/opa/util" "github.com/open-policy-agent/opa/util/test" ) @@ -16,13 +18,32 @@ func TestParseExit0(t *testing.T) { p = 1 `, } - errc, _, stderr := testParse(t, files) + errc, stdout, stderr, _ := testParse(t, files, &configuredParseParams) if errc != 0 { t.Fatalf("Expected exit code 0, got %v", errc) } if len(stderr) > 0 { t.Fatalf("Expected no stderr output, got:\n%s\n", string(stderr)) } + + expectedOutput := `module + package + ref + data + "x" + rule + head + ref + p + 1 + body + expr index=0 + true +` + + if got, want := string(stdout), expectedOutput; got != want { + t.Fatalf("Expected output\n%v\n, got\n%v", want, got) + } } func TestParseExit1(t *testing.T) { @@ -30,7 +51,7 @@ func TestParseExit1(t *testing.T) { files := map[string]string{ "x.rego": `???`, } - errc, _, stderr := testParse(t, files) + errc, _, stderr, _ := testParse(t, files, &configuredParseParams) if errc != 1 { t.Fatalf("Expected exit code 1, got %v", errc) } @@ -39,21 +60,202 @@ func TestParseExit1(t *testing.T) { } } +func TestParseJSONOutput(t *testing.T) { + + files := map[string]string{ + "x.rego": `package x + + p = 1 + `, + } + errc, stdout, stderr, _ := testParse(t, files, &parseParams{ + format: util.NewEnumFlag(parseFormatJSON, []string{parseFormatPretty, parseFormatJSON}), + }) + if errc != 0 { + t.Fatalf("Expected exit code 0, got %v", errc) + } + if len(stderr) > 0 { + t.Fatalf("Expected no stderr output, got:\n%s\n", string(stderr)) + } + + expectedOutput := `{ + "package": { + "path": [ + { + "type": "var", + "value": "data" + }, + { + "type": "string", + "value": "x" + } + ] + }, + "rules": [ + { + "body": [ + { + "index": 0, + "terms": { + "type": "boolean", + "value": true + } + } + ], + "head": { + "name": "p", + "value": { + "type": "number", + "value": 1 + }, + "ref": [ + { + "type": "var", + "value": "p" + } + ] + } + } + ] +} +` + + if got, want := string(stdout), expectedOutput; got != want { + t.Fatalf("Expected output\n%v\n, got\n%v", want, got) + } +} + +func TestParseJSONOutputWithLocations(t *testing.T) { + + files := map[string]string{ + "x.rego": `package x + + p = 1 + `, + } + errc, stdout, stderr, tempDirPath := testParse(t, files, &parseParams{ + format: util.NewEnumFlag(parseFormatJSON, []string{parseFormatPretty, parseFormatJSON}), + jsonInclude: "locations", + }) + if errc != 0 { + t.Fatalf("Expected exit code 0, got %v", errc) + } + if len(stderr) > 0 { + t.Fatalf("Expected no stderr output, got:\n%s\n", string(stderr)) + } + + expectedOutput := strings.Replace(`{ + "package": { + "location": { + "file": "TEMPDIR/x.rego", + "row": 1, + "col": 1 + }, + "path": [ + { + "location": { + "file": "TEMPDIR/x.rego", + "row": 1, + "col": 9 + }, + "type": "var", + "value": "data" + }, + { + "location": { + "file": "TEMPDIR/x.rego", + "row": 1, + "col": 9 + }, + "type": "string", + "value": "x" + } + ] + }, + "rules": [ + { + "body": [ + { + "index": 0, + "terms": { + "type": "boolean", + "value": true + } + } + ], + "head": { + "name": "p", + "value": { + "location": { + "file": "TEMPDIR/x.rego", + "row": 3, + "col": 7 + }, + "type": "number", + "value": 1 + }, + "ref": [ + { + "type": "var", + "value": "p" + } + ] + } + } + ] +} +`, "TEMPDIR", tempDirPath, -1) + + if got, want := string(stdout), expectedOutput; got != want { + t.Fatalf("Expected output\n%v\n, got\n%v", want, got) + } +} + +func TestParseJSONOutputComments(t *testing.T) { + + files := map[string]string{ + "x.rego": `package x + + # comment + p = 1 + `, + } + errc, stdout, stderr, _ := testParse(t, files, &parseParams{ + format: util.NewEnumFlag(parseFormatJSON, []string{parseFormatPretty, parseFormatJSON}), + jsonInclude: "comments", + }) + if errc != 0 { + t.Fatalf("Expected exit code 0, got %v", errc) + } + if len(stderr) > 0 { + t.Fatalf("Expected no stderr output, got:\n%s\n", string(stderr)) + } + + expectedCommentTextValue := "IGNvbW1lbnQ=" + + if !strings.Contains(string(stdout), expectedCommentTextValue) { + t.Fatalf("Comment text value %q missing in output: %s", expectedCommentTextValue, string(stdout)) + } +} + // Runs parse and returns the exit code, stdout, and stderr contents -func testParse(t *testing.T, files map[string]string) (int, []byte, []byte) { +func testParse(t *testing.T, files map[string]string, params *parseParams) (int, []byte, []byte, string) { t.Helper() stdout := new(bytes.Buffer) stderr := new(bytes.Buffer) var errc int + var tempDirUsed string test.WithTempFS(files, func(path string) { var args []string for file := range files { args = append(args, filepath.Join(path, file)) } - errc = parse(args, stdout, stderr) + errc = parse(args, params, stdout, stderr) + + tempDirUsed = path }) - return errc, stdout.Bytes(), stderr.Bytes() + return errc, stdout.Bytes(), stderr.Bytes(), tempDirUsed } From b872438cea2c7adba496534a2090e9c57d502a94 Mon Sep 17 00:00:00 2001 From: Charlie Egan Date: Thu, 2 Mar 2023 11:47:37 +0000 Subject: [PATCH 7/8] Clarify term UnmarshalJSON functionality Signed-off-by: Charlie Egan --- ast/term.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ast/term.go b/ast/term.go index e9bb2f1207..79e21ccae6 100644 --- a/ast/term.go +++ b/ast/term.go @@ -444,7 +444,7 @@ func (term *Term) String() string { } // UnmarshalJSON parses the byte array and stores the result in term. -// Specialized unmarshalling is required to handle Value. +// Specialized unmarshalling is required to handle Value and Location. func (term *Term) UnmarshalJSON(bs []byte) error { v := map[string]interface{}{} if err := util.UnmarshalJSON(bs, &v); err != nil { From 2c38a7cd8978dde44841425ec8d8c2b7c54f8b39 Mon Sep 17 00:00:00 2001 From: Charlie Egan Date: Thu, 2 Mar 2023 15:35:14 +0000 Subject: [PATCH 8/8] Make default flag behavior clearer Signed-off-by: Charlie Egan --- cmd/parse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/parse.go b/cmd/parse.go index 13bb569b2e..ef529c8ea7 100644 --- a/cmd/parse.go +++ b/cmd/parse.go @@ -124,7 +124,7 @@ func parse(args []string, params *parseParams, stdout io.Writer, stderr io.Write func init() { parseCommand.Flags().VarP(configuredParseParams.format, "format", "f", "set output format") - parseCommand.Flags().StringVarP(&configuredParseParams.jsonInclude, "json-include", "", "", "select optional elements, current options: locations, comments. E.g. --json-include locations,-comments will include locations and exclude comments.") + parseCommand.Flags().StringVarP(&configuredParseParams.jsonInclude, "json-include", "", "", "include or exclude optional elements. By default comments are included. Current options: locations, comments. E.g. --json-include locations,-comments will include locations and exclude comments.") RootCommand.AddCommand(parseCommand) }