internal/encoding: support input validation

Also fixes a bug in go.go, which did not obey
the `json:"-"` field tag.

This does not yet introduce a file extension for
possible combinations like cue+data.

Closes #130

Change-Id: I03f18afb64e48d84cf6a645f5fb5f7c91299b032
Reviewed-on: https://cue-review.googlesource.com/c/cue/+/5261
Reviewed-by: Marcel van Lohuizen <mpvl@golang.org>

cmd/cue/cmd/common.go

@@ -448,6 +448,7 @@ func (b *buildPlan) parseFlags() (err error) {
 		Stdin:     stdin,
 		Stdout:    b.cmd.OutOrStdout(),
 		ProtoPath: flagProtoPath.StringArray(b.cmd),
+		AllErrors: flagAllErrors.Bool(b.cmd),
 	}
 	return nil
 }

cmd/cue/cmd/flags.go

@@ -21,15 +21,16 @@ import (
 
 // Common flags
 const (
-	flagAll      flagName = "all"
-	flagDryrun   flagName = "dryrun"
-	flagVerbose  flagName = "verbose"
-	flagTrace    flagName = "trace"
-	flagForce    flagName = "force"
-	flagIgnore   flagName = "ignore"
-	flagSimplify flagName = "simplify"
-	flagPackage  flagName = "package"
-	flagTags     flagName = "tags"
+	flagAll       flagName = "all"
+	flagDryrun    flagName = "dryrun"
+	flagVerbose   flagName = "verbose"
+	flagAllErrors flagName = "all-errors"
+	flagTrace     flagName = "trace"
+	flagForce     flagName = "force"
+	flagIgnore    flagName = "ignore"
+	flagSimplify  flagName = "simplify"
+	flagPackage   flagName = "package"
+	flagTags      flagName = "tags"
 
 	flagExpression flagName = "expression"
 	flagSchema     flagName = "schema"
@@ -61,6 +62,7 @@ func addGlobalFlags(f *pflag.FlagSet) {
 		"proceed in the presence of errors")
 	f.BoolP(string(flagVerbose), "v", false,
 		"print information about progress")
+	f.BoolP(string(flagAllErrors), "E", false, "print all available errors")
 }
 
 func addOrphanFlags(f *pflag.FlagSet) {

cmd/cue/cmd/testdata/script/file_forms.txt

@@ -0,0 +1,39 @@
+cue eval data: foo.cue
+cmp stdout expect-data-foo
+
+! cue eval data: bar.cue
+cmp stderr expect-data-bar
+
+cue eval graph: bar.cue
+cmp stdout expect-graph-bar
+
+-- foo.cue --
+a: 4
+b: {
+    c: 1
+}
+// Duplicates are still allowed.
+b: {
+    d: 2
+}
+
+-- bar.cue --
+a: 4
+b: {
+    c: a
+}
+
+-- expect-data-foo --
+a: 4
+b: {
+    c: 1
+    d: 2
+}
+-- expect-data-bar --
+references not allowed in data mode:
+    ./bar.cue:3:8
+-- expect-graph-bar --
+a: 4
+b: {
+    c: 4
+}

cmd/cue/cmd/testdata/script/help_cmd.txt

@@ -225,9 +225,10 @@ Flags:
   -t, --tags stringArray   set the value of a tagged field
 
 Global Flags:
-  -i, --ignore     proceed in the presence of errors
-  -s, --simplify   simplify output
-      --trace      trace computation
-  -v, --verbose    print information about progress
+  -E, --all-errors   print all available errors
+  -i, --ignore       proceed in the presence of errors
+  -s, --simplify     simplify output
+      --trace        trace computation
+  -v, --verbose      print information about progress
 
 Use "cue cmd [command] --help" for more information about a command.

cmd/cue/cmd/testdata/script/help_hello.txt

@@ -29,7 +29,8 @@ Flags:
   -h, --help   help for hello
 
 Global Flags:
-  -i, --ignore     proceed in the presence of errors
-  -s, --simplify   simplify output
-      --trace      trace computation
-  -v, --verbose    print information about progress
+  -E, --all-errors   print all available errors
+  -i, --ignore       proceed in the presence of errors
+  -s, --simplify     simplify output
+      --trace        trace computation
+  -v, --verbose      print information about progress

cue/build/file.go

@@ -23,7 +23,7 @@ type File struct {
 	Form           Form              `json:"form,omitempty"`
 	Tags           map[string]string `json:"tags,omitempty"` // code=go
 
-	Source interface{} // TODO: swap out with concrete type.
+	Source interface{} `json:"-"` // TODO: swap out with concrete type.
 }
 
 // A Encoding indicates a file format for representing a program.

cue/go.go

@@ -362,6 +362,9 @@ func convertRec(ctx *context, src source, nilIsTop bool, x interface{}) evaluate
 				if !nilIsTop && isNil(val) {
 					continue
 				}
+				if tag, _ := t.Tag.Lookup("json"); tag == "-" {
+					continue
+				}
 				if isOmitEmpty(&t) && isZero(val) {
 					continue
 				}

cue/go_test.go

@@ -113,6 +113,13 @@ func TestConvert(t *testing.T) {
 			b int
 		}{3, 4},
 		"<0>{}",
+	}, {
+		struct {
+			A int
+			B int `json:"-"`
+			C int `json:",omitempty"`
+		}{3, 4, 0},
+		"<0>{A: 3}",
 	}, {
 		struct {
 			A int

internal/encoding/encoding.go

@@ -25,15 +25,18 @@ import (
 	"cuelang.org/go/cue"
 	"cuelang.org/go/cue/ast"
 	"cuelang.org/go/cue/build"
+	"cuelang.org/go/cue/errors"
 	"cuelang.org/go/cue/format"
 	"cuelang.org/go/cue/parser"
+	"cuelang.org/go/cue/token"
 	"cuelang.org/go/encoding/json"
 	"cuelang.org/go/encoding/protobuf"
 	"cuelang.org/go/internal/filetypes"
 	"cuelang.org/go/internal/third_party/yaml"
 )
 
 type Decoder struct {
+	cfg      *Config
 	closer   io.Closer
 	next     func() (ast.Expr, error)
 	expr     ast.Expr
@@ -103,8 +106,9 @@ type Config struct {
 	Stdin  io.Reader
 	Stdout io.Writer
 
-	Force  bool // overwrite existing files.
-	Stream bool // will potentially write more than one document per file
+	Force     bool // overwrite existing files.
+	Stream    bool // will potentially write more than one document per file
+	AllErrors bool
 
 	EscapeHTML bool
 	ProtoPath  []string
@@ -115,18 +119,21 @@ type Config struct {
 // type of f must be a data type, but does not have to be an encoding that
 // can stream. stdin is used in case the file is "-".
 func NewDecoder(f *build.File, cfg *Config) *Decoder {
-	i := &Decoder{filename: f.Filename}
+	if cfg == nil {
+		cfg = &Config{}
+	}
+	i := &Decoder{filename: f.Filename, cfg: cfg}
 	i.next = func() (ast.Expr, error) {
 		if i.err != nil {
 			return nil, i.err
 		}
 		return nil, io.EOF
 	}
 
-	if f, ok := f.Source.(*ast.File); ok {
-		i.file = f
+	if file, ok := f.Source.(*ast.File); ok {
+		i.file = file
 		i.closer = ioutil.NopCloser(strings.NewReader(""))
-		// TODO: verify input format for CUE.
+		i.validate(file, f)
 		return i
 	}
 
@@ -141,7 +148,7 @@ func NewDecoder(f *build.File, cfg *Config) *Decoder {
 	switch f.Encoding {
 	case build.CUE:
 		i.file, i.err = parser.ParseFile(path, r, parser.ParseComments)
-		// TODO: verify input format
+		i.validate(i.file, f)
 	case build.JSON, build.JSONL:
 		i.next = json.NewDecoder(nil, path, r).Extract
 		i.Next()
@@ -186,3 +193,121 @@ func reader(f *build.File, stdin io.Reader) (io.ReadCloser, error) {
 	}
 	return os.Open(f.Filename)
 }
+
+func shouldValidate(i *filetypes.FileInfo) bool {
+	// TODO: We ignore attributes for now. They should be enabled by default.
+	return false ||
+		!i.Definitions ||
+		!i.Data ||
+		!i.Optional ||
+		!i.Constraints ||
+		!i.References ||
+		!i.Cycles ||
+		!i.KeepDefaults ||
+		!i.Incomplete ||
+		!i.Imports ||
+		!i.Docs
+}
+
+type validator struct {
+	allErrors bool
+	count     int
+	errs      errors.Error
+	fileinfo  *filetypes.FileInfo
+}
+
+func (d *Decoder) validate(f *ast.File, b *build.File) {
+	if d.err != nil {
+		return
+	}
+	fi, err := filetypes.FromFile(b, filetypes.Input)
+	if err != nil {
+		d.err = err
+		return
+	}
+	if !shouldValidate(fi) {
+		return
+	}
+
+	v := validator{fileinfo: fi, allErrors: d.cfg.AllErrors}
+	ast.Walk(f, v.validate, nil)
+	d.err = v.errs
+}
+
+func (v *validator) validate(n ast.Node) bool {
+	if v.count > 10 {
+		return false
+	}
+
+	i := v.fileinfo
+
+	// TODO: Cycles
+
+	ok := true
+	check := func(n ast.Node, option bool, s string, cond bool) {
+		if !option && cond {
+			v.errs = errors.Append(v.errs, errors.Newf(n.Pos(),
+				"%s not allowed in %s mode", s, v.fileinfo.Form))
+			v.count++
+			ok = false
+		}
+	}
+
+	// For now we don't make any distinction between these modes.
+
+	constraints := i.Constraints && i.Incomplete && i.Optional && i.References
+
+	check(n, i.Docs, "comments", len(ast.Comments(n)) > 0)
+
+	switch x := n.(type) {
+	case *ast.CommentGroup:
+		check(n, i.Docs, "comments", len(ast.Comments(n)) > 0)
+		return false
+
+	case *ast.ImportDecl, *ast.ImportSpec:
+		check(n, i.Imports, "imports", true)
+
+	case *ast.Field:
+		check(n, i.Definitions, "definitions", x.Token == token.ISA)
+		check(n, i.Data, "regular fields", x.Token != token.ISA)
+		check(n, constraints, "optional fields", x.Optional != token.NoPos)
+
+		_, _, err := ast.LabelName(x.Label)
+		check(n, constraints, "optional fields", err != nil)
+
+		check(n, i.Attributes, "attributes", len(x.Attrs) > 0)
+		ast.Walk(x.Value, v.validate, nil)
+		return false
+
+	case *ast.UnaryExpr:
+		switch x.Op {
+		case token.MUL:
+			check(n, i.KeepDefaults, "default values", true)
+		case token.SUB, token.ADD:
+			// The parser represents negative numbers as an unary expression.
+			// Allow one `-` or `+`.
+			_, ok := x.X.(*ast.BasicLit)
+			check(n, constraints, "expressions", !ok)
+		case token.LSS, token.LEQ, token.EQL, token.GEQ, token.GTR,
+			token.NEQ, token.NMAT, token.MAT:
+			check(n, constraints, "constraints", true)
+		default:
+			check(n, constraints, "expressions", true)
+		}
+
+	case *ast.BinaryExpr, *ast.ParenExpr, *ast.IndexExpr, *ast.SliceExpr,
+		*ast.CallExpr, *ast.Comprehension, *ast.ListComprehension,
+		*ast.Interpolation:
+		check(n, constraints, "expressions", true)
+
+	case *ast.Ellipsis:
+		check(n, constraints, "ellipsis", true)
+
+	case *ast.Ident, *ast.SelectorExpr, *ast.Alias:
+		check(n, i.References, "references", true)
+
+	default:
+		// Other types are either always okay or handled elsewhere.
+	}
+	return ok
+}