From 841567df66d7d89eacfb04a4533c13850529ee70 Mon Sep 17 00:00:00 2001 From: Marcel van Lohuizen Date: Thu, 25 Mar 2021 17:10:52 +0100 Subject: [PATCH] encoding/protobuf/textproto: add encoder Doesn't really copy positioning and all comments, but the textproto API is sort of hit or miss anyway whether this information is supported at all. So good enough for now. Issue #5 Change-Id: Ia0d09a0c4b92756f68c2a09a114311b363fef33a --- encoding/protobuf/pbinternal/attribute.go | 8 +- encoding/protobuf/pbinternal/symbol.go | 44 ++++ encoding/protobuf/textproto/encoder.go | 203 ++++++++++++++++++ encoding/protobuf/textproto/encoder_test.go | 73 +++++++ .../textproto/testdata/encoder/enum.txtar | 76 +++++++ .../textproto/testdata/encoder/list.txtar | 24 +++ .../textproto/testdata/encoder/map.txtar | 44 ++++ .../textproto/testdata/encoder/simple.txtar | 23 ++ go.sum | 2 + 9 files changed, 496 insertions(+), 1 deletion(-) create mode 100644 encoding/protobuf/textproto/encoder.go create mode 100644 encoding/protobuf/textproto/encoder_test.go create mode 100644 encoding/protobuf/textproto/testdata/encoder/enum.txtar create mode 100644 encoding/protobuf/textproto/testdata/encoder/list.txtar create mode 100644 encoding/protobuf/textproto/testdata/encoder/map.txtar create mode 100644 encoding/protobuf/textproto/testdata/encoder/simple.txtar diff --git a/encoding/protobuf/pbinternal/attribute.go b/encoding/protobuf/pbinternal/attribute.go index 73fb34904..e3d36bd1b 100644 --- a/encoding/protobuf/pbinternal/attribute.go +++ b/encoding/protobuf/pbinternal/attribute.go @@ -16,6 +16,8 @@ package pbinternal import ( "strings" + "unicode" + "unicode/utf8" "cuelang.org/go/cue" ) @@ -50,6 +52,8 @@ type Info struct { ValueType ValueType Type string + IsEnum bool + // For maps only KeyType ValueType // only for maps KeyTypeString string @@ -97,7 +101,7 @@ func FromValue(name string, v cue.Value) (info Info, err error) { case cue.StructKind: if strings.HasPrefix(info.Type, "map[") { - a := strings.SplitN(info.Type[len("map["):], ",", 2) + a := strings.SplitN(info.Type[len("map["):], "]", 2) info.KeyTypeString = strings.TrimSpace(a[0]) switch info.KeyTypeString { case "string": @@ -133,6 +137,8 @@ func FromValue(name string, v cue.Value) (info Info, err error) { case cue.IntKind: info.ValueType = Int + r, _ := utf8.DecodeRuneInString(info.Type) + info.IsEnum = unicode.In(r, unicode.Upper) case cue.FloatKind, cue.NumberKind: info.ValueType = Float diff --git a/encoding/protobuf/pbinternal/symbol.go b/encoding/protobuf/pbinternal/symbol.go index 8a6f5f014..848028070 100644 --- a/encoding/protobuf/pbinternal/symbol.go +++ b/encoding/protobuf/pbinternal/symbol.go @@ -64,3 +64,47 @@ func matchBySymbol(v cue.Value, name string, x *ast.BasicLit) bool { return false } + +// MatchByInt finds a symbol for a given enum value and sets it in x. +func MatchByInt(v cue.Value, val int64) string { + if op, a := v.Expr(); op == cue.AndOp { + for _, v := range a { + if s := MatchByInt(v, val); s != "" { + return s + } + } + } + v = cue.Dereference(v) + return matchByInt(v, val) +} + +func matchByInt(v cue.Value, val int64) string { + switch op, a := v.Expr(); op { + case cue.OrOp, cue.AndOp: + for _, v := range a { + if s := matchByInt(v, val); s != "" { + return s + } + } + + default: + if i, err := v.Int64(); err != nil || i != val { + break + } + + _, path := v.ReferencePath() + a := path.Selectors() + if len(a) == 0 { + break + } + + sel := a[len(a)-1] + if !sel.IsDefinition() { + break + } + + return sel.String()[1:] + } + + return "" +} diff --git a/encoding/protobuf/textproto/encoder.go b/encoding/protobuf/textproto/encoder.go new file mode 100644 index 000000000..dcb9ed3e0 --- /dev/null +++ b/encoding/protobuf/textproto/encoder.go @@ -0,0 +1,203 @@ +// Copyright 2021 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package textproto + +import ( + "fmt" + "strings" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/errors" + "cuelang.org/go/encoding/protobuf/pbinternal" + + "github.com/protocolbuffers/txtpbfmt/ast" + pbast "github.com/protocolbuffers/txtpbfmt/ast" + "github.com/protocolbuffers/txtpbfmt/parser" +) + +// Encoder marshals CUE into text proto. +// +type Encoder struct { + // Schema +} + +// NewEncoder returns a new encoder, where the given options are default +// options. +func NewEncoder(options ...Option) *Encoder { + return &Encoder{} +} + +// Encode converts a CUE value to a text proto file. +// +// Fields do not need to have a @protobuf attribute except for in the following +// cases: +// +// - it is explicitly required that only fields with an attribute are exported +// - a struct represents a Protobuf map +// - custom naming +// +func (e *Encoder) Encode(v cue.Value, options ...Option) ([]byte, error) { + n := &pbast.Node{} + enc := &encoder{} + + enc.encodeMsg(n, v) + + if enc.errs != nil { + return nil, enc.errs + } + + // Pretty printing does not do errors, and returns a string (why o why?). + s := parser.Pretty(n.Children, 0) + return []byte(s), nil +} + +type encoder struct { + errs errors.Error +} + +func (e *encoder) addErr(err error) { + e.errs = errors.Append(e.errs, errors.Promote(err, "textproto")) +} + +func (e *encoder) encodeMsg(parent *pbast.Node, v cue.Value) { + i, err := v.Fields() + if err != nil { + e.addErr(err) + return + } + for i.Next() { + v := i.Value() + if !v.IsConcrete() { + continue + } + + info, err := pbinternal.FromIter(i) + if err != nil { + e.addErr(err) + } + + switch info.CompositeType { + case pbinternal.List: + elems, err := v.List() + if err != nil { + e.addErr(err) + return + } + for first := true; elems.Next(); first = false { + n := &pbast.Node{Name: info.Name} + if first { + copyMeta(n, v) + } + elem := elems.Value() + copyMeta(n, elem) + parent.Children = append(parent.Children, n) + e.encodeValue(n, elem) + } + + case pbinternal.Map: + i, err := v.Fields() + if err != nil { + e.addErr(err) + return + } + for first := true; i.Next(); first = false { + n := &pbast.Node{Name: info.Name} + if first { + copyMeta(n, v) + } + parent.Children = append(parent.Children, n) + var key *pbast.Node + switch info.KeyType { + case pbinternal.String, pbinternal.Bytes: + key = pbast.StringNode("key", i.Label()) + default: + key = &pbast.Node{ + Name: "key", + Values: []*ast.Value{{Value: i.Label()}}, + } + } + n.Children = append(n.Children, key) + + value := &pbast.Node{Name: "value"} + e.encodeValue(value, i.Value()) + n.Children = append(n.Children, value) + } + + default: + n := &pbast.Node{Name: info.Name} + copyMeta(n, v) + e.encodeValue(n, v) + // Don't add if there are no values or children. + parent.Children = append(parent.Children, n) + } + } +} + +// copyMeta copies metadata from nodes to values. +// +// TODO: also copy positions. The textproto API is rather messy and complex, +// though, and so far it seems to be quite buggy too. Not sure if it is worth +// the effort. +func copyMeta(x *pbast.Node, v cue.Value) { + for _, doc := range v.Doc() { + s := strings.TrimRight(doc.Text(), "\n") + for _, c := range strings.Split(s, "\n") { + x.PreComments = append(x.PreComments, "# "+c) + } + } +} + +func (e *encoder) encodeValue(n *pbast.Node, v cue.Value) { + var value string + switch v.Kind() { + case cue.StructKind: + e.encodeMsg(n, v) + + case cue.StringKind: + s, err := v.String() + if err != nil { + e.addErr(err) + } + sn := pbast.StringNode("foo", s) + n.Values = append(n.Values, sn.Values...) + + case cue.BytesKind: + b, err := v.Bytes() + if err != nil { + e.addErr(err) + } + sn := pbast.StringNode("foo", string(b)) + n.Values = append(n.Values, sn.Values...) + + case cue.BoolKind: + value = fmt.Sprint(v) + n.Values = append(n.Values, &pbast.Value{Value: value}) + + case cue.IntKind, cue.FloatKind, cue.NumberKind: + d, _ := v.Decimal() + value := d.String() + + if info, _ := pbinternal.FromValue("", v); !info.IsEnum { + } else if i, err := v.Int64(); err != nil { + } else if s := pbinternal.MatchByInt(v, i); s != "" { + value = s + } + + n.Values = append(n.Values, &pbast.Value{Value: value}) + + default: + e.addErr(errors.Newf(v.Pos(), "textproto: unknown type %v", v.Kind())) + } +} diff --git a/encoding/protobuf/textproto/encoder_test.go b/encoding/protobuf/textproto/encoder_test.go new file mode 100644 index 000000000..05430881d --- /dev/null +++ b/encoding/protobuf/textproto/encoder_test.go @@ -0,0 +1,73 @@ +// Copyright 2021 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package textproto_test + +import ( + "strings" + "testing" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/errors" + "cuelang.org/go/encoding/protobuf/textproto" + "cuelang.org/go/internal/cuetest" + "cuelang.org/go/internal/cuetxtar" +) + +func TestEncode(t *testing.T) { + test := cuetxtar.TxTarTest{ + Root: "./testdata/encoder", + Name: "encode", + Update: cuetest.UpdateGoldenFiles, + } + + r := cue.Runtime{} + + test.Run(t, func(t *cuetxtar.Test) { + // TODO: use high-level API. + + var schema, value cue.Value + + for _, f := range t.Archive.Files { + switch { + case strings.HasSuffix(f.Name, ".cue"): + inst, err := r.Compile(f.Name, f.Data) + if err != nil { + t.WriteErrors(errors.Promote(err, "test")) + return + } + switch f.Name { + case "schema.cue": + schema = inst.Value() + case "value.cue": + value = inst.Value() + } + } + } + + v := schema.Unify(value) + if err := v.Err(); err != nil { + t.WriteErrors(errors.Promote(err, "test")) + return + } + + b, err := textproto.NewEncoder().Encode(v) + if err != nil { + t.WriteErrors(errors.Promote(err, "test")) + return + } + _, _ = t.Write(b) + + }) +} diff --git a/encoding/protobuf/textproto/testdata/encoder/enum.txtar b/encoding/protobuf/textproto/testdata/encoder/enum.txtar new file mode 100644 index 000000000..a77c7d0db --- /dev/null +++ b/encoding/protobuf/textproto/testdata/encoder/enum.txtar @@ -0,0 +1,76 @@ +-- schema.cue -- +enum: [string]: + { "foo", #enumValue: 1 } | + { "bar", #enumValue: 2 } @protobuf(1,Enum) + +defEnum: [string]: #FOO | #BAR @protobuf(2,Enum) + +#FOO: 1 +#BAR: 2 + + +typeEnum: [string]: #Enum @protobuf(3,Enum) + +#Enum: #FOO | #BAR | 3 + +// TODO: consider supporting @symbol(foo) or @json(,symbol=foo) +// symbolEnum: [string]: +// { 1, @symbol(foo) } | +// { 2, @symbol(bar) } + + +singleEnum: #single @protobuf(3,Enum) + +#single: 1 + +badEnum: { string, #enumValue: 1 } | { "two", #enumValue: 2 } + + +-- value.cue -- +enum: asIs: "foo" + +// Convert integers to strings +defEnum: foo: 1 +defEnum: bar: 2 + +typeEnum: foo: 1 +typeEnum: bar: 2 +typeEnum: baz: 3 + +// TODO: consider supporting @symbol(foo) or @json(,symbol=foo) +// symbolEnum: foo: "foo" +// symbolEnum: bar: "bar" +// symbolEnum: baz: "baz" + +singleEnum: 1 + +-- out/jsonpb -- +enum: asIs: "foo" +enum: asIsUnknown: "foobar" + +// Convert integers to strings +defEnum: foo: "foo" +defEnum: bar: "bar" +defEnum: baz: 3 + +// TODO: consider supporting @symbol(foo) or @json(,symbol=foo) +// symbolEnum: foo: "foo" +// symbolEnum: bar: "bar" +// symbolEnum: baz: "baz" + +singleEnum: "single" +-- out/encode -- +enum: { + asIs: "foo" +} +defEnum: { + # Convert integers to strings + foo: FOO + bar: BAR +} +typeEnum: { + foo: FOO + bar: BAR + baz: 3 +} +singleEnum: single diff --git a/encoding/protobuf/textproto/testdata/encoder/list.txtar b/encoding/protobuf/textproto/testdata/encoder/list.txtar new file mode 100644 index 000000000..9ad92137f --- /dev/null +++ b/encoding/protobuf/textproto/testdata/encoder/list.txtar @@ -0,0 +1,24 @@ +-- value.cue -- +// List comment +intList: [ 1, 2, 3 ] + +structList: [{ + foo: 1 + bar: 2 +}, { + foo: 3 + bar: 4 +}] +-- out/encode -- +# List comment +intList: 1 +intList: 2 +intList: 3 +structList: { + foo: 1 + bar: 2 +} +structList: { + foo: 3 + bar: 4 +} diff --git a/encoding/protobuf/textproto/testdata/encoder/map.txtar b/encoding/protobuf/textproto/testdata/encoder/map.txtar new file mode 100644 index 000000000..793b1d046 --- /dev/null +++ b/encoding/protobuf/textproto/testdata/encoder/map.txtar @@ -0,0 +1,44 @@ +-- value.cue -- +m: _ @protobuf(1,map[string]string) +m: foo: "bar" +m: qux: "quux" +m: "1": "one" + +// Doc 1 +intMap: _ @protobuf(1,map[int]string) +// Doc 2 +intMap: { "1": "one" } +// Doc 3 (seems not to be supported by proto) +intMap: "2": "two" +intMap: { + // Doc inner (seems not to be supported by proto) + "3": "three" +} + +-- out/encode -- +m: { + key: "foo" + value: "bar" +} +m: { + key: "qux" + value: "quux" +} +m: { + key: "1" + value: "one" +} +# Doc 1 +# Doc 2 +intMap: { + key: 1 + value: "one" +} +intMap: { + key: 2 + value: "two" +} +intMap: { + key: 3 + value: "three" +} diff --git a/encoding/protobuf/textproto/testdata/encoder/simple.txtar b/encoding/protobuf/textproto/testdata/encoder/simple.txtar new file mode 100644 index 000000000..bccaa005e --- /dev/null +++ b/encoding/protobuf/textproto/testdata/encoder/simple.txtar @@ -0,0 +1,23 @@ +-- value.cue -- +a: 1 +b: 2 +c: 3.4 + +d: "foo\u1234" +e: '\000' + +f: false +// Doc comment +t: true + +notConcrete: string + +-- out/encode -- +a: 1 +b: 2 +c: 3.4 +d: "fooሴ" +e: "\x00" +f: false +# Doc comment +t: true diff --git a/go.sum b/go.sum index de875ad49..8c5d68a83 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=