Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add encoders to avrogen #292

Merged
merged 4 commits into from
Aug 20, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion cmd/avrogen/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ type config struct {
Out string
Tags string
FullName bool
Encoders bool
}

func main() {
Expand All @@ -33,6 +34,7 @@ func realMain(args []string, out io.Writer) int {
flgs.StringVar(&cfg.Out, "o", "", "The output file path.")
flgs.StringVar(&cfg.Tags, "tags", "", "The additional field tags <tag-name>:{snake|camel|upper-camel|kebab}>[,...]")
flgs.BoolVar(&cfg.FullName, "fullname", false, "Use the full name of the Record schema to create the struct name.")
flgs.BoolVar(&cfg.Encoders, "encoders", false, "Generate encoders for the structs.")
flgs.Usage = func() {
_, _ = fmt.Fprintln(out, "Usage: avrogen [options] schemas")
_, _ = fmt.Fprintln(out, "Options:")
Expand All @@ -52,7 +54,11 @@ func realMain(args []string, out io.Writer) int {
return 1
}

g := gen.NewGenerator(cfg.Pkg, tags, gen.WithFullName(cfg.FullName))
opts := []gen.OptsFunc{
gen.WithFullName(cfg.FullName),
gen.WithEncoders(cfg.Encoders),
}
g := gen.NewGenerator(cfg.Pkg, tags, opts...)
for _, file := range flgs.Args() {
schema, err := avro.ParseFiles(filepath.Clean(file))
if err != nil {
Expand Down
23 changes: 23 additions & 0 deletions cmd/avrogen/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,29 @@ func TestAvroGen_GeneratesSchemaWithFullname(t *testing.T) {
assert.Equal(t, want, got)
}

func TestAvroGen_GeneratesSchemaWithEncoders(t *testing.T) {
path, err := os.MkdirTemp("./", "avrogen")
require.NoError(t, err)
t.Cleanup(func() { _ = os.RemoveAll(path) })

file := filepath.Join(path, "test.go")
args := []string{"avrogen", "-pkg", "testpkg", "-o", file, "-encoders", "testdata/schema.avsc"}
gotCode := realMain(args, io.Discard)
require.Equal(t, 0, gotCode)

got, err := os.ReadFile(file)
require.NoError(t, err)

if *update {
err = os.WriteFile("testdata/golden_encoders.go", got, 0600)
require.NoError(t, err)
}

want, err := os.ReadFile("testdata/golden_encoders.go")
require.NoError(t, err)
assert.Equal(t, want, got)
}

func TestParseTags(t *testing.T) {
tests := []struct {
name string
Expand Down
1 change: 1 addition & 0 deletions cmd/avrogen/testdata/golden.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

32 changes: 32 additions & 0 deletions cmd/avrogen/testdata/golden_encoders.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions cmd/avrogen/testdata/golden_fullname.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

48 changes: 46 additions & 2 deletions gen/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ type Config struct {
PackageName string
Tags map[string]TagStyle
FullName bool
Encoders bool
}

// TagStyle defines the styling for a tag.
Expand All @@ -39,6 +40,7 @@ const outputTemplate = `package {{ .PackageName }}

// Code generated by avro/gen. DO NOT EDIT.

{{- $encoders := .WithEncoders }}
{{ if len .Imports }}
import (
{{- range .Imports }}
Expand All @@ -60,11 +62,36 @@ import (
{{ end }}

{{- range .Typedefs }}
// {{ .Name }} is a generated struct.
type {{ .Name }} struct {
{{- range .Fields }}
{{ .Name }} {{ .Type }} {{ .Tag }}
{{- end }}
{{- if $encoders }}

schema avro.Schema
{{- end }}
}

{{- if $encoders }}
// Schema returns the schema for {{ .Name }}.
func (o *{{ .Name }}) Schema() avro.Schema {
if o.schema == nil {
o.schema = avro.MustParse(` + "`{{ .Schema }}`" + `)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a way to avoid parsing the schema whenever we decode a new struct? What is the cost of decoding a stream of events (in my case, more than 10K events per second), then processing and encoding them back? Also, each struct will now have an embedded schema struct, adding some memory overhead.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thats what this does. It caches the schema that is parsed.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The cache seems to be in the struct level, so each time a new struct is created will have to parse the schema?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in #293

}
return o.schema
}

// Unmarshal decodes b into the receiver.
func (o *{{ .Name }}) Unmarshal(b []byte) error {
return avro.Unmarshal(o.Schema(), b, o)
}

// Marshal encodes the receiver.
func (o *{{ .Name }}) Marshal() ([]byte, error) {
return avro.Marshal(o.Schema(), o)
}
{{- end }}
{{ end }}`

var primitiveMappings = map[avro.Type]string{
Expand Down Expand Up @@ -95,6 +122,7 @@ func StructFromSchema(schema avro.Schema, w io.Writer, cfg Config) error {

opts := []OptsFunc{
WithFullName(cfg.FullName),
WithEncoders(cfg.Encoders),
}
g := NewGenerator(strcase.ToSnake(cfg.PackageName), cfg.Tags, opts...)
g.Parse(rec)
Expand Down Expand Up @@ -124,11 +152,23 @@ func WithFullName(b bool) OptsFunc {
}
}

// WithEncoders configures the generator to generate schema and encoders on
// all objects.
func WithEncoders(b bool) OptsFunc {
return func(g *Generator) {
g.encoders = b
if b {
g.thirdPartyImports = append(g.thirdPartyImports, "github.com/hamba/avro/v2")
}
}
}

// Generator generates Go structs from schemas.
type Generator struct {
pkg string
tags map[string]TagStyle
fullName bool
encoders bool

imports []string
thirdPartyImports []string
Expand Down Expand Up @@ -209,7 +249,7 @@ func (g *Generator) resolveRecordSchema(schema *avro.RecordSchema) string {

typeName := g.resolveTypeName(schema)
if !g.hasTypeDef(typeName) {
g.typedefs = append(g.typedefs, newType(typeName, fields))
g.typedefs = append(g.typedefs, newType(typeName, fields, schema.String()))
}
return typeName
}
Expand Down Expand Up @@ -325,11 +365,13 @@ func (g *Generator) Write(w io.Writer) error {
}

data := struct {
WithEncoders bool
PackageName string
Imports []string
ThirdPartyImports []string
Typedefs []typedef
}{
WithEncoders: g.encoders,
PackageName: g.pkg,
Imports: g.imports,
ThirdPartyImports: g.thirdPartyImports,
Expand All @@ -341,12 +383,14 @@ func (g *Generator) Write(w io.Writer) error {
type typedef struct {
Name string
Fields []field
Schema string
}

func newType(name string, fields []field) typedef {
func newType(name string, fields []field, schema string) typedef {
return typedef{
Name: name,
Fields: fields,
Schema: schema,
}
}

Expand Down
17 changes: 17 additions & 0 deletions gen/gen_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,23 @@ func TestStruct_GenFromRecordSchemaWithFullName(t *testing.T) {
assert.Equal(t, string(want), string(file))
}

func TestStruct_GenFromRecordSchemaWithEncoders(t *testing.T) {
schema, err := os.ReadFile("testdata/golden.avsc")
require.NoError(t, err)

gc := gen.Config{PackageName: "Something", Encoders: true}
file, _ := generate(t, string(schema), gc)

if *update {
err = os.WriteFile("testdata/golden_encoders.go", file, 0600)
require.NoError(t, err)
}

want, err := os.ReadFile("testdata/golden_encoders.go")
require.NoError(t, err)
assert.Equal(t, string(want), string(file))
}

func TestGenerator(t *testing.T) {
unionSchema, err := avro.ParseFiles("testdata/uniontype.avsc")
require.NoError(t, err)
Expand Down
9 changes: 9 additions & 0 deletions gen/testdata/golden.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading