Skip to content

Commit

Permalink
WIP: Support arbitrary metadata values
Browse files Browse the repository at this point in the history
  • Loading branch information
@Jeffail
Jeffail committed Oct 12, 2022
1 parent 5ba8448 commit cefa288
Show file tree
Hide file tree
Showing 89 changed files with 648 additions and 646 deletions.
18 changes: 9 additions & 9 deletions internal/bloblang/field/expression_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func TestStaticExpressionOptimization(t *testing.T) {
func TestExpressions(t *testing.T) {
type easyMsg struct {
content string
meta map[string]string
meta map[string]any
}

tests := map[string]struct {
Expand Down Expand Up @@ -228,7 +228,7 @@ func TestExpressions(t *testing.T) {
numDyn: 1,
output: `from foo`,
messages: []easyMsg{
{content: `hello world`, meta: map[string]string{
{content: `hello world`, meta: map[string]any{
"foo": "from foo",
"bar": "from bar",
}},
Expand All @@ -245,7 +245,7 @@ func TestExpressions(t *testing.T) {
numDyn: 1,
output: `from foo`,
messages: []easyMsg{
{content: `hello world`, meta: map[string]string{
{content: `hello world`, meta: map[string]any{
"foo": "from foo",
"bar": "from bar",
}},
Expand All @@ -262,7 +262,7 @@ func TestExpressions(t *testing.T) {
numDyn: 1,
output: `null`,
messages: []easyMsg{
{content: `hello world`, meta: map[string]string{
{content: `hello world`, meta: map[string]any{
"bar": "from bar",
}},
},
Expand All @@ -278,7 +278,7 @@ func TestExpressions(t *testing.T) {
numDyn: 1,
output: `{"bar":"from bar","foo":"from foo"}`,
messages: []easyMsg{
{content: `hello world`, meta: map[string]string{
{content: `hello world`, meta: map[string]any{
"foo": "from foo",
"bar": "from bar",
}},
Expand All @@ -295,7 +295,7 @@ func TestExpressions(t *testing.T) {
numDyn: 1,
output: `{"bar":"from bar","foo":"from foo"}`,
messages: []easyMsg{
{content: `hello world`, meta: map[string]string{
{content: `hello world`, meta: map[string]any{
"foo": "from foo",
"bar": "from bar",
}},
Expand All @@ -314,11 +314,11 @@ func TestExpressions(t *testing.T) {
numDyn: 1,
output: `from bar from 1`,
messages: []easyMsg{
{content: `first`, meta: map[string]string{
{content: `first`, meta: map[string]any{
"foo": "from foo from 0",
"bar": "from bar from 0",
}},
{content: `second`, meta: map[string]string{
{content: `second`, meta: map[string]any{
"foo": "from foo from 1",
"bar": "from bar from 1",
}},
Expand All @@ -336,7 +336,7 @@ func TestExpressions(t *testing.T) {
part := message.NewPart([]byte(m.content))
if m.meta != nil {
for k, v := range m.meta {
part.MetaSet(k, v)
part.MetaSetMut(k, v)
}
}
msg = append(msg, part)
Expand Down
15 changes: 9 additions & 6 deletions internal/bloblang/mapping/assignment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ import (
//------------------------------------------------------------------------------

type metaMsg interface {
MetaSet(key, value string)
MetaSetMut(key string, value any)
MetaDelete(key string)
MetaIter(f func(k, v string) error) error
MetaIterMut(f func(k string, v any) error) error
}

// AssignmentContext contains references to all potential assignment
Expand Down Expand Up @@ -86,20 +86,23 @@ func (m *MetaAssignment) Apply(value any, ctx AssignmentContext) error {
return errors.New("unable to assign metadata in the current context")
}
_, deleted := value.(query.Delete)
if !deleted {
value = query.IClone(value)
}
if m.key == nil {
if deleted {
_ = ctx.Meta.MetaIter(func(k, _ string) error {
_ = ctx.Meta.MetaIterMut(func(k string, _ any) error {
ctx.Meta.MetaDelete(k)
return nil
})
} else {
if m, ok := value.(map[string]any); ok {
_ = ctx.Meta.MetaIter(func(k, _ string) error {
_ = ctx.Meta.MetaIterMut(func(k string, _ any) error {
ctx.Meta.MetaDelete(k)
return nil
})
for k, v := range m {
ctx.Meta.MetaSet(k, query.IToString(v))
ctx.Meta.MetaSetMut(k, v)
}
} else {
return fmt.Errorf("setting root meta object requires object value, received: %T", value)
Expand All @@ -110,7 +113,7 @@ func (m *MetaAssignment) Apply(value any, ctx AssignmentContext) error {
if deleted {
ctx.Meta.MetaDelete(*m.key)
} else {
ctx.Meta.MetaSet(*m.key, query.IToString(value))
ctx.Meta.MetaSetMut(*m.key, value)
}
return nil
}
Expand Down
32 changes: 16 additions & 16 deletions internal/bloblang/mapping/executor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
func TestAssignments(t *testing.T) {
type part struct {
Content string
Meta map[string]string
Meta map[string]any
}

metaKey := func(k string) *string {
Expand Down Expand Up @@ -115,7 +115,7 @@ func TestAssignments(t *testing.T) {
input: []part{{Content: `{}`}},
output: &part{
Content: `{}`,
Meta: map[string]string{
Meta: map[string]any{
"foo": "exists now",
},
},
Expand All @@ -126,14 +126,14 @@ func TestAssignments(t *testing.T) {
),
input: []part{{
Content: `{}`,
Meta: map[string]string{
Meta: map[string]any{
"ignore": "me",
"and": "delete me",
},
}},
output: &part{
Content: `{}`,
Meta: map[string]string{
Meta: map[string]any{
"ignore": "me",
},
},
Expand All @@ -154,14 +154,14 @@ func TestAssignments(t *testing.T) {
),
input: []part{{
Content: `{}`,
Meta: map[string]string{
Meta: map[string]any{
"foo": "first",
"bar": "second",
},
}},
output: &part{
Content: `{}`,
Meta: map[string]string{
Meta: map[string]any{
"new1": "value1",
"new2": "value2",
},
Expand All @@ -173,7 +173,7 @@ func TestAssignments(t *testing.T) {
),
input: []part{{
Content: `{}`,
Meta: map[string]string{
Meta: map[string]any{
"foo": "first",
"bar": "second",
},
Expand All @@ -187,13 +187,13 @@ func TestAssignments(t *testing.T) {
),
input: []part{{
Content: `{}`,
Meta: map[string]string{
Meta: map[string]any{
"foo": "old value",
},
}},
output: &part{
Content: `{}`,
Meta: map[string]string{
Meta: map[string]any{
"foo": "new value",
"bar": "old value",
},
Expand All @@ -207,7 +207,7 @@ func TestAssignments(t *testing.T) {
input: []part{{Content: `{}`}},
output: &part{
Content: `{}`,
Meta: map[string]string{
Meta: map[string]any{
"foo": "exists now",
"bar": "exists now",
},
Expand Down Expand Up @@ -243,7 +243,7 @@ func TestAssignments(t *testing.T) {
part = message.NewPart(nil)
}
for k, v := range p.Meta {
part.MetaSet(k, v)
part.MetaSetMut(k, v)
}
msg = append(msg, part)
}
Expand All @@ -258,14 +258,14 @@ func TestAssignments(t *testing.T) {

if test.output != nil {
if test.output.Meta == nil {
test.output.Meta = map[string]string{}
test.output.Meta = map[string]any{}
}

newPart := part{
Content: string(resPart.AsBytes()),
Meta: map[string]string{},
Meta: map[string]any{},
}
_ = resPart.MetaIter(func(k, v string) error {
_ = resPart.MetaIterMut(func(k string, v any) error {
newPart.Meta[k] = v
return nil
})
Expand Down Expand Up @@ -456,7 +456,7 @@ func TestExec(t *testing.T) {
func TestQueries(t *testing.T) {
type part struct {
Content string
Meta map[string]string
Meta map[string]any
}

initFunc := func(name string, args ...any) query.Function {
Expand Down Expand Up @@ -525,7 +525,7 @@ func TestQueries(t *testing.T) {
for _, p := range test.input {
part := message.NewPart([]byte(p.Content))
for k, v := range p.Meta {
part.MetaSet(k, v)
part.MetaSetMut(k, v)
}
msg = append(msg, part)
}
Expand Down
4 changes: 2 additions & 2 deletions internal/bloblang/parser/field_parser_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ func TestFieldExpressionParserErrors(t *testing.T) {
func TestFieldExpressions(t *testing.T) {
type easyMsg struct {
content string
meta map[string]string
meta map[string]any
}

tests := map[string]struct {
Expand Down Expand Up @@ -197,7 +197,7 @@ func TestFieldExpressions(t *testing.T) {
part := message.NewPart([]byte(m.content))
if m.meta != nil {
for k, v := range m.meta {
part.MetaSet(k, v)
part.MetaSetMut(k, v)
}
}
msg = append(msg, part)
Expand Down
32 changes: 16 additions & 16 deletions internal/bloblang/parser/mapping_parser_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ func TestMappings(t *testing.T) {

type part struct {
Content string
Meta map[string]string
Meta map[string]any
}

tests := map[string]struct {
Expand Down Expand Up @@ -272,14 +272,14 @@ meta "bar baz" = deleted()`,
input: []part{
{
Content: `{"foo":"bar"}`,
Meta: map[string]string{
Meta: map[string]any{
"bar baz": "test1",
},
},
},
output: part{
Content: `{"bar":{"baz":"test1"}}`,
Meta: map[string]string{
Meta: map[string]any{
"foo": "bar",
},
},
Expand All @@ -292,7 +292,7 @@ meta "bar baz" = "test1"`,
},
output: part{
Content: `{"foo":{"bar":"baz"}}`,
Meta: map[string]string{
Meta: map[string]any{
"bar": "baz",
"bar baz": "test1",
},
Expand All @@ -305,7 +305,7 @@ meta = deleted()`,
input: []part{
{
Content: `{"foo":"bar"}`,
Meta: map[string]string{
Meta: map[string]any{
"bar baz": "test1",
},
},
Expand Down Expand Up @@ -348,9 +348,9 @@ meta bar = 5 + 2`,
},
output: part{
Content: `this isn't json`,
Meta: map[string]string{
Meta: map[string]any{
"foo": "foo",
"bar": "7",
"bar": int64(7),
},
},
},
Expand All @@ -372,7 +372,7 @@ root."bar baz".test = 5 + 2`,
},
output: part{
Content: `{"bar baz":{"test":7}}`,
Meta: map[string]string{
Meta: map[string]any{
"foo bar": "hello world",
},
},
Expand All @@ -385,8 +385,8 @@ foo = "static"`,
},
output: part{
Content: `{"foo":"static"}`,
Meta: map[string]string{
"content": `hello world`,
Meta: map[string]any{
"content": []byte(`hello world`),
},
},
},
Expand All @@ -398,8 +398,8 @@ foo = "static"`,
},
output: part{
Content: `{"foo":"static"}`,
Meta: map[string]string{
"content": `{"foo":{"bar":"baz"}}`,
Meta: map[string]any{
"content": []byte(`{"foo":{"bar":"baz"}}`),
},
},
},
Expand Down Expand Up @@ -475,12 +475,12 @@ root = this.apply("foo")`, goodMapFile),
for _, p := range test.input {
part := message.NewPart([]byte(p.Content))
for k, v := range p.Meta {
part.MetaSet(k, v)
part.MetaSetMut(k, v)
}
msg = append(msg, part)
}
if test.output.Meta == nil {
test.output.Meta = map[string]string{}
test.output.Meta = map[string]any{}
}

exec, perr := ParseMapping(GlobalContext(), test.mapping)
Expand All @@ -491,9 +491,9 @@ root = this.apply("foo")`, goodMapFile),

newPart := part{
Content: string(resPart.AsBytes()),
Meta: map[string]string{},
Meta: map[string]any{},
}
_ = resPart.MetaIter(func(k, v string) error {
_ = resPart.MetaIterMut(func(k string, v any) error {
newPart.Meta[k] = v
return nil
})
Expand Down
Loading

0 comments on commit cefa288

Please sign in to comment.