From c00c4dbceebc06d295c5a87dc8ffd9cb8858911a Mon Sep 17 00:00:00 2001 From: Edmo Vamerlatti Costa <11836452+edmocosta@users.noreply.github.com> Date: Fri, 6 Sep 2024 19:01:10 +0200 Subject: [PATCH] [pkg/ottl] Change grammar to support expressing statements context via path names (#34875) **Description:** <Describe what has changed.> <!--Ex. Fixing a bug - Describe the bug and how this fixes the issue. Ex. Adding a feature - Explain what this achieves.--> This PR is part of the https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/29017, and introduces the necessary OTTL Grammar/Parser changes to make it possible for statements express their context via path names. This first iteration changes the grammar to parse the first `path` segment as the `Context` name, for example, the path `foo.bar.field` would be parsed as the following AST example: `{Context: "foo", Fields: [{Name: "bar"}, {Name: "field"}]}`, instead of `{Fields: [{Name: "foo"},{Name: "bar"}, {Name: "field"}]}`. To make it non-breaking to all components during the transition time and development, this PR also changes the `ottl.Parser[k].newPath` function to, by default, fall this new behavior back to the previous one, using the grammar's parsed `Context` value as the first path segment (`Fields[0]`). Two new `Parser[K]` options were added, `WithPathContextNames` and `WithPathContextNameValidation`, The first option will be used to enable the statements context support, parsing the first path segment as Context when the value maches any configured context names, or falling it back otherwise. The second option will be used to enable the validation and turning off the backward compatible mode, raising an error when paths have no context prefix, or when it's invalid. For more details, please check the https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/29017 discussion out. **Link to tracking Issue:** https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/29017 **Testing:** - Unit tests **Documentation:** - No documentation changes were made at this point. --- ...ttl_statements_context_change_grammar.yaml | 27 ++++ pkg/ottl/contexts/internal/path.go | 5 + pkg/ottl/expression.go | 2 +- pkg/ottl/functions.go | 78 ++++++++++- pkg/ottl/functions_test.go | 121 +++++++++++++++++- pkg/ottl/grammar.go | 3 +- pkg/ottl/parser.go | 17 +++ pkg/ottl/parser_test.go | 64 ++++++--- 8 files changed, 286 insertions(+), 31 deletions(-) create mode 100644 .chloggen/ottl_statements_context_change_grammar.yaml diff --git a/.chloggen/ottl_statements_context_change_grammar.yaml b/.chloggen/ottl_statements_context_change_grammar.yaml new file mode 100644 index 000000000000..ba582e9ac615 --- /dev/null +++ b/.chloggen/ottl_statements_context_change_grammar.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: breaking + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: pkg/ottl + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "Change the OTTL grammar to support expressing statements context via path names" + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [29017] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: "The `ottl.Path` interface requires a new method: `Context() string`" + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [api] diff --git a/pkg/ottl/contexts/internal/path.go b/pkg/ottl/contexts/internal/path.go index c7d9d802b664..954d14329646 100644 --- a/pkg/ottl/contexts/internal/path.go +++ b/pkg/ottl/contexts/internal/path.go @@ -12,6 +12,7 @@ import ( var _ ottl.Path[any] = &TestPath[any]{} type TestPath[K any] struct { + C string N string KeySlice []ottl.Key[K] NextPath *TestPath[K] @@ -21,6 +22,10 @@ func (p *TestPath[K]) Name() string { return p.N } +func (p *TestPath[K]) Context() string { + return p.C +} + func (p *TestPath[K]) Next() ottl.Path[K] { if p.NextPath == nil { return nil diff --git a/pkg/ottl/expression.go b/pkg/ottl/expression.go index ceae91386a84..c69c9f0e319a 100644 --- a/pkg/ottl/expression.go +++ b/pkg/ottl/expression.go @@ -741,7 +741,7 @@ func (p *Parser[K]) newGetter(val value) (Getter[K], error) { return &literal[K]{value: *i}, nil } if eL.Path != nil { - np, err := newPath[K](eL.Path.Fields) + np, err := p.newPath(eL.Path) if err != nil { return nil, err } diff --git a/pkg/ottl/functions.go b/pkg/ottl/functions.go index 9c6ccc226aaf..251e79587b75 100644 --- a/pkg/ottl/functions.go +++ b/pkg/ottl/functions.go @@ -22,9 +22,15 @@ type Enum int64 type EnumSymbol string -func buildOriginalText(fields []field) string { +func buildOriginalText(path *path) string { var builder strings.Builder - for i, f := range fields { + if path.Context != "" { + builder.WriteString(path.Context) + if len(path.Fields) > 0 { + builder.WriteString(".") + } + } + for i, f := range path.Fields { builder.WriteString(f.Name) if len(f.Keys) > 0 { for _, k := range f.Keys { @@ -38,21 +44,28 @@ func buildOriginalText(fields []field) string { builder.WriteString("]") } } - if i != len(fields)-1 { + if i != len(path.Fields)-1 { builder.WriteString(".") } } return builder.String() } -func newPath[K any](fields []field) (*basePath[K], error) { - if len(fields) == 0 { +func (p *Parser[K]) newPath(path *path) (*basePath[K], error) { + if len(path.Fields) == 0 { return nil, fmt.Errorf("cannot make a path from zero fields") } - originalText := buildOriginalText(fields) + + pathContext, fields, err := p.parsePathContext(path) + if err != nil { + return nil, err + } + + originalText := buildOriginalText(path) var current *basePath[K] for i := len(fields) - 1; i >= 0; i-- { current = &basePath[K]{ + context: pathContext, name: fields[i].Name, keys: newKeys[K](fields[i].Keys), nextPath: current, @@ -64,10 +77,56 @@ func newPath[K any](fields []field) (*basePath[K], error) { return current, nil } +func (p *Parser[K]) parsePathContext(path *path) (string, []field, error) { + hasPathContextNames := len(p.pathContextNames) > 0 + if path.Context != "" { + // no pathContextNames means the Parser isn't handling the grammar path's context yet, + // so it falls back to the previous behavior with the path.Context value as the first + // path's segment. + if !hasPathContextNames { + return "", append([]field{{Name: path.Context}}, path.Fields...), nil + } + + if _, ok := p.pathContextNames[path.Context]; !ok { + return "", path.Fields, fmt.Errorf(`context "%s" from path "%s" is not valid, it must be replaced by one of: %s`, path.Context, buildOriginalText(path), p.buildPathContextNamesText("")) + } + + return path.Context, path.Fields, nil + } + + if hasPathContextNames { + originalText := buildOriginalText(path) + return "", nil, fmt.Errorf(`missing context name for path "%s", possibly valid options are: %s`, originalText, p.buildPathContextNamesText(originalText)) + } + + return "", path.Fields, nil +} + +func (p *Parser[K]) buildPathContextNamesText(path string) string { + var builder strings.Builder + var suffix string + if path != "" { + suffix = "." + path + } + + i := 0 + for ctx := range p.pathContextNames { + builder.WriteString(fmt.Sprintf(`"%s%s"`, ctx, suffix)) + if i != len(p.pathContextNames)-1 { + builder.WriteString(", ") + } + i++ + } + return builder.String() +} + // Path represents a chain of path parts in an OTTL statement, such as `body.string`. // A Path has a name, and potentially a set of keys. // If the path in the OTTL statement contains multiple parts (separated by a dot (`.`)), then the Path will have a pointer to the next Path. type Path[K any] interface { + // Context is the OTTL context name of this Path. + Context() string + // Name is the name of this segment of the path. Name() string @@ -86,6 +145,7 @@ type Path[K any] interface { var _ Path[any] = &basePath[any]{} type basePath[K any] struct { + context string name string keys []Key[K] nextPath *basePath[K] @@ -94,6 +154,10 @@ type basePath[K any] struct { originalText string } +func (p *basePath[K]) Context() string { + return p.context +} + func (p *basePath[K]) Name() string { return p.name } @@ -412,7 +476,7 @@ func (p *Parser[K]) buildArg(argVal value, argType reflect.Type) (any, error) { if argVal.Literal == nil || argVal.Literal.Path == nil { return nil, fmt.Errorf("must be a path") } - np, err := newPath[K](argVal.Literal.Path.Fields) + np, err := p.newPath(argVal.Literal.Path) if err != nil { return nil, err } diff --git a/pkg/ottl/functions_test.go b/pkg/ottl/functions_test.go index cc9905f08e5f..5fc00bd1a0ca 100644 --- a/pkg/ottl/functions_test.go +++ b/pkg/ottl/functions_test.go @@ -2230,6 +2230,14 @@ func Test_basePath_Name(t *testing.T) { assert.Equal(t, "test", n) } +func Test_basePath_Context(t *testing.T) { + bp := basePath[any]{ + context: "log", + } + n := bp.Context() + assert.Equal(t, "log", n) +} + func Test_basePath_Next(t *testing.T) { bp := basePath[any]{ nextPath: &basePath[any]{}, @@ -2352,6 +2360,13 @@ func Test_basePath_NextWithIsComplete(t *testing.T) { } func Test_newPath(t *testing.T) { + ps, _ := NewParser[any]( + defaultFunctionsForTests(), + testParsePath[any], + componenttest.NewNopTelemetrySettings(), + WithEnumParser[any](testParseEnum), + ) + fields := []field{ { Name: "body", @@ -2365,7 +2380,8 @@ func Test_newPath(t *testing.T) { }, }, } - np, err := newPath[any](fields) + + np, err := ps.newPath(&path{Fields: fields}) assert.NoError(t, err) p := Path[any](np) assert.Equal(t, "body", p.Name()) @@ -2384,6 +2400,109 @@ func Test_newPath(t *testing.T) { assert.Nil(t, i) } +func Test_newPath_WithPathContextNames(t *testing.T) { + tests := []struct { + name string + pathContext string + pathContextNames []string + expectedError string + }{ + { + name: "with no path context", + pathContextNames: []string{"log"}, + expectedError: `missing context name for path "body.string[key]", valid options are: "log.body.string[key]"`, + }, + { + name: "with no path context and configuration", + }, + { + name: "with valid path context", + pathContext: "log", + pathContextNames: []string{"log"}, + }, + { + name: "with invalid path context", + pathContext: "span", + pathContextNames: []string{"log"}, + expectedError: `context "span" from path "span.body.string[key]" is not valid, it must be replaced by one of: "log"`, + }, + { + name: "with multiple configured contexts", + pathContext: "span", + pathContextNames: []string{"log", "span"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ps, _ := NewParser[any]( + defaultFunctionsForTests(), + testParsePath[any], + componenttest.NewNopTelemetrySettings(), + WithEnumParser[any](testParseEnum), + WithPathContextNames[any](tt.pathContextNames), + ) + + gp := &path{ + Context: tt.pathContext, + Fields: []field{ + { + Name: "body", + }, + { + Name: "string", + Keys: []key{ + { + String: ottltest.Strp("key"), + }, + }, + }, + }} + + np, err := ps.newPath(gp) + if tt.expectedError != "" { + assert.Error(t, err, tt.expectedError) + return + } + assert.NoError(t, err) + p := Path[any](np) + contextParsedAsField := len(tt.pathContextNames) == 0 && tt.pathContext != "" + if contextParsedAsField { + assert.Equal(t, tt.pathContext, p.Name()) + assert.Equal(t, "", p.Context()) + assert.Nil(t, p.Keys()) + p = p.Next() + } + var bodyStringFuncValue string + if tt.pathContext != "" { + bodyStringFuncValue = fmt.Sprintf("%s.body.string[key]", tt.pathContext) + } else { + bodyStringFuncValue = "body.string[key]" + } + assert.Equal(t, "body", p.Name()) + assert.Nil(t, p.Keys()) + assert.Equal(t, bodyStringFuncValue, p.String()) + if !contextParsedAsField { + assert.Equal(t, tt.pathContext, p.Context()) + } + p = p.Next() + assert.Equal(t, "string", p.Name()) + assert.Equal(t, bodyStringFuncValue, p.String()) + if !contextParsedAsField { + assert.Equal(t, tt.pathContext, p.Context()) + } + assert.Nil(t, p.Next()) + assert.Len(t, p.Keys(), 1) + v, err := p.Keys()[0].String(context.Background(), struct{}{}) + assert.NoError(t, err) + assert.Equal(t, "key", *v) + i, err := p.Keys()[0].Int(context.Background(), struct{}{}) + assert.NoError(t, err) + assert.Nil(t, i) + }) + } +} + func Test_baseKey_String(t *testing.T) { bp := baseKey[any]{ s: ottltest.Strp("test"), diff --git a/pkg/ottl/grammar.go b/pkg/ottl/grammar.go index 44f642b78504..369eff00ecbd 100644 --- a/pkg/ottl/grammar.go +++ b/pkg/ottl/grammar.go @@ -253,7 +253,8 @@ func (v *value) checkForCustomError() error { // path represents a telemetry path mathExpression. type path struct { - Fields []field `parser:"@@ ( '.' @@ )*"` + Context string `parser:"(@Lowercase '.')?"` + Fields []field `parser:"@@ ( '.' @@ )*"` } // field is an item within a path. diff --git a/pkg/ottl/parser.go b/pkg/ottl/parser.go index 4a0b3cabe7be..ed8457603f7c 100644 --- a/pkg/ottl/parser.go +++ b/pkg/ottl/parser.go @@ -58,6 +58,7 @@ type Parser[K any] struct { pathParser PathExpressionParser[K] enumParser EnumParser telemetrySettings component.TelemetrySettings + pathContextNames map[string]struct{} } func NewParser[K any]( @@ -91,6 +92,22 @@ func WithEnumParser[K any](parser EnumParser) Option[K] { } } +// WithPathContextNames sets the context names to be considered when parsing a Path value. +// When this option is empty or nil, all Path segments are considered fields, and the +// Path.Context value is always empty. +// When this option is configured, and the path's context is empty or is not present in +// this context names list, it results into an error. +func WithPathContextNames[K any](contexts []string) Option[K] { + return func(p *Parser[K]) { + pathContextNames := make(map[string]struct{}, len(contexts)) + for _, ctx := range contexts { + pathContextNames[ctx] = struct{}{} + } + + p.pathContextNames = pathContextNames + } +} + // ParseStatements parses string statements into ottl.Statement objects ready for execution. // Returns a slice of statements and a nil error on successful parsing. // If parsing fails, returns nil and a joined error containing each error per failed statement. diff --git a/pkg/ottl/parser_test.go b/pkg/ottl/parser_test.go index dc475b2b7d6a..d0a5e4b47add 100644 --- a/pkg/ottl/parser_test.go +++ b/pkg/ottl/parser_test.go @@ -207,10 +207,8 @@ func Test_parse(t *testing.T) { Value: &value{ Literal: &mathExprLiteral{ Path: &path{ + Context: "bear", Fields: []field{ - { - Name: "bear", - }, { Name: "honey", }, @@ -269,10 +267,8 @@ func Test_parse(t *testing.T) { Value: value{ Literal: &mathExprLiteral{ Path: &path{ + Context: "bear", Fields: []field{ - { - Name: "bear", - }, { Name: "honey", }, @@ -302,10 +298,8 @@ func Test_parse(t *testing.T) { Value: value{ Literal: &mathExprLiteral{ Path: &path{ + Context: "foo", Fields: []field{ - { - Name: "foo", - }, { Name: "attributes", Keys: []key{ @@ -332,6 +326,42 @@ func Test_parse(t *testing.T) { WhereClause: nil, }, }, + { + name: "single field segment", + statement: `set(attributes["bar"], "dog")`, + expected: &parsedStatement{ + Editor: editor{ + Function: "set", + Arguments: []argument{ + { + Value: value{ + Literal: &mathExprLiteral{ + Path: &path{ + Context: "", + Fields: []field{ + { + Name: "attributes", + Keys: []key{ + { + String: ottltest.Strp("bar"), + }, + }, + }, + }, + }, + }, + }, + }, + { + Value: value{ + String: ottltest.Strp("dog"), + }, + }, + }, + }, + WhereClause: nil, + }, + }, { name: "Converter parameters (All Uppercase)", statement: `replace_pattern(attributes["message"], "device=*", attributes["device_name"], SHA256)`, @@ -469,10 +499,8 @@ func Test_parse(t *testing.T) { Value: value{ Literal: &mathExprLiteral{ Path: &path{ + Context: "foo", Fields: []field{ - { - Name: "foo", - }, { Name: "bar", Keys: []key{ @@ -525,10 +553,8 @@ func Test_parse(t *testing.T) { Value: value{ Literal: &mathExprLiteral{ Path: &path{ + Context: "foo", Fields: []field{ - { - Name: "foo", - }, { Name: "attributes", Keys: []key{ @@ -588,10 +614,8 @@ func Test_parse(t *testing.T) { Value: value{ Literal: &mathExprLiteral{ Path: &path{ + Context: "foo", Fields: []field{ - { - Name: "foo", - }, { Name: "attributes", Keys: []key{ @@ -651,10 +675,8 @@ func Test_parse(t *testing.T) { Value: value{ Literal: &mathExprLiteral{ Path: &path{ + Context: "foo", Fields: []field{ - { - Name: "foo", - }, { Name: "attributes", Keys: []key{