diff --git a/CHANGELOG.md b/CHANGELOG.md index d1c165c09365b..34646e11c83a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Features +* [1677](https://github.com/grafana/loki/pull/1677) **owen-d**: Introduces numeric literals in LogQL * [1686](https://github.com/grafana/loki/pull/1686) **owen-d**: Introduces the `distributor.max-line-size` flag and associated yaml config. When enabled, lines longer than this config will not be accepted. * [1662](https://github.com/grafana/loki/pull/1662) **owen-d**: Introduces binary operators in LogQL * [1572](https://github.com/grafana/loki/pull/1572) **owen-d**: Introduces the `querier.query-ingesters-within` flag and associated yaml config. When enabled, queries for a time range that do not overlap this lookback interval will not be sent to the ingesters. diff --git a/docs/logql.md b/docs/logql.md index 284e825af054c..935a02830c1fd 100644 --- a/docs/logql.md +++ b/docs/logql.md @@ -166,12 +166,24 @@ The following binary arithmetic operators exist in Loki: - `%` (modulo) - `^` (power/exponentiation) -Binary arithmetic operators are defined only between two vectors. +Binary arithmetic operators are defined between two literals (scalars), a literal and a vector, and two vectors. -Between two instant vectors, a binary arithmetic operator is applied to each entry in the left-hand side vector and its matching element in the right-hand vector. The result is propagated into the result vector with the grouping labels becoming the output label set. Entries for which no matching entry in the right-hand vector can be found are not part of the result. +Between two literals, the behavior is obvious: they evaluate to another literal that is the result of the operator applied to both scalar operands (1 + 1 = 2). + +Between a vector and a literal, the operator is applied to the value of every data sample in the vector. E.g. if a time series vector is multiplied by 2, the result is another vector in which every sample value of the original vector is multiplied by 2. + +Between two vectors, a binary arithmetic operator is applied to each entry in the left-hand side vector and its matching element in the right-hand vector. The result is propagated into the result vector with the grouping labels becoming the output label set. Entries for which no matching entry in the right-hand vector can be found are not part of the result. ##### Examples +Implement a health check with a simple query: + +> `1 + 1` + +Double the rate of a a log stream's entries: + +> `sum(rate({app="foo"})) * 2` + Get proportion of warning logs to error logs for the `foo` app > `sum(rate({app="foo", level="warn"}[1m])) / sum(rate({app="foo", level="error"}[1m]))` diff --git a/pkg/loghttp/query.go b/pkg/loghttp/query.go index c6d17c4b3d3ff..c6180474ecd8a 100644 --- a/pkg/loghttp/query.go +++ b/pkg/loghttp/query.go @@ -46,6 +46,7 @@ type ResultType string // ResultType values const ( ResultTypeStream = "streams" + ResultTypeScalar = "scalar" ResultTypeVector = "vector" ResultTypeMatrix = "matrix" ) @@ -65,6 +66,9 @@ type QueryResponseData struct { // Type implements the promql.Value interface func (Streams) Type() ResultType { return ResultTypeStream } +// Type implements the promql.Value interface +func (Scalar) Type() ResultType { return ResultTypeScalar } + // Type implements the promql.Value interface func (Vector) Type() ResultType { return ResultTypeVector } @@ -127,6 +131,10 @@ func (q *QueryResponseData) UnmarshalJSON(data []byte) error { var v Vector err = json.Unmarshal(unmarshal.Result, &v) value = v + case ResultTypeScalar: + var v Scalar + err = json.Unmarshal(unmarshal.Result, &v) + value = v default: return fmt.Errorf("unknown type: %s", unmarshal.Type) } @@ -171,6 +179,9 @@ func (e *Entry) UnmarshalJSON(data []byte) error { return nil } +// Scalar is a single timestamp/float with no labels +type Scalar model.Scalar + // Vector is a slice of Samples type Vector []model.Sample diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index e00659f9f5900..162cc068c1294 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -3,7 +3,6 @@ package logql import ( "bytes" "context" - "errors" "fmt" "regexp" "strconv" @@ -228,18 +227,25 @@ const ( OpTypeCountOverTime = "count_over_time" OpTypeRate = "rate" - // binops + // binops - logical/set OpTypeOr = "or" OpTypeAnd = "and" OpTypeUnless = "unless" - OpTypeAdd = "+" - OpTypeSub = "-" - OpTypeMul = "*" - OpTypeDiv = "/" - OpTypeMod = "%" - OpTypePow = "^" + + // binops - operations + OpTypeAdd = "+" + OpTypeSub = "-" + OpTypeMul = "*" + OpTypeDiv = "/" + OpTypeMod = "%" + OpTypePow = "^" ) +// IsLogicalBinOp tests whether an operation is a logical/set binary operation +func IsLogicalBinOp(op string) bool { + return op == OpTypeOr || op == OpTypeAnd || op == OpTypeUnless +} + // SampleExpr is a LogQL expression filtering logs and returning metric samples. type SampleExpr interface { // Selector is the LogQL selector to apply when retrieving logs. @@ -247,41 +253,6 @@ type SampleExpr interface { Expr } -// StepEvaluator evaluate a single step of a query. -type StepEvaluator interface { - Next() (bool, int64, promql.Vector) - // Close all resources used. - Close() error -} - -type stepEvaluator struct { - fn func() (bool, int64, promql.Vector) - close func() error -} - -func newStepEvaluator(fn func() (bool, int64, promql.Vector), close func() error) (StepEvaluator, error) { - if fn == nil { - return nil, errors.New("nil step evaluator fn") - } - - if close == nil { - close = func() error { return nil } - } - - return &stepEvaluator{ - fn: fn, - close: close, - }, nil -} - -func (e *stepEvaluator) Next() (bool, int64, promql.Vector) { - return e.fn() -} - -func (e *stepEvaluator) Close() error { - return e.close() -} - type rangeAggregationExpr struct { left *logRange operation string @@ -348,6 +319,7 @@ func mustNewVectorAggregationExpr(left SampleExpr, operation string, gr *groupin if p, err = strconv.Atoi(*params); err != nil { panic(newParseError(fmt.Sprintf("invalid parameter %s(%s,", operation, *params), 0, 0)) } + default: if params != nil { panic(newParseError(fmt.Sprintf("unsupported parameter for operation %s(%s,", operation, *params), 0, 0)) @@ -409,6 +381,33 @@ func mustNewBinOpExpr(op string, lhs, rhs Expr) SampleExpr { rhs, ), 0, 0)) } + + leftLit, lOk := left.(*literalExpr) + rightLit, rOk := right.(*literalExpr) + + if IsLogicalBinOp(op) { + if lOk { + panic(newParseError(fmt.Sprintf( + "unexpected literal for left leg of logical/set binary operation (%s): %f", + op, + leftLit.value, + ), 0, 0)) + } + + if rOk { + panic(newParseError(fmt.Sprintf( + "unexpected literal for right leg of logical/set binary operation (%s): %f", + op, + rightLit.value, + ), 0, 0)) + } + } + + // map expr like (1+1) -> 2 + if lOk && rOk { + return reduceBinOp(op, leftLit, rightLit) + } + return &binOpExpr{ SampleExpr: left, RHS: right, @@ -416,6 +415,50 @@ func mustNewBinOpExpr(op string, lhs, rhs Expr) SampleExpr { } } +// Reduces a binary operation expression. A binop is reducable if both of its legs are literal expressions. +// This is because literals need match all labels, which is currently difficult to encode into StepEvaluators. +// Therefore, we ensure a binop can be reduced/simplified, maintaining the invariant that it does not have two literal legs. +func reduceBinOp(op string, left, right *literalExpr) *literalExpr { + merged := (&defaultEvaluator{}).mergeBinOp( + op, + &promql.Sample{Point: promql.Point{V: left.value}}, + &promql.Sample{Point: promql.Point{V: right.value}}, + ) + return &literalExpr{value: merged.V} +} + +type literalExpr struct { + value float64 +} + +func mustNewLiteralExpr(s string, invert bool) *literalExpr { + n, err := strconv.ParseFloat(s, 64) + if err != nil { + panic(newParseError(fmt.Sprintf("unable to parse literal as a float: %s", err.Error()), 0, 0)) + } + + if invert { + n = -n + } + + return &literalExpr{ + value: n, + } +} + +func (e *literalExpr) logQLExpr() {} + +func (e *literalExpr) String() string { + return fmt.Sprintf("%f", e.value) +} + +// literlExpr impls SampleExpr & LogSelectorExpr mainly to reduce the need for more complicated typings +// to facilitate sum types. We'll be type switching when evaluating them anyways +// and they will only be present in binary operation legs. +func (e *literalExpr) Selector() LogSelectorExpr { return e } +func (e *literalExpr) Filter() (Filter, error) { return nil, nil } +func (e *literalExpr) Matchers() []*labels.Matcher { return nil } + // helper used to impl Stringer for vector and range aggregations // nolint:interfacer func formatOperation(op string, grouping *grouping, params ...string) string { diff --git a/pkg/logql/engine.go b/pkg/logql/engine.go index 12d6572fa6e7d..cb94254458ae7 100644 --- a/pkg/logql/engine.go +++ b/pkg/logql/engine.go @@ -178,13 +178,6 @@ func (ng *engine) exec(ctx context.Context, q *query) (promql.Value, error) { defer cancel() qs := q.String() - // This is a legacy query used for health checking. Not the best practice, but it works. - if qs == "1+1" { - if GetRangeType(q) == InstantType { - return promql.Vector{}, nil - } - return promql.Matrix{}, nil - } expr, err := ParseExpr(qs) if err != nil { @@ -211,6 +204,9 @@ func (ng *engine) exec(ctx context.Context, q *query) (promql.Value, error) { // evalSample evaluate a sampleExpr func (ng *engine) evalSample(ctx context.Context, expr SampleExpr, q *query) (promql.Value, error) { + if lit, ok := expr.(*literalExpr); ok { + return ng.evalLiteral(ctx, lit, q) + } stepEvaluator, err := ng.evaluator.Evaluator(ctx, expr, q) if err != nil { @@ -225,7 +221,9 @@ func (ng *engine) evalSample(ctx context.Context, expr SampleExpr, q *query) (pr sort.Slice(vec, func(i, j int) bool { return labels.Compare(vec[i].Metric, vec[j].Metric) < 0 }) return vec, nil } + for next { + for _, p := range vec { var ( series *promql.Series @@ -257,6 +255,44 @@ func (ng *engine) evalSample(ctx context.Context, expr SampleExpr, q *query) (pr return result, nil } +func (ng *engine) evalLiteral(_ context.Context, expr *literalExpr, q *query) (promql.Value, error) { + s := promql.Scalar{ + T: q.Start().UnixNano() / int64(time.Millisecond), + V: expr.value, + } + + if GetRangeType(q) == InstantType { + return s, nil + } + + return PopulateMatrixFromScalar(s, q.LiteralParams), nil + +} + +func PopulateMatrixFromScalar(data promql.Scalar, params LiteralParams) promql.Matrix { + var ( + start = params.Start() + end = params.End() + step = params.Step() + series = promql.Series{ + Points: make( + []promql.Point, + 0, + // allocate enough space for all needed entries + int(params.End().Sub(params.Start())/params.Step())+1, + ), + } + ) + + for ts := start; !ts.After(end); ts = ts.Add(step) { + series.Points = append(series.Points, promql.Point{ + T: ts.UnixNano() / int64(time.Millisecond), + V: data.V, + }) + } + return promql.Matrix{series} +} + func readStreams(i iter.EntryIterator, size uint32) (Streams, error) { streams := map[string]*logproto.Stream{} respSize := uint32(0) diff --git a/pkg/logql/engine_test.go b/pkg/logql/engine_test.go index 347ce06687de3..b226d116c65df 100644 --- a/pkg/logql/engine_test.go +++ b/pkg/logql/engine_test.go @@ -288,6 +288,36 @@ func TestEngine_NewInstantQuery(t *testing.T) { promql.Sample{Point: promql.Point{T: 60 * 1000, V: 0.2}, Metric: labels.Labels{labels.Label{Name: "app", Value: "fuzz"}}}, }, }, + { + `bottomk(3,rate(({app=~"foo|bar"} |~".+bar")[1m])) without (app) + 1`, time.Unix(60, 0), logproto.FORWARD, 100, + [][]*logproto.Stream{ + {newStream(testSize, factor(10, identity), `{app="foo"}`), newStream(testSize, offset(46, identity), `{app="bar"}`), + newStream(testSize, factor(5, identity), `{app="fuzz"}`), newStream(testSize, identity, `{app="buzz"}`)}, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(60, 0), Limit: 0, Selector: `{app=~"foo|bar"}|~".+bar"`}}, + }, + promql.Vector{ + promql.Sample{Point: promql.Point{T: 60 * 1000, V: 1.25}, Metric: labels.Labels{labels.Label{Name: "app", Value: "bar"}}}, + promql.Sample{Point: promql.Point{T: 60 * 1000, V: 1.1}, Metric: labels.Labels{labels.Label{Name: "app", Value: "foo"}}}, + promql.Sample{Point: promql.Point{T: 60 * 1000, V: 1.2}, Metric: labels.Labels{labels.Label{Name: "app", Value: "fuzz"}}}, + }, + }, + { + // healthcheck + `1+1`, time.Unix(60, 0), logproto.FORWARD, 100, + [][]*logproto.Stream{}, + []SelectParams{}, + promql.Scalar{T: 60 * 1000, V: 2}, + }, + { + // single literal + `2`, + time.Unix(60, 0), logproto.FORWARD, 100, + [][]*logproto.Stream{}, + []SelectParams{}, + promql.Scalar{T: 60 * 1000, V: 2}, + }, } { test := test t.Run(fmt.Sprintf("%s %s", test.qs, test.direction), func(t *testing.T) { @@ -945,9 +975,72 @@ func TestEngine_NewRangeQuery(t *testing.T) { }, }, { - ` - count_over_time({app="bar"}[1m]) ^ count_over_time({app="bar"}[1m]) - `, + `1+1--1`, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{}, + []SelectParams{}, + promql.Matrix{ + promql.Series{ + Points: []promql.Point{{T: 60000, V: 3}, {T: 90000, V: 3}, {T: 120000, V: 3}, {T: 150000, V: 3}, {T: 180000, V: 3}}, + }, + }, + }, + { + `rate({app="bar"}[1m]) - 1`, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: -0.8}, {T: 90 * 1000, V: -0.8}, {T: 120 * 1000, V: -0.8}, {T: 150 * 1000, V: -0.8}, {T: 180 * 1000, V: -0.8}}, + }, + }, + }, + { + `1 - rate({app="bar"}[1m])`, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.8}, {T: 90 * 1000, V: 0.8}, {T: 120 * 1000, V: 0.8}, {T: 150 * 1000, V: 0.8}, {T: 180 * 1000, V: 0.8}}, + }, + }, + }, + { + `rate({app="bar"}[1m]) - 1 / 2`, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: -0.3}, {T: 90 * 1000, V: -0.3}, {T: 120 * 1000, V: -0.3}, {T: 150 * 1000, V: -0.3}, {T: 180 * 1000, V: -0.3}}, + }, + }, + }, + { + `count_over_time({app="bar"}[1m]) ^ count_over_time({app="bar"}[1m])`, time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, [][]*logproto.Stream{ { @@ -964,6 +1057,17 @@ func TestEngine_NewRangeQuery(t *testing.T) { }, }, }, + { + `2`, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{}, + []SelectParams{}, + promql.Matrix{ + promql.Series{ + Points: []promql.Point{{T: 60 * 1000, V: 2}, {T: 90 * 1000, V: 2}, {T: 120 * 1000, V: 2}, {T: 150 * 1000, V: 2}, {T: 180 * 1000, V: 2}}, + }, + }, + }, } { test := test t.Run(fmt.Sprintf("%s %s", test.qs, test.direction), func(t *testing.T) { diff --git a/pkg/logql/evaluator.go b/pkg/logql/evaluator.go index b7539121228c2..2140eda558228 100644 --- a/pkg/logql/evaluator.go +++ b/pkg/logql/evaluator.go @@ -106,7 +106,6 @@ func (ev *defaultEvaluator) Evaluator(ctx context.Context, expr SampleExpr, q Pa return ev.rangeAggEvaluator(ctx, e, q) case *binOpExpr: return ev.binOpEvaluator(ctx, e, q) - default: return nil, errors.Errorf("unexpected type (%T): %v", e, e) } @@ -120,6 +119,7 @@ func (ev *defaultEvaluator) vectorAggEvaluator(ctx context.Context, expr *vector return newStepEvaluator(func() (bool, int64, promql.Vector) { next, ts, vec := nextEvaluator.Next() + if !next { return false, 0, promql.Vector{} } @@ -338,11 +338,34 @@ func (ev *defaultEvaluator) rangeAggEvaluator(ctx context.Context, expr *rangeAg }, vecIter.Close) } +// binOpExpr explicly does not handle when both legs are literals as +// it makes the type system simpler and these are reduced in mustNewBinOpExpr func (ev *defaultEvaluator) binOpEvaluator( ctx context.Context, expr *binOpExpr, q Params, ) (StepEvaluator, error) { + // first check if either side is a literal + leftLit, lOk := expr.SampleExpr.(*literalExpr) + rightLit, rOk := expr.RHS.(*literalExpr) + + // match a literal expr with all labels in the other leg + if lOk { + rhs, err := ev.Evaluator(ctx, expr.RHS, q) + if err != nil { + return nil, err + } + return ev.literalEvaluator(expr.op, leftLit, rhs, false) + } + if rOk { + lhs, err := ev.Evaluator(ctx, expr.SampleExpr, q) + if err != nil { + return nil, err + } + return ev.literalEvaluator(expr.op, rightLit, lhs, true) + } + + // we have two non literal legs lhs, err := ev.Evaluator(ctx, expr.SampleExpr, q) if err != nil { return nil, err @@ -359,6 +382,7 @@ func (ev *defaultEvaluator) binOpEvaluator( // populate pairs for i, eval := range []StepEvaluator{lhs, rhs} { next, timestamp, vec := eval.Next() + ts = timestamp // These should _always_ happen at the same step on each evaluator. @@ -528,3 +552,42 @@ func (ev *defaultEvaluator) mergeBinOp(op string, left, right *promql.Sample) *p return merger(left, right) } + +// literalEvaluator merges a literal with a StepEvaluator. Since order matters in +// non commutative operations, inverted should be true when the literalExpr is not the left argument. +func (ev *defaultEvaluator) literalEvaluator( + op string, + lit *literalExpr, + eval StepEvaluator, + inverted bool, +) (StepEvaluator, error) { + return newStepEvaluator( + func() (bool, int64, promql.Vector) { + ok, ts, vec := eval.Next() + + results := make(promql.Vector, 0, len(vec)) + for _, sample := range vec { + literalPoint := promql.Sample{ + Metric: sample.Metric, + Point: promql.Point{T: ts, V: lit.value}, + } + + left, right := &literalPoint, &sample + if inverted { + left, right = right, left + } + + if merged := ev.mergeBinOp( + op, + left, + right, + ); merged != nil { + results = append(results, *merged) + } + } + + return ok, ts, results + }, + eval.Close, + ) +} diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index 9a811985191f6..ee042818c35c6 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -25,7 +25,7 @@ import ( binOp string str string duration time.Duration - int int64 + LiteralExpr *literalExpr } %start root @@ -44,8 +44,9 @@ import ( %type vectorAggregationExpr %type vectorOp %type binOpExpr +%type literalExpr -%token IDENTIFIER STRING +%token IDENTIFIER STRING NUMBER %token DURATION %token MATCHERS LABELS EQ NEQ RE NRE OPEN_BRACE CLOSE_BRACE OPEN_BRACKET CLOSE_BRACKET COMMA DOT PIPE_MATCH PIPE_EXACT OPEN_PARENTHESIS CLOSE_PARENTHESIS BY WITHOUT COUNT_OVER_TIME RATE SUM AVG MAX MIN COUNT STDDEV STDVAR BOTTOMK TOPK @@ -66,6 +67,7 @@ expr: | rangeAggregationExpr { $$ = $1 } | vectorAggregationExpr { $$ = $1 } | binOpExpr { $$ = $1 } + | literalExpr { $$ = $1 } | OPEN_PARENTHESIS expr CLOSE_PARENTHESIS { $$ = $2 } ; @@ -96,10 +98,10 @@ vectorAggregationExpr: | vectorOp OPEN_PARENTHESIS rangeAggregationExpr CLOSE_PARENTHESIS grouping { $$ = mustNewVectorAggregationExpr($3, $1, $5, nil) } | vectorOp OPEN_PARENTHESIS vectorAggregationExpr CLOSE_PARENTHESIS grouping { $$ = mustNewVectorAggregationExpr($3, $1, $5, nil) } // Aggregations with 2 arguments. - | vectorOp OPEN_PARENTHESIS IDENTIFIER COMMA vectorAggregationExpr CLOSE_PARENTHESIS { $$ = mustNewVectorAggregationExpr($5, $1, nil, &$3) } - | vectorOp OPEN_PARENTHESIS IDENTIFIER COMMA vectorAggregationExpr CLOSE_PARENTHESIS grouping { $$ = mustNewVectorAggregationExpr($5, $1, $7, &$3) } - | vectorOp OPEN_PARENTHESIS IDENTIFIER COMMA rangeAggregationExpr CLOSE_PARENTHESIS { $$ = mustNewVectorAggregationExpr($5, $1, nil, &$3) } - | vectorOp OPEN_PARENTHESIS IDENTIFIER COMMA rangeAggregationExpr CLOSE_PARENTHESIS grouping { $$ = mustNewVectorAggregationExpr($5, $1, $7, &$3) } + | vectorOp OPEN_PARENTHESIS NUMBER COMMA vectorAggregationExpr CLOSE_PARENTHESIS { $$ = mustNewVectorAggregationExpr($5, $1, nil, &$3) } + | vectorOp OPEN_PARENTHESIS NUMBER COMMA vectorAggregationExpr CLOSE_PARENTHESIS grouping { $$ = mustNewVectorAggregationExpr($5, $1, $7, &$3) } + | vectorOp OPEN_PARENTHESIS NUMBER COMMA rangeAggregationExpr CLOSE_PARENTHESIS { $$ = mustNewVectorAggregationExpr($5, $1, nil, &$3) } + | vectorOp OPEN_PARENTHESIS NUMBER COMMA rangeAggregationExpr CLOSE_PARENTHESIS grouping { $$ = mustNewVectorAggregationExpr($5, $1, $7, &$3) } ; filter: @@ -143,6 +145,11 @@ binOpExpr: | expr MOD expr { $$ = mustNewBinOpExpr("%", $1, $3) } | expr POW expr { $$ = mustNewBinOpExpr("^", $1, $3) } +literalExpr: + NUMBER { $$ = mustNewLiteralExpr( $1, false ) } + | ADD NUMBER { $$ = mustNewLiteralExpr( $2, false ) } + | SUB NUMBER { $$ = mustNewLiteralExpr( $2, true ) } + vectorOp: SUM { $$ = OpTypeSum } | AVG { $$ = OpTypeAvg } diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index 6e2a851a6a0a8..60052c3d7f667 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -29,50 +29,51 @@ type exprSymType struct { binOp string str string duration time.Duration - int int64 + LiteralExpr *literalExpr } const IDENTIFIER = 57346 const STRING = 57347 -const DURATION = 57348 -const MATCHERS = 57349 -const LABELS = 57350 -const EQ = 57351 -const NEQ = 57352 -const RE = 57353 -const NRE = 57354 -const OPEN_BRACE = 57355 -const CLOSE_BRACE = 57356 -const OPEN_BRACKET = 57357 -const CLOSE_BRACKET = 57358 -const COMMA = 57359 -const DOT = 57360 -const PIPE_MATCH = 57361 -const PIPE_EXACT = 57362 -const OPEN_PARENTHESIS = 57363 -const CLOSE_PARENTHESIS = 57364 -const BY = 57365 -const WITHOUT = 57366 -const COUNT_OVER_TIME = 57367 -const RATE = 57368 -const SUM = 57369 -const AVG = 57370 -const MAX = 57371 -const MIN = 57372 -const COUNT = 57373 -const STDDEV = 57374 -const STDVAR = 57375 -const BOTTOMK = 57376 -const TOPK = 57377 -const OR = 57378 -const AND = 57379 -const UNLESS = 57380 -const ADD = 57381 -const SUB = 57382 -const MUL = 57383 -const DIV = 57384 -const MOD = 57385 -const POW = 57386 +const NUMBER = 57348 +const DURATION = 57349 +const MATCHERS = 57350 +const LABELS = 57351 +const EQ = 57352 +const NEQ = 57353 +const RE = 57354 +const NRE = 57355 +const OPEN_BRACE = 57356 +const CLOSE_BRACE = 57357 +const OPEN_BRACKET = 57358 +const CLOSE_BRACKET = 57359 +const COMMA = 57360 +const DOT = 57361 +const PIPE_MATCH = 57362 +const PIPE_EXACT = 57363 +const OPEN_PARENTHESIS = 57364 +const CLOSE_PARENTHESIS = 57365 +const BY = 57366 +const WITHOUT = 57367 +const COUNT_OVER_TIME = 57368 +const RATE = 57369 +const SUM = 57370 +const AVG = 57371 +const MAX = 57372 +const MIN = 57373 +const COUNT = 57374 +const STDDEV = 57375 +const STDVAR = 57376 +const BOTTOMK = 57377 +const TOPK = 57378 +const OR = 57379 +const AND = 57380 +const UNLESS = 57381 +const ADD = 57382 +const SUB = 57383 +const MUL = 57384 +const DIV = 57385 +const MOD = 57386 +const POW = 57387 var exprToknames = [...]string{ "$end", @@ -80,6 +81,7 @@ var exprToknames = [...]string{ "$unk", "IDENTIFIER", "STRING", + "NUMBER", "DURATION", "MATCHERS", "LABELS", @@ -133,8 +135,7 @@ var exprExca = [...]int{ -2, 0, -1, 3, 1, 2, - 22, 2, - 36, 2, + 23, 2, 37, 2, 38, 2, 39, 2, @@ -143,9 +144,9 @@ var exprExca = [...]int{ 42, 2, 43, 2, 44, 2, + 45, 2, -2, 0, - -1, 39, - 36, 2, + -1, 43, 37, 2, 38, 2, 39, 2, @@ -154,106 +155,111 @@ var exprExca = [...]int{ 42, 2, 43, 2, 44, 2, + 45, 2, -2, 0, } const exprPrivate = 57344 -const exprLast = 202 +const exprLast = 220 var exprAct = [...]int{ - 42, 5, 4, 32, 47, 90, 60, 3, 62, 26, - 27, 28, 29, 30, 31, 39, 28, 29, 30, 31, - 23, 24, 25, 26, 27, 28, 29, 30, 31, 23, - 24, 25, 26, 27, 28, 29, 30, 31, 31, 41, - 70, 43, 44, 66, 65, 43, 44, 111, 63, 24, - 25, 26, 27, 28, 29, 30, 31, 33, 107, 107, - 110, 82, 106, 109, 108, 37, 79, 36, 105, 86, - 89, 88, 83, 84, 34, 35, 92, 61, 93, 85, - 69, 68, 40, 11, 87, 11, 101, 102, 79, 103, - 104, 7, 67, 64, 72, 12, 13, 14, 15, 17, - 18, 16, 19, 20, 21, 22, 71, 74, 97, 73, - 96, 113, 114, 12, 13, 14, 15, 17, 18, 16, - 19, 20, 21, 22, 12, 13, 14, 15, 17, 18, - 16, 19, 20, 21, 22, 2, 80, 95, 99, 59, - 33, 98, 58, 38, 37, 94, 36, 46, 37, 48, - 36, 112, 48, 34, 35, 91, 100, 34, 35, 49, - 50, 51, 52, 53, 54, 55, 56, 57, 80, 6, - 10, 8, 33, 9, 45, 1, 37, 0, 36, 0, - 37, 0, 36, 33, 0, 34, 35, 82, 81, 34, - 35, 37, 61, 36, 75, 76, 77, 78, 0, 0, - 34, 35, + 46, 5, 4, 36, 53, 96, 12, 3, 68, 32, + 33, 34, 35, 35, 15, 45, 43, 47, 48, 47, + 48, 117, 8, 93, 116, 112, 16, 17, 18, 19, + 21, 22, 20, 23, 24, 25, 26, 113, 113, 66, + 13, 14, 115, 114, 111, 92, 91, 72, 71, 76, + 75, 74, 69, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 27, 28, 29, 30, 31, 32, 33, 34, + 35, 44, 85, 80, 50, 15, 95, 94, 89, 90, + 49, 103, 98, 70, 99, 30, 31, 32, 33, 34, + 35, 102, 107, 108, 85, 109, 110, 28, 29, 30, + 31, 32, 33, 34, 35, 73, 16, 17, 18, 19, + 21, 22, 20, 23, 24, 25, 26, 119, 120, 81, + 82, 83, 84, 101, 100, 16, 17, 18, 19, 21, + 22, 20, 23, 24, 25, 26, 2, 37, 78, 7, + 105, 86, 88, 104, 65, 42, 41, 64, 40, 118, + 41, 77, 40, 54, 79, 38, 39, 97, 67, 38, + 39, 52, 106, 54, 55, 56, 57, 58, 59, 60, + 61, 62, 63, 86, 6, 11, 9, 37, 10, 51, + 1, 0, 41, 0, 40, 0, 41, 0, 40, 0, + 0, 38, 39, 0, 87, 38, 39, 37, 67, 0, + 37, 0, 88, 0, 0, 0, 41, 0, 40, 41, + 0, 40, 0, 0, 0, 38, 39, 0, 38, 39, } var exprPact = [...]int{ - 70, -1000, -7, 138, -1000, -1000, -1000, 70, -1000, 61, - 18, 145, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, 70, 70, 70, 70, 70, 70, 70, - 70, 70, 137, -1000, -1000, -1000, -1000, -1000, -16, 170, - 72, 88, 60, 59, 19, 92, 93, -1000, 185, 12, - -30, -30, -25, -25, -6, -6, -6, -6, -1000, -1000, - -1000, -1000, 166, 181, 72, 57, 47, 67, 99, 151, - 151, -1000, -1000, 148, -1000, 140, 132, 105, 103, 136, - -1000, -1000, -1000, 55, 134, 22, 22, 99, 46, 40, - 42, -1000, 41, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, 38, 25, -1000, -1000, 147, -1000, -1000, - 22, 22, -1000, -1000, -1000, + 0, -1000, 25, 198, -1000, -1000, -1000, -1000, 0, -1000, + 49, -7, -1000, 74, 68, 159, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 142, -1000, -1000, -1000, + -1000, -1000, 16, 175, 61, 99, 29, 28, 27, -1000, + -1000, 136, 58, -1000, 109, 59, 45, 45, -33, -33, + -32, -32, -32, -32, -1000, -1000, -1000, -1000, 171, 195, + 61, 23, 22, 5, 80, 153, 153, -1000, -1000, 149, + -1000, 119, 118, 86, 76, 138, -1000, -1000, -1000, 135, + 139, -5, -5, 80, 21, 2, 20, -1000, 19, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 1, + -2, -1000, -1000, 145, -1000, -1000, -5, -5, -1000, -1000, + -1000, } var exprPgo = [...]int{ - 0, 175, 135, 3, 0, 5, 7, 8, 4, 174, - 2, 173, 171, 1, 170, 169, + 0, 180, 136, 3, 0, 5, 7, 8, 4, 179, + 2, 178, 176, 1, 175, 174, 139, } var exprR1 = [...]int{ - 0, 1, 2, 2, 2, 2, 2, 6, 6, 6, - 6, 6, 7, 7, 7, 7, 7, 10, 13, 13, - 13, 13, 13, 13, 13, 13, 13, 13, 3, 3, - 3, 3, 12, 12, 12, 9, 9, 8, 8, 8, - 8, 15, 15, 15, 15, 15, 15, 15, 15, 15, - 14, 14, 14, 14, 14, 14, 14, 14, 14, 11, - 11, 5, 5, 4, 4, + 0, 1, 2, 2, 2, 2, 2, 2, 6, 6, + 6, 6, 6, 7, 7, 7, 7, 7, 10, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 3, + 3, 3, 3, 12, 12, 12, 9, 9, 8, 8, + 8, 8, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 16, 16, 16, 14, 14, 14, 14, 14, 14, + 14, 14, 14, 11, 11, 5, 5, 4, 4, } var exprR2 = [...]int{ - 0, 1, 1, 1, 1, 1, 3, 1, 3, 3, - 3, 2, 2, 3, 3, 3, 2, 4, 4, 4, - 5, 5, 5, 5, 6, 7, 6, 7, 1, 1, - 1, 1, 3, 3, 3, 1, 3, 3, 3, 3, + 0, 1, 1, 1, 1, 1, 1, 3, 1, 3, + 3, 3, 2, 2, 3, 3, 3, 2, 4, 4, + 4, 5, 5, 5, 5, 6, 7, 6, 7, 1, + 1, 1, 1, 3, 3, 3, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 3, 4, 4, + 3, 1, 2, 2, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 3, 4, 4, } var exprChk = [...]int{ - -1000, -1, -2, -6, -10, -13, -15, 21, -12, -11, - -14, 13, 25, 26, 27, 28, 31, 29, 30, 32, - 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, - 43, 44, -3, 2, 19, 20, 12, 10, -2, -6, - 21, 21, -4, 23, 24, -9, 2, -8, 4, -2, - -2, -2, -2, -2, -2, -2, -2, -2, 5, 2, - 22, 22, -7, -6, 21, -10, -13, 4, 21, 21, - 21, 14, 2, 17, 14, 9, 10, 11, 12, -3, - 2, 22, 6, -6, -7, 22, 22, 17, -10, -13, - -5, 4, -5, -8, 5, 5, 5, 5, 5, 2, - 22, -4, -4, -13, -10, 22, 22, 17, 22, 22, - 22, 22, 4, -4, -4, + -1000, -1, -2, -6, -10, -13, -15, -16, 22, -12, + -11, -14, 6, 40, 41, 14, 26, 27, 28, 29, + 32, 30, 31, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, -3, 2, 20, 21, + 13, 11, -2, -6, 22, 22, -4, 24, 25, 6, + 6, -9, 2, -8, 4, -2, -2, -2, -2, -2, + -2, -2, -2, -2, 5, 2, 23, 23, -7, -6, + 22, -10, -13, 6, 22, 22, 22, 15, 2, 18, + 15, 10, 11, 12, 13, -3, 2, 23, 7, -6, + -7, 23, 23, 18, -10, -13, -5, 4, -5, -8, + 5, 5, 5, 5, 5, 2, 23, -4, -4, -13, + -10, 23, 23, 18, 23, 23, 23, 23, 4, -4, + -4, } var exprDef = [...]int{ - 0, -2, 1, -2, 3, 4, 5, 0, 7, 0, - 0, 0, 59, 60, 50, 51, 52, 53, 54, 55, - 56, 57, 58, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 11, 28, 29, 30, 31, 0, -2, - 0, 0, 0, 0, 0, 0, 0, 35, 0, 41, - 42, 43, 44, 45, 46, 47, 48, 49, 8, 10, - 6, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 32, 33, 0, 34, 0, 0, 0, 0, 0, - 16, 17, 12, 0, 0, 18, 19, 0, 0, 0, - 0, 61, 0, 36, 37, 38, 39, 40, 13, 15, - 14, 22, 23, 0, 0, 20, 21, 0, 63, 64, - 24, 26, 62, 25, 27, + 0, -2, 1, -2, 3, 4, 5, 6, 0, 8, + 0, 0, 51, 0, 0, 0, 63, 64, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 12, 29, 30, + 31, 32, 0, -2, 0, 0, 0, 0, 0, 52, + 53, 0, 0, 36, 0, 42, 43, 44, 45, 46, + 47, 48, 49, 50, 9, 11, 7, 10, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 33, 34, 0, + 35, 0, 0, 0, 0, 0, 17, 18, 13, 0, + 0, 19, 20, 0, 0, 0, 0, 65, 0, 37, + 38, 39, 40, 41, 14, 16, 15, 23, 24, 0, + 0, 21, 22, 0, 67, 68, 25, 27, 66, 26, + 28, } var exprTok1 = [...]int{ @@ -265,7 +271,7 @@ var exprTok2 = [...]int{ 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, - 42, 43, 44, + 42, 43, 44, 45, } var exprTok3 = [...]int{ 0, @@ -633,275 +639,295 @@ exprdefault: exprVAL.Expr = exprDollar[1].BinOpExpr } case 6: + exprDollar = exprS[exprpt-1 : exprpt+1] + { + exprVAL.Expr = exprDollar[1].LiteralExpr + } + case 7: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Expr = exprDollar[2].Expr } - case 7: + case 8: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.LogExpr = newMatcherExpr(exprDollar[1].Selector) } - case 8: + case 9: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LogExpr = NewFilterExpr(exprDollar[1].LogExpr, exprDollar[2].Filter, exprDollar[3].str) } - case 9: + case 10: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LogExpr = exprDollar[2].LogExpr } - case 12: + case 13: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.LogRangeExpr = newLogRange(exprDollar[1].LogExpr, exprDollar[2].duration) } - case 13: + case 14: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LogRangeExpr = addFilterToLogRangeExpr(exprDollar[1].LogRangeExpr, exprDollar[2].Filter, exprDollar[3].str) } - case 14: + case 15: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LogRangeExpr = exprDollar[2].LogRangeExpr } - case 17: + case 18: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.RangeAggregationExpr = newRangeAggregationExpr(exprDollar[3].LogRangeExpr, exprDollar[1].RangeOp) } - case 18: + case 19: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].RangeAggregationExpr, exprDollar[1].VectorOp, nil, nil) } - case 19: + case 20: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].VectorAggregationExpr, exprDollar[1].VectorOp, nil, nil) } - case 20: + case 21: exprDollar = exprS[exprpt-5 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[4].RangeAggregationExpr, exprDollar[1].VectorOp, exprDollar[2].Grouping, nil) } - case 21: + case 22: exprDollar = exprS[exprpt-5 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[4].VectorAggregationExpr, exprDollar[1].VectorOp, exprDollar[2].Grouping, nil) } - case 22: + case 23: exprDollar = exprS[exprpt-5 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].RangeAggregationExpr, exprDollar[1].VectorOp, exprDollar[5].Grouping, nil) } - case 23: + case 24: exprDollar = exprS[exprpt-5 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].VectorAggregationExpr, exprDollar[1].VectorOp, exprDollar[5].Grouping, nil) } - case 24: + case 25: exprDollar = exprS[exprpt-6 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].VectorAggregationExpr, exprDollar[1].VectorOp, nil, &exprDollar[3].str) } - case 25: + case 26: exprDollar = exprS[exprpt-7 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].VectorAggregationExpr, exprDollar[1].VectorOp, exprDollar[7].Grouping, &exprDollar[3].str) } - case 26: + case 27: exprDollar = exprS[exprpt-6 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].RangeAggregationExpr, exprDollar[1].VectorOp, nil, &exprDollar[3].str) } - case 27: + case 28: exprDollar = exprS[exprpt-7 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].RangeAggregationExpr, exprDollar[1].VectorOp, exprDollar[7].Grouping, &exprDollar[3].str) } - case 28: + case 29: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Filter = labels.MatchRegexp } - case 29: + case 30: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Filter = labels.MatchEqual } - case 30: + case 31: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Filter = labels.MatchNotRegexp } - case 31: + case 32: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Filter = labels.MatchNotEqual } - case 32: + case 33: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Selector = exprDollar[2].Matchers } - case 33: + case 34: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Selector = exprDollar[2].Matchers } - case 34: + case 35: exprDollar = exprS[exprpt-3 : exprpt+1] { } - case 35: + case 36: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Matchers = []*labels.Matcher{exprDollar[1].Matcher} } - case 36: + case 37: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matchers = append(exprDollar[1].Matchers, exprDollar[3].Matcher) } - case 37: + case 38: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matcher = mustNewMatcher(labels.MatchEqual, exprDollar[1].str, exprDollar[3].str) } - case 38: + case 39: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matcher = mustNewMatcher(labels.MatchNotEqual, exprDollar[1].str, exprDollar[3].str) } - case 39: + case 40: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matcher = mustNewMatcher(labels.MatchRegexp, exprDollar[1].str, exprDollar[3].str) } - case 40: + case 41: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matcher = mustNewMatcher(labels.MatchNotRegexp, exprDollar[1].str, exprDollar[3].str) } - case 41: + case 42: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("or", exprDollar[1].Expr, exprDollar[3].Expr) } - case 42: + case 43: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("and", exprDollar[1].Expr, exprDollar[3].Expr) } - case 43: + case 44: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("unless", exprDollar[1].Expr, exprDollar[3].Expr) } - case 44: + case 45: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("+", exprDollar[1].Expr, exprDollar[3].Expr) } - case 45: + case 46: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("-", exprDollar[1].Expr, exprDollar[3].Expr) } - case 46: + case 47: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("*", exprDollar[1].Expr, exprDollar[3].Expr) } - case 47: + case 48: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("/", exprDollar[1].Expr, exprDollar[3].Expr) } - case 48: + case 49: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("%", exprDollar[1].Expr, exprDollar[3].Expr) } - case 49: + case 50: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("^", exprDollar[1].Expr, exprDollar[3].Expr) } - case 50: + case 51: + exprDollar = exprS[exprpt-1 : exprpt+1] + { + exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[1].str, false) + } + case 52: + exprDollar = exprS[exprpt-2 : exprpt+1] + { + exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[2].str, false) + } + case 53: + exprDollar = exprS[exprpt-2 : exprpt+1] + { + exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[2].str, true) + } + case 54: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeSum } - case 51: + case 55: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeAvg } - case 52: + case 56: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeCount } - case 53: + case 57: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMax } - case 54: + case 58: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMin } - case 55: + case 59: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStddev } - case 56: + case 60: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStdvar } - case 57: + case 61: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeBottomK } - case 58: + case 62: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeTopK } - case 59: + case 63: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeCountOverTime } - case 60: + case 64: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeRate } - case 61: + case 65: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Labels = []string{exprDollar[1].str} } - case 62: + case 66: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Labels = append(exprDollar[1].Labels, exprDollar[3].str) } - case 63: + case 67: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: false, groups: exprDollar[3].Labels} } - case 64: + case 68: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: true, groups: exprDollar[3].Labels} diff --git a/pkg/logql/lex.go b/pkg/logql/lex.go index eedc0dbc9b572..cd557c27f71ab 100644 --- a/pkg/logql/lex.go +++ b/pkg/logql/lex.go @@ -62,6 +62,10 @@ func (l *lexer) Lex(lval *exprSymType) int { case scanner.EOF: return 0 + case scanner.Int, scanner.Float: + lval.str = l.TokenText() + return NUMBER + case scanner.String: var err error lval.str, err = strconv.Unquote(l.TokenText()) diff --git a/pkg/logql/lex_test.go b/pkg/logql/lex_test.go index 9505a9fc1450e..0a433caa2c3e8 100644 --- a/pkg/logql/lex_test.go +++ b/pkg/logql/lex_test.go @@ -24,8 +24,8 @@ func TestLex(t *testing.T) { {`rate({foo="bar"}[10s])`, []int{RATE, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS}}, {`count_over_time({foo="bar"}[5m])`, []int{COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS}}, {`sum(count_over_time({foo="bar"}[5m])) by (foo,bar)`, []int{SUM, OPEN_PARENTHESIS, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS}}, - {`topk(3,count_over_time({foo="bar"}[5m])) by (foo,bar)`, []int{TOPK, OPEN_PARENTHESIS, IDENTIFIER, COMMA, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS}}, - {`bottomk(10,sum(count_over_time({foo="bar"}[5m])) by (foo,bar))`, []int{BOTTOMK, OPEN_PARENTHESIS, IDENTIFIER, COMMA, SUM, OPEN_PARENTHESIS, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS}}, + {`topk(3,count_over_time({foo="bar"}[5m])) by (foo,bar)`, []int{TOPK, OPEN_PARENTHESIS, NUMBER, COMMA, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS}}, + {`bottomk(10,sum(count_over_time({foo="bar"}[5m])) by (foo,bar))`, []int{BOTTOMK, OPEN_PARENTHESIS, NUMBER, COMMA, SUM, OPEN_PARENTHESIS, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS}}, {`sum(max(rate({foo="bar"}[5m])) by (foo,bar)) by (foo)`, []int{SUM, OPEN_PARENTHESIS, MAX, OPEN_PARENTHESIS, RATE, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, CLOSE_PARENTHESIS}}, } { t.Run(tc.input, func(t *testing.T) { diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index a2c9d0103427f..3a35fb4f76261 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -219,7 +219,15 @@ func TestParse(t *testing.T) { { in: `bottomk(he,count_over_time({ foo !~ "bar" }[5h]))`, err: ParseError{ - msg: "invalid parameter bottomk(he,", + msg: "syntax error: unexpected IDENTIFIER", + line: 1, + col: 9, + }, + }, + { + in: `bottomk(1.2,count_over_time({ foo !~ "bar" }[5h]))`, + err: ParseError{ + msg: "invalid parameter bottomk(1.2,", line: 0, col: 0, }, @@ -547,6 +555,68 @@ func TestParse(t *testing.T) { col: 13, }, }, + { + // require left associativity + in: ` + sum(count_over_time({foo="bar"}[5m])) by (foo) / + sum(count_over_time({foo="bar"}[5m])) by (foo) / + sum(count_over_time({foo="bar"}[5m])) by (foo) + `, + exp: mustNewBinOpExpr( + OpTypeDiv, + mustNewBinOpExpr( + OpTypeDiv, + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + ), + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + ), + }, { in: ` sum(count_over_time({foo="bar"}[5m])) by (foo) ^ @@ -670,6 +740,54 @@ func TestParse(t *testing.T) { ), ), }, + { + // reduces binop with two literalExprs + in: `sum(count_over_time({foo="bar"}[5m])) by (foo) + 1 / 2`, + exp: mustNewBinOpExpr( + OpTypeAdd, + mustNewVectorAggregationExpr( + newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + &literalExpr{value: 0.5}, + ), + }, + { + // test signs + in: `1 + -2 / 1`, + exp: mustNewBinOpExpr( + OpTypeAdd, + &literalExpr{value: 1}, + mustNewBinOpExpr(OpTypeDiv, &literalExpr{value: -2}, &literalExpr{value: 1}), + ), + }, + { + // test signs/ops with equal associativity + in: `1 + 1 - -1`, + exp: mustNewBinOpExpr( + OpTypeSub, + mustNewBinOpExpr(OpTypeAdd, &literalExpr{value: 1}, &literalExpr{value: 1}), + &literalExpr{value: -1}, + ), + }, + { + // ensure binary ops with two literals are reduced recursively + in: `1 + 1 + 1`, + exp: &literalExpr{value: 3}, + }, { in: `{foo="bar"} + {foo="bar"}`, err: ParseError{ @@ -694,6 +812,30 @@ func TestParse(t *testing.T) { col: 0, }, }, + { + in: `sum(count_over_time({foo="bar"}[5m])) by (foo) or 1`, + err: ParseError{ + msg: `unexpected literal for right leg of logical/set binary operation (or): 1.000000`, + line: 0, + col: 0, + }, + }, + { + in: `1 unless sum(count_over_time({foo="bar"}[5m])) by (foo)`, + err: ParseError{ + msg: `unexpected literal for left leg of logical/set binary operation (unless): 1.000000`, + line: 0, + col: 0, + }, + }, + { + in: `sum(count_over_time({foo="bar"}[5m])) by (foo) + 1 or 1`, + err: ParseError{ + msg: `unexpected literal for right leg of logical/set binary operation (or): 1.000000`, + line: 0, + col: 0, + }, + }, } { t.Run(tc.in, func(t *testing.T) { ast, err := ParseExpr(tc.in) diff --git a/pkg/logql/step_evaluator.go b/pkg/logql/step_evaluator.go new file mode 100644 index 0000000000000..571b2453a350d --- /dev/null +++ b/pkg/logql/step_evaluator.go @@ -0,0 +1,43 @@ +package logql + +import ( + "errors" + + "github.com/prometheus/prometheus/promql" +) + +// StepEvaluator evaluate a single step of a query. +type StepEvaluator interface { + // while Next returns a promql.Value, the only acceptable types are Scalar and Vector. + Next() (bool, int64, promql.Vector) + // Close all resources used. + Close() error +} + +type stepEvaluator struct { + fn func() (bool, int64, promql.Vector) + close func() error +} + +func newStepEvaluator(fn func() (bool, int64, promql.Vector), close func() error) (StepEvaluator, error) { + if fn == nil { + return nil, errors.New("nil step evaluator fn") + } + + if close == nil { + close = func() error { return nil } + } + + return &stepEvaluator{ + fn: fn, + close: close, + }, nil +} + +func (e *stepEvaluator) Next() (bool, int64, promql.Vector) { + return e.fn() +} + +func (e *stepEvaluator) Close() error { + return e.close() +}