From 11573ccc54b2b6cdabf9ed604f916802c3e81655 Mon Sep 17 00:00:00 2001 From: garrettlish Date: Tue, 28 Sep 2021 10:50:40 +0800 Subject: [PATCH 1/6] add on and ignoring clauses in binOpExpr --- pkg/logql/ast.go | 31 ++- pkg/logql/engine_test.go | 66 +++++ pkg/logql/evaluator.go | 38 ++- pkg/logql/expr.y | 37 ++- pkg/logql/expr.y.go | 467 ++++++++++++++++++---------------- pkg/logql/lex.go | 2 + pkg/logql/parser_test.go | 210 +++++++++++++-- pkg/logql/shardmapper_test.go | 10 +- 8 files changed, 617 insertions(+), 244 deletions(-) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 0a54651b6e1ff..98f99968de90e 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -712,6 +712,9 @@ const ( OpUnwrap = "unwrap" OpOffset = "offset" + OpOn = "on" + OpIgnoring = "ignoring" + // conversion Op OpConvBytes = "bytes" OpConvDuration = "duration" @@ -959,22 +962,38 @@ func (e *VectorAggregationExpr) Walk(f WalkFn) { e.left.Walk(f) } +type VectorMatching struct { + On bool + Include []string +} + type BinOpOptions struct { - ReturnBool bool + ReturnBool bool + VectorMatching *VectorMatching } type BinOpExpr struct { SampleExpr RHS SampleExpr op string - opts BinOpOptions + opts *BinOpOptions } func (e *BinOpExpr) String() string { - if e.opts.ReturnBool { - return fmt.Sprintf("(%s %s bool %s)", e.SampleExpr.String(), e.op, e.RHS.String()) + op := e.op + if e.opts != nil { + if e.opts.ReturnBool { + op = fmt.Sprintf("%s bool", op) + } + if e.opts.VectorMatching != nil { + if e.opts.VectorMatching.On { + op = fmt.Sprintf("%s %s (%s)", op, OpOn, strings.Join(e.opts.VectorMatching.Include, ",")) + } else { + op = fmt.Sprintf("%s %s (%s)", op, OpIgnoring, strings.Join(e.opts.VectorMatching.Include, ",")) + } + } } - return fmt.Sprintf("(%s %s %s)", e.SampleExpr.String(), e.op, e.RHS.String()) + return fmt.Sprintf("(%s %s %s)", e.SampleExpr.String(), op, e.RHS.String()) } // impl SampleExpr @@ -986,7 +1005,7 @@ func (e *BinOpExpr) Walk(f WalkFn) { walkAll(f, e.SampleExpr, e.RHS) } -func mustNewBinOpExpr(op string, opts BinOpOptions, lhs, rhs Expr) SampleExpr { +func mustNewBinOpExpr(op string, opts *BinOpOptions, lhs, rhs Expr) SampleExpr { left, ok := lhs.(SampleExpr) if !ok { panic(logqlmodel.NewParseError(fmt.Sprintf( diff --git a/pkg/logql/engine_test.go b/pkg/logql/engine_test.go index 2482e5bbbd22d..03ae33144f13e 100644 --- a/pkg/logql/engine_test.go +++ b/pkg/logql/engine_test.go @@ -603,6 +603,72 @@ func TestEngine_LogsInstantQuery(t *testing.T) { }, promql.Vector{promql.Sample{Point: promql.Point{T: 60 * 1000, V: 50}, Metric: labels.Labels{labels.Label{Name: "app", Value: "foo"}}}}, }, + { + `sum by (app) (count_over_time({app="foo"}[1m])) + sum by (app) (count_over_time({app="bar"}[1m]))`, + time.Unix(60, 0), + logproto.FORWARD, + 0, + [][]logproto.Series{ + {newSeries(testSize, identity, `{app="foo"}`)}, + {newSeries(testSize, identity, `{app="bar"}`)}, + }, + []SelectSampleParams{ + {&logproto.SampleQueryRequest{Start: time.Unix(0, 0), End: time.Unix(60, 0), Selector: `sum by (app) (count_over_time({app="foo"}[1m]))`}}, + {&logproto.SampleQueryRequest{Start: time.Unix(0, 0), End: time.Unix(60, 0), Selector: `sum by (app) (count_over_time({app="bar"}[1m]))`}}, + }, + promql.Vector{}, + }, + { + `sum by (app) (count_over_time({app="foo"}[1m])) + sum by (app) (count_over_time({app="foo"}[1m]))`, + time.Unix(60, 0), + logproto.FORWARD, + 0, + [][]logproto.Series{ + {newSeries(testSize, identity, `{app="foo"}`)}, + {newSeries(testSize, identity, `{app="foo"}`)}, + }, + []SelectSampleParams{ + {&logproto.SampleQueryRequest{Start: time.Unix(0, 0), End: time.Unix(60, 0), Selector: `sum by (app) (count_over_time({app="foo"}[1m]))`}}, + {&logproto.SampleQueryRequest{Start: time.Unix(0, 0), End: time.Unix(60, 0), Selector: `sum by (app) (count_over_time({app="foo"}[1m]))`}}, + }, + promql.Vector{ + promql.Sample{Point: promql.Point{T: 60 * 1000, V: 120}, Metric: labels.Labels{labels.Label{Name: "app", Value: "foo"}}}, + }, + }, + { + `sum by (app,machine) (count_over_time({app="foo"}[1m])) + on (app) sum by (app) (count_over_time({app="foo"}[1m]))`, + time.Unix(60, 0), + logproto.FORWARD, + 0, + [][]logproto.Series{ + {newSeries(testSize, identity, `{app="foo",machine="fuzz"}`)}, + {newSeries(testSize, identity, `{app="foo"}`)}, + }, + []SelectSampleParams{ + {&logproto.SampleQueryRequest{Start: time.Unix(0, 0), End: time.Unix(60, 0), Selector: `sum by (app,machine) (count_over_time({app="foo"}[1m]))`}}, + {&logproto.SampleQueryRequest{Start: time.Unix(0, 0), End: time.Unix(60, 0), Selector: `sum by (app) (count_over_time({app="foo"}[1m]))`}}, + }, + promql.Vector{ + promql.Sample{Point: promql.Point{T: 60 * 1000, V: 120}, Metric: labels.Labels{labels.Label{Name: "app", Value: "foo"}, labels.Label{Name: "machine", Value: "fuzz"}}}, + }, + }, + { + `sum by (app,machine) (count_over_time({app="foo"}[1m])) > bool ignoring (machine) sum by (app) (count_over_time({app="foo"}[1m]))`, + time.Unix(60, 0), + logproto.FORWARD, + 0, + [][]logproto.Series{ + {newSeries(testSize, identity, `{app="foo",machine="fuzz"}`)}, + {newSeries(testSize, identity, `{app="foo"}`)}, + }, + []SelectSampleParams{ + {&logproto.SampleQueryRequest{Start: time.Unix(0, 0), End: time.Unix(60, 0), Selector: `sum by (app,machine) (count_over_time({app="foo"}[1m]))`}}, + {&logproto.SampleQueryRequest{Start: time.Unix(0, 0), End: time.Unix(60, 0), Selector: `sum by (app) (count_over_time({app="foo"}[1m]))`}}, + }, + promql.Vector{ + promql.Sample{Point: promql.Point{T: 60 * 1000, V: 0}, Metric: labels.Labels{labels.Label{Name: "app", Value: "foo"}, labels.Label{Name: "machine", Value: "fuzz"}}}, + }, + }, } { test := test t.Run(fmt.Sprintf("%s %s", test.qs, test.direction), func(t *testing.T) { diff --git a/pkg/logql/evaluator.go b/pkg/logql/evaluator.go index eaf6cb3038c62..8431e782d62f8 100644 --- a/pkg/logql/evaluator.go +++ b/pkg/logql/evaluator.go @@ -575,8 +575,24 @@ func binOpStepEvaluator( // TODO(owen-d): this seems wildly inefficient: we're calculating // the hash on each sample & step per evaluator. // We seem limited to this approach due to using the StepEvaluator ifc. - hash := sample.Metric.Hash() + + var hash uint64 + if expr.opts == nil || expr.opts.VectorMatching == nil { + hash = sample.Metric.Hash() + } else if expr.opts.VectorMatching.On { + hash = sample.Metric.WithLabels(expr.opts.VectorMatching.Include...).Hash() + } else { + hash = sample.Metric.WithoutLabels(expr.opts.VectorMatching.Include...).Hash() + } pair := pairs[hash] + if pair[i] != nil { + if i == 0 { + failStepEvaluator(lhs, errors.New("multiple matches for labels")) + } else { + failStepEvaluator(rhs, errors.New("multiple matches for labels")) + } + return false, ts, nil + } pair[i] = &promql.Sample{ Metric: sample.Metric, Point: sample.Point, @@ -588,7 +604,11 @@ func binOpStepEvaluator( results := make(promql.Vector, 0, len(pairs)) for _, pair := range pairs { // merge - if merged := mergeBinOp(expr.op, pair[0], pair[1], !expr.opts.ReturnBool, IsComparisonOperator(expr.op)); merged != nil { + filter := true + if expr.opts != nil && expr.opts.ReturnBool { + filter = false + } + if merged := mergeBinOp(expr.op, pair[0], pair[1], filter, IsComparisonOperator(expr.op)); merged != nil { results = append(results, *merged) } } @@ -1017,3 +1037,17 @@ func absentLabels(expr SampleExpr) labels.Labels { } return m } + +// failStepEvaluator marks the step evaluator as failed +func failStepEvaluator(se StepEvaluator, err error) { + switch s := se.(type) { + case *stepEvaluator: + s.err = func() error { + return err + } + case *rangeVectorEvaluator: + s.err = err + case *absentRangeVectorEvaluator: + s.err = err + } +} diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index b9d1c65e2dacc..ab20c72d605d6 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -33,7 +33,8 @@ import ( str string duration time.Duration LiteralExpr *LiteralExpr - BinOpModifier BinOpOptions + BinOpModifier *BinOpOptions + BoolModifier *BinOpOptions LabelParser *LabelParserExpr LineFilters *LineFilterExpr LineFilter *LineFilterExpr @@ -78,6 +79,7 @@ import ( %type literalExpr %type labelReplaceExpr %type binOpModifier +%type boolModifier %type labelParser %type pipelineExpr %type pipelineStage @@ -106,7 +108,7 @@ import ( OPEN_PARENTHESIS CLOSE_PARENTHESIS BY WITHOUT COUNT_OVER_TIME RATE SUM AVG MAX MIN COUNT STDDEV STDVAR BOTTOMK TOPK BYTES_OVER_TIME BYTES_RATE BOOL JSON REGEXP LOGFMT PIPE LINE_FMT LABEL_FMT UNWRAP AVG_OVER_TIME SUM_OVER_TIME MIN_OVER_TIME MAX_OVER_TIME STDVAR_OVER_TIME STDDEV_OVER_TIME QUANTILE_OVER_TIME BYTES_CONV DURATION_CONV DURATION_SECONDS_CONV - FIRST_OVER_TIME LAST_OVER_TIME ABSENT_OVER_TIME LABEL_REPLACE UNPACK OFFSET PATTERN IP + FIRST_OVER_TIME LAST_OVER_TIME ABSENT_OVER_TIME LABEL_REPLACE UNPACK OFFSET PATTERN IP ON IGNORING // Operators are listed with increasing precedence. %left OR @@ -342,7 +344,6 @@ numberFilter: | IDENTIFIER CMP_EQ NUMBER { $$ = log.NewNumericLabelFilter(log.LabelFilterEqual, $1, mustNewFloat($3))} ; -// TODO(owen-d): add (on,ignoring) clauses to binOpExpr // Operator precedence only works if each of these is listed separately. binOpExpr: expr OR binOpModifier expr { $$ = mustNewBinOpExpr("or", $3, $1, $4) } @@ -363,9 +364,33 @@ binOpExpr: ; binOpModifier: - { $$ = BinOpOptions{} } - | BOOL { $$ = BinOpOptions{ ReturnBool: true } } - ; + boolModifier { $$ = $1 } + | boolModifier ON OPEN_PARENTHESIS labels CLOSE_PARENTHESIS + { + $$ = $1 + $$.VectorMatching = &VectorMatching{On: true, Include: $4} + } + | boolModifier ON OPEN_PARENTHESIS CLOSE_PARENTHESIS + { + $$ = $1 + $$.VectorMatching = &VectorMatching{On: true, Include: []string{}} + } + | boolModifier IGNORING OPEN_PARENTHESIS labels CLOSE_PARENTHESIS + { + $$ = $1 + $$.VectorMatching = &VectorMatching{On: false, Include: $4} + } + | boolModifier IGNORING OPEN_PARENTHESIS CLOSE_PARENTHESIS + { + $$ = $1 + $$.VectorMatching = &VectorMatching{On: false, Include: []string{}} + } + ; + +boolModifier: + { $$ = &BinOpOptions{} } + | BOOL { $$ = &BinOpOptions{ ReturnBool: true } } + ; literalExpr: NUMBER { $$ = mustNewLiteralExpr( $1, false ) } diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index 11053f201bf7b..31ce3950c66b9 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -36,7 +36,8 @@ type exprSymType struct { str string duration time.Duration LiteralExpr *LiteralExpr - BinOpModifier BinOpOptions + BinOpModifier *BinOpOptions + BoolModifier *BinOpOptions LabelParser *LabelParserExpr LineFilters *LineFilterExpr LineFilter *LineFilterExpr @@ -121,21 +122,23 @@ const UNPACK = 57404 const OFFSET = 57405 const PATTERN = 57406 const IP = 57407 -const OR = 57408 -const AND = 57409 -const UNLESS = 57410 -const CMP_EQ = 57411 -const NEQ = 57412 -const LT = 57413 -const LTE = 57414 -const GT = 57415 -const GTE = 57416 -const ADD = 57417 -const SUB = 57418 -const MUL = 57419 -const DIV = 57420 -const MOD = 57421 -const POW = 57422 +const ON = 57408 +const IGNORING = 57409 +const OR = 57410 +const AND = 57411 +const UNLESS = 57412 +const CMP_EQ = 57413 +const NEQ = 57414 +const LT = 57415 +const LTE = 57416 +const GT = 57417 +const GTE = 57418 +const ADD = 57419 +const SUB = 57420 +const MUL = 57421 +const DIV = 57422 +const MOD = 57423 +const POW = 57424 var exprToknames = [...]string{ "$end", @@ -203,6 +206,8 @@ var exprToknames = [...]string{ "OFFSET", "PATTERN", "IP", + "ON", + "IGNORING", "OR", "AND", "UNLESS", @@ -234,126 +239,130 @@ var exprExca = [...]int{ const exprPrivate = 57344 -const exprLast = 502 +const exprLast = 539 var exprAct = [...]int{ - 238, 189, 76, 4, 170, 58, 158, 5, 163, 198, - 67, 110, 50, 57, 241, 133, 69, 2, 45, 46, + 243, 192, 76, 4, 173, 58, 161, 5, 166, 201, + 67, 111, 50, 57, 246, 134, 69, 2, 45, 46, 47, 48, 49, 50, 72, 42, 43, 44, 51, 52, 55, 56, 53, 54, 45, 46, 47, 48, 49, 50, 43, 44, 51, 52, 55, 56, 53, 54, 45, 46, - 47, 48, 49, 50, 47, 48, 49, 50, 172, 131, - 132, 246, 298, 98, 243, 244, 120, 102, 65, 298, - 65, 129, 131, 132, 83, 63, 64, 63, 64, 137, - 284, 241, 135, 77, 78, 142, 51, 52, 55, 56, - 53, 54, 45, 46, 47, 48, 49, 50, 191, 143, - 191, 144, 145, 146, 147, 148, 149, 150, 151, 152, - 153, 154, 155, 156, 157, 178, 173, 176, 177, 174, - 175, 167, 188, 65, 66, 122, 66, 65, 117, 130, - 63, 64, 185, 61, 63, 64, 276, 247, 242, 180, - 318, 313, 160, 196, 192, 75, 114, 77, 78, 190, - 244, 201, 193, 191, 268, 65, 218, 191, 182, 219, - 217, 306, 63, 64, 255, 188, 204, 205, 206, 257, - 65, 243, 241, 243, 65, 305, 12, 63, 64, 66, - 65, 63, 64, 66, 136, 191, 185, 63, 64, 236, - 239, 117, 245, 99, 248, 135, 98, 251, 102, 252, - 191, 200, 240, 237, 60, 160, 249, 117, 250, 114, - 295, 66, 185, 117, 276, 261, 259, 216, 303, 255, - 202, 160, 286, 117, 256, 114, 66, 160, 200, 301, - 66, 114, 209, 267, 186, 214, 66, 181, 215, 213, - 269, 114, 271, 273, 253, 275, 98, 199, 117, 243, - 274, 285, 270, 159, 277, 98, 194, 124, 287, 105, - 107, 106, 134, 115, 116, 246, 114, 266, 161, 159, - 12, 292, 293, 242, 161, 159, 98, 294, 136, 265, - 108, 123, 109, 296, 297, 179, 141, 128, 283, 302, - 140, 139, 81, 74, 15, 316, 212, 312, 308, 289, - 309, 310, 12, 254, 279, 280, 281, 210, 243, 207, - 6, 203, 314, 126, 19, 20, 33, 34, 36, 37, - 35, 38, 39, 40, 41, 21, 22, 125, 195, 187, - 127, 211, 208, 311, 300, 23, 24, 25, 26, 27, - 28, 29, 299, 282, 197, 30, 31, 32, 18, 272, - 233, 80, 12, 234, 232, 230, 79, 307, 231, 229, - 6, 317, 16, 17, 19, 20, 33, 34, 36, 37, - 35, 38, 39, 40, 41, 21, 22, 227, 315, 224, - 228, 226, 225, 223, 304, 23, 24, 25, 26, 27, - 28, 29, 3, 117, 138, 30, 31, 32, 18, 68, - 221, 291, 12, 222, 220, 263, 264, 290, 260, 258, - 6, 114, 16, 17, 19, 20, 33, 34, 36, 37, - 35, 38, 39, 40, 41, 21, 22, 82, 235, 105, - 107, 106, 184, 115, 116, 23, 24, 25, 26, 27, - 28, 29, 183, 182, 181, 30, 31, 32, 18, 168, - 108, 262, 109, 166, 171, 71, 165, 288, 73, 164, - 73, 171, 16, 17, 111, 112, 162, 101, 169, 104, - 103, 84, 85, 86, 87, 88, 89, 90, 91, 92, - 93, 94, 95, 96, 97, 59, 118, 113, 119, 100, - 11, 10, 9, 121, 14, 8, 278, 13, 7, 70, - 62, 1, + 47, 48, 49, 50, 47, 48, 49, 50, 130, 132, + 133, 65, 309, 99, 175, 132, 133, 103, 63, 64, + 223, 251, 185, 224, 222, 145, 146, 248, 61, 138, + 309, 246, 136, 121, 84, 143, 51, 52, 55, 56, + 53, 54, 45, 46, 47, 48, 49, 50, 75, 144, + 77, 78, 147, 148, 149, 150, 151, 152, 153, 154, + 155, 156, 157, 158, 159, 160, 77, 78, 131, 66, + 285, 118, 170, 181, 176, 179, 180, 177, 178, 65, + 219, 221, 184, 220, 218, 163, 63, 64, 100, 115, + 183, 285, 123, 260, 199, 195, 203, 247, 300, 249, + 193, 247, 204, 196, 65, 248, 312, 188, 260, 194, + 329, 63, 64, 299, 293, 267, 292, 260, 191, 209, + 210, 211, 262, 65, 188, 203, 248, 324, 246, 277, + 63, 64, 248, 252, 194, 162, 248, 66, 203, 317, + 118, 217, 241, 244, 265, 250, 255, 253, 136, 99, + 256, 103, 257, 194, 163, 245, 242, 205, 115, 254, + 260, 249, 66, 188, 191, 261, 65, 264, 266, 65, + 270, 268, 203, 63, 64, 316, 63, 64, 314, 65, + 295, 66, 65, 276, 258, 189, 63, 64, 197, 63, + 64, 202, 306, 118, 286, 278, 194, 280, 282, 194, + 284, 99, 118, 164, 162, 283, 294, 279, 118, 194, + 99, 115, 60, 296, 125, 12, 163, 135, 124, 275, + 115, 214, 163, 137, 66, 12, 115, 66, 274, 208, + 303, 304, 207, 137, 182, 99, 305, 66, 142, 141, + 66, 140, 307, 308, 288, 289, 290, 81, 313, 74, + 327, 323, 298, 127, 259, 15, 215, 212, 206, 319, + 198, 320, 321, 12, 190, 164, 162, 126, 129, 216, + 128, 6, 213, 325, 322, 19, 20, 33, 34, 36, + 37, 35, 38, 39, 40, 41, 21, 22, 238, 311, + 235, 239, 237, 236, 234, 281, 23, 24, 25, 26, + 27, 28, 29, 310, 291, 80, 30, 31, 32, 18, + 232, 200, 229, 233, 231, 230, 228, 226, 79, 12, + 227, 225, 272, 273, 328, 16, 17, 6, 326, 315, + 302, 19, 20, 33, 34, 36, 37, 35, 38, 39, + 40, 41, 21, 22, 3, 301, 271, 269, 263, 174, + 318, 68, 23, 24, 25, 26, 27, 28, 29, 240, + 187, 186, 30, 31, 32, 18, 297, 139, 185, 184, + 171, 169, 168, 71, 167, 12, 73, 73, 174, 112, + 113, 16, 17, 6, 165, 102, 118, 19, 20, 33, + 34, 36, 37, 35, 38, 39, 40, 41, 21, 22, + 172, 105, 104, 59, 115, 119, 114, 120, 23, 24, + 25, 26, 27, 28, 29, 118, 101, 83, 30, 31, + 32, 18, 106, 108, 107, 11, 116, 117, 251, 10, + 9, 82, 122, 115, 14, 8, 287, 16, 17, 13, + 7, 70, 62, 109, 1, 110, 0, 0, 0, 0, + 0, 106, 108, 107, 0, 116, 117, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 109, 0, 110, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, } var exprPact = [...]int{ - 287, -1000, -41, -1000, -1000, 160, 287, -1000, -1000, -1000, - -1000, -1000, 453, 270, 122, -1000, 349, 344, 269, -1000, + 298, -1000, -43, -1000, -1000, 218, 298, -1000, -1000, -1000, + -1000, -1000, 421, 276, 75, -1000, 361, 348, 274, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 160, -1000, 166, - 388, -1000, 60, -1000, -1000, -1000, -1000, 257, 233, -41, - 311, 271, -1000, 59, 255, 387, 268, 267, 263, -1000, - -1000, 287, 287, -1000, 287, 287, 287, 287, 287, 287, - 287, 287, 287, 287, 287, 287, 287, 287, -1000, -1000, - -1000, -1000, 202, -1000, -1000, 454, -1000, 450, -1000, 447, - -1000, -1000, -1000, -1000, 243, 443, 456, 46, -1000, -1000, - -1000, 262, -1000, -1000, -1000, -1000, -1000, 455, -1000, 438, - 437, 436, 426, 210, 310, 156, 161, 232, 309, 337, - 223, 196, 292, -27, 17, 17, -23, -23, -68, -68, - -68, -68, -57, -57, -57, -57, -57, -57, 202, 243, - 243, 243, 290, -1000, 320, -1000, -1000, 208, -1000, 288, - -1000, 319, 231, 152, 396, 375, 373, 351, 346, 422, - -1000, -1000, -1000, -1000, -1000, -1000, 58, 161, 109, 129, - 141, 218, 113, 184, 58, 287, 220, 284, 200, -1000, - -1000, 145, -1000, 403, 123, 202, 186, 454, 402, -1000, - 449, 400, 256, -1000, -1000, -1000, 244, -1000, -1000, -1000, + -1000, -1000, 44, 44, 44, 44, 44, 44, 44, 44, + 44, 44, 44, 44, 44, 44, 44, 218, -1000, 47, + 460, -1000, 77, -1000, -1000, -1000, -1000, 244, 240, -43, + 301, 302, -1000, 46, 260, 410, 268, 266, 265, -1000, + -1000, 298, 298, 9, -1000, 298, 298, 298, 298, 298, + 298, 298, 298, 298, 298, 298, 298, 298, 298, -1000, + -1000, -1000, -1000, 185, -1000, -1000, 419, -1000, 416, -1000, + 415, -1000, -1000, -1000, -1000, 238, 414, 423, 52, -1000, + -1000, -1000, 261, -1000, -1000, -1000, -1000, -1000, 422, -1000, + 413, 412, 405, 404, 211, 295, 205, 250, 214, 291, + 354, 217, 183, 289, -29, 259, 256, 15, 15, -25, + -25, -70, -70, -70, -70, -59, -59, -59, -59, -59, + -59, 185, 238, 238, 238, 288, -1000, 310, -1000, -1000, + 247, -1000, 287, -1000, 307, 126, 66, 363, 358, 356, + 336, 334, 403, -1000, -1000, -1000, -1000, -1000, -1000, 91, + 250, 115, 138, 202, 431, 159, 172, 91, 298, 210, + 285, 191, -1000, -1000, 148, -1000, 392, 170, 141, 253, + 185, 116, 419, 391, -1000, 394, 367, 255, -1000, -1000, + -1000, 246, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, 209, -1000, 130, 54, 20, - 54, 341, -49, 243, -49, 127, 249, 334, 264, 56, - -1000, -1000, 198, -1000, 287, 452, -1000, -1000, 280, -1000, - -1000, -1000, -1000, -1000, -1000, 401, 395, -1000, 58, 20, - 54, 20, -1000, -1000, 202, -1000, -49, -1000, 187, -1000, - -1000, -1000, 18, 333, 325, 205, 58, 194, -1000, 378, - 151, 137, -1000, 20, -1000, 352, 25, 20, 14, -49, - -49, 324, -1000, -1000, 278, -1000, -1000, 117, 20, -1000, - -1000, -49, 372, -1000, -1000, 276, 355, 116, -1000, + 209, -1000, 155, 215, 33, 215, 337, -49, 238, -49, + 111, 239, 345, 142, 140, -1000, -1000, 206, -1000, 298, + 411, -1000, -1000, 283, 139, -1000, 124, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, 389, 374, -1000, 91, 33, 215, + 33, -1000, -1000, 185, -1000, -49, -1000, 219, -1000, -1000, + -1000, 18, 344, 330, 132, 91, 204, -1000, 373, -1000, + -1000, 201, 165, -1000, 33, -1000, 395, 36, 33, 24, + -49, -49, 315, -1000, -1000, 282, -1000, -1000, 153, 33, + -1000, -1000, -49, 372, -1000, -1000, 281, 368, 136, -1000, } var exprPgo = [...]int{ - 0, 501, 16, 500, 2, 9, 392, 3, 15, 11, - 499, 498, 497, 496, 7, 495, 494, 493, 492, 491, - 490, 427, 489, 13, 5, 488, 487, 486, 6, 485, - 133, 470, 469, 4, 468, 467, 8, 466, 1, 465, - 464, 0, + 0, 494, 16, 492, 2, 9, 394, 3, 15, 11, + 491, 490, 489, 486, 7, 485, 484, 482, 480, 479, + 475, 481, 467, 466, 13, 5, 457, 456, 455, 6, + 453, 78, 452, 451, 4, 450, 435, 8, 434, 1, + 430, 429, 0, } var exprR1 = [...]int{ 0, 1, 2, 2, 7, 7, 7, 7, 7, 7, 6, 6, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 8, 8, 38, - 38, 38, 13, 13, 13, 11, 11, 11, 11, 15, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 39, + 39, 39, 13, 13, 13, 11, 11, 11, 11, 15, 15, 15, 15, 15, 15, 20, 3, 3, 3, 3, - 14, 14, 14, 10, 10, 9, 9, 9, 9, 23, - 23, 24, 24, 24, 24, 24, 24, 17, 30, 30, - 29, 29, 22, 22, 22, 22, 22, 35, 31, 33, - 33, 34, 34, 34, 32, 28, 28, 28, 28, 28, - 28, 28, 28, 28, 36, 37, 37, 40, 40, 39, - 39, 27, 27, 27, 27, 27, 27, 27, 25, 25, - 25, 25, 25, 25, 25, 26, 26, 26, 26, 26, - 26, 26, 18, 18, 18, 18, 18, 18, 18, 18, - 18, 18, 18, 18, 18, 18, 18, 21, 21, 19, - 19, 19, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 12, 12, 12, 12, 12, 12, 12, 12, 12, - 12, 12, 12, 12, 12, 41, 5, 5, 4, 4, - 4, 4, + 14, 14, 14, 10, 10, 9, 9, 9, 9, 24, + 24, 25, 25, 25, 25, 25, 25, 17, 31, 31, + 30, 30, 23, 23, 23, 23, 23, 36, 32, 34, + 34, 35, 35, 35, 33, 29, 29, 29, 29, 29, + 29, 29, 29, 29, 37, 38, 38, 41, 41, 40, + 40, 28, 28, 28, 28, 28, 28, 28, 26, 26, + 26, 26, 26, 26, 26, 27, 27, 27, 27, 27, + 27, 27, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 21, 21, 21, + 21, 21, 22, 22, 19, 19, 19, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 12, 12, 12, 12, + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + 42, 5, 5, 4, 4, 4, 4, } var exprR2 = [...]int{ @@ -371,81 +380,83 @@ var exprR2 = [...]int{ 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 0, 1, 1, - 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, + 4, 4, 4, 4, 4, 4, 4, 1, 5, 4, + 5, 4, 0, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 2, 1, 3, 4, 4, - 3, 3, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 1, 3, 4, 4, 3, 3, } var exprChk = [...]int{ -1000, -1, -2, -6, -7, -14, 23, -11, -15, -18, - -19, -20, 15, -12, -16, 7, 75, 76, 61, 27, + -19, -20, 15, -12, -16, 7, 77, 78, 61, 27, 28, 38, 39, 48, 49, 50, 51, 52, 53, 54, 58, 59, 60, 29, 30, 33, 31, 32, 34, 35, - 36, 37, 66, 67, 68, 75, 76, 77, 78, 79, - 80, 69, 70, 73, 74, 71, 72, -23, -24, -29, - 44, -30, -3, 21, 22, 14, 70, -7, -6, -2, + 36, 37, 68, 69, 70, 77, 78, 79, 80, 81, + 82, 71, 72, 75, 76, 73, 74, -24, -25, -30, + 44, -31, -3, 21, 22, 14, 72, -7, -6, -2, -10, 2, -9, 5, 23, 23, -4, 25, 26, 7, - 7, 23, -21, 40, -21, -21, -21, -21, -21, -21, - -21, -21, -21, -21, -21, -21, -21, -21, -24, -30, - -22, -35, -28, -31, -32, 41, 43, 42, 62, 64, - -9, -40, -39, -26, 23, 45, 46, 5, -27, -25, - 6, -17, 65, 24, 24, 16, 2, 19, 16, 12, - 70, 13, 14, -8, 7, -14, 23, -7, 7, 23, - 23, 23, -7, -2, -2, -2, -2, -2, -2, -2, - -2, -2, -2, -2, -2, -2, -2, -2, -28, 67, - 19, 66, -37, -36, 5, 6, 6, -28, 6, -34, - -33, 5, 12, 70, 73, 74, 71, 72, 69, 23, - -9, 6, 6, 6, 6, 2, 24, 19, 9, -38, - -23, 44, -14, -8, 24, 19, -7, 7, -5, 24, - 5, -5, 24, 19, -28, -28, -28, 19, 12, 24, - 19, 12, 65, 8, 4, 7, 65, 8, 4, 7, - 8, 4, 7, 8, 4, 7, 8, 4, 7, 8, - 4, 7, 8, 4, 7, 6, -4, -8, -41, -38, - -23, 63, 9, 44, 9, -38, 47, 24, -38, -23, - 24, -4, -7, 24, 19, 19, 24, 24, 6, -36, - 6, -33, 2, 5, 6, 23, 23, 24, 24, -38, - -23, -38, 8, -41, -28, -41, 9, 5, -13, 55, - 56, 57, 9, 24, 24, -38, 24, -7, 5, 19, - 6, 6, -4, -38, -41, 23, -41, -38, 44, 9, - 9, 24, -4, 24, 6, 24, 24, 5, -38, -41, - -41, 9, 19, 24, -41, 6, 19, 6, 24, + 7, 23, -21, -22, 40, -21, -21, -21, -21, -21, + -21, -21, -21, -21, -21, -21, -21, -21, -21, -25, + -31, -23, -36, -29, -32, -33, 41, 43, 42, 62, + 64, -9, -41, -40, -27, 23, 45, 46, 5, -28, + -26, 6, -17, 65, 24, 24, 16, 2, 19, 16, + 12, 72, 13, 14, -8, 7, -14, 23, -7, 7, + 23, 23, 23, -7, -2, 66, 67, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -29, 69, 19, 68, -38, -37, 5, 6, 6, + -29, 6, -35, -34, 5, 12, 72, 75, 76, 73, + 74, 71, 23, -9, 6, 6, 6, 6, 2, 24, + 19, 9, -39, -24, 44, -14, -8, 24, 19, -7, + 7, -5, 24, 5, -5, 24, 19, 23, 23, -29, + -29, -29, 19, 12, 24, 19, 12, 65, 8, 4, + 7, 65, 8, 4, 7, 8, 4, 7, 8, 4, + 7, 8, 4, 7, 8, 4, 7, 8, 4, 7, + 6, -4, -8, -42, -39, -24, 63, 9, 44, 9, + -39, 47, 24, -39, -24, 24, -4, -7, 24, 19, + 19, 24, 24, 6, -5, 24, -5, 24, -37, 6, + -34, 2, 5, 6, 23, 23, 24, 24, -39, -24, + -39, 8, -42, -29, -42, 9, 5, -13, 55, 56, + 57, 9, 24, 24, -39, 24, -7, 5, 19, 24, + 24, 6, 6, -4, -39, -42, 23, -42, -39, 44, + 9, 9, 24, -4, 24, 6, 24, 24, 5, -39, + -42, -42, 9, 19, 24, -42, 6, 19, 6, 24, } var exprDef = [...]int{ 0, -2, 1, 2, 3, 10, 0, 4, 5, 6, - 7, 8, 0, 0, 0, 149, 0, 0, 0, 161, - 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, - 172, 173, 174, 152, 153, 154, 155, 156, 157, 158, - 159, 160, 147, 147, 147, 147, 147, 147, 147, 147, - 147, 147, 147, 147, 147, 147, 147, 11, 69, 71, + 7, 8, 0, 0, 0, 154, 0, 0, 0, 166, + 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, + 177, 178, 179, 157, 158, 159, 160, 161, 162, 163, + 164, 165, 152, 152, 152, 152, 152, 152, 152, 152, + 152, 152, 152, 152, 152, 152, 152, 11, 69, 71, 0, 80, 0, 56, 57, 58, 59, 3, 2, 0, - 0, 0, 63, 0, 0, 0, 0, 0, 0, 150, - 151, 0, 0, 148, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 70, 81, - 72, 73, 74, 75, 76, 82, 83, 0, 85, 0, - 95, 96, 97, 98, 0, 0, 0, 0, 109, 110, - 78, 0, 77, 9, 12, 60, 61, 0, 62, 0, - 0, 0, 0, 0, 0, 0, 0, 3, 149, 0, - 0, 0, 3, 132, 133, 134, 135, 136, 137, 138, - 139, 140, 141, 142, 143, 144, 145, 146, 100, 0, - 0, 0, 87, 105, 0, 84, 86, 0, 88, 94, - 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 64, 65, 66, 67, 68, 38, 45, 0, 13, 0, - 0, 0, 0, 0, 49, 0, 3, 149, 0, 180, - 176, 0, 181, 0, 101, 102, 103, 0, 0, 99, - 0, 0, 0, 116, 123, 130, 0, 115, 122, 129, - 111, 118, 125, 112, 119, 126, 113, 120, 127, 114, - 121, 128, 117, 124, 131, 0, 47, 0, 14, 17, - 33, 0, 21, 0, 25, 0, 0, 0, 0, 0, - 37, 51, 3, 50, 0, 0, 178, 179, 0, 106, - 104, 92, 93, 89, 90, 0, 0, 79, 46, 18, - 34, 35, 175, 22, 41, 26, 29, 39, 0, 42, - 43, 44, 15, 0, 0, 0, 52, 3, 177, 0, - 0, 0, 48, 36, 30, 0, 16, 19, 0, 23, - 27, 0, 53, 54, 0, 107, 108, 0, 20, 24, - 28, 31, 0, 40, 32, 0, 0, 0, 55, + 0, 0, 63, 0, 0, 0, 0, 0, 0, 155, + 156, 0, 0, 147, 153, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, + 81, 72, 73, 74, 75, 76, 82, 83, 0, 85, + 0, 95, 96, 97, 98, 0, 0, 0, 0, 109, + 110, 78, 0, 77, 9, 12, 60, 61, 0, 62, + 0, 0, 0, 0, 0, 0, 0, 0, 3, 154, + 0, 0, 0, 3, 132, 0, 0, 133, 134, 135, + 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, + 146, 100, 0, 0, 0, 87, 105, 0, 84, 86, + 0, 88, 94, 91, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 64, 65, 66, 67, 68, 38, 45, + 0, 13, 0, 0, 0, 0, 0, 49, 0, 3, + 154, 0, 185, 181, 0, 186, 0, 0, 0, 101, + 102, 103, 0, 0, 99, 0, 0, 0, 116, 123, + 130, 0, 115, 122, 129, 111, 118, 125, 112, 119, + 126, 113, 120, 127, 114, 121, 128, 117, 124, 131, + 0, 47, 0, 14, 17, 33, 0, 21, 0, 25, + 0, 0, 0, 0, 0, 37, 51, 3, 50, 0, + 0, 183, 184, 0, 0, 149, 0, 151, 106, 104, + 92, 93, 89, 90, 0, 0, 79, 46, 18, 34, + 35, 180, 22, 41, 26, 29, 39, 0, 42, 43, + 44, 15, 0, 0, 0, 52, 3, 182, 0, 148, + 150, 0, 0, 48, 36, 30, 0, 16, 19, 0, + 23, 27, 0, 53, 54, 0, 107, 108, 0, 20, + 24, 28, 31, 0, 40, 32, 0, 0, 0, 55, } var exprTok1 = [...]int{ @@ -460,7 +471,8 @@ var exprTok2 = [...]int{ 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, - 72, 73, 74, 75, 76, 77, 78, 79, 80, + 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, + 82, } var exprTok3 = [...]int{ 0, @@ -1522,176 +1534,205 @@ exprdefault: exprVAL.BinOpExpr = mustNewBinOpExpr("<=", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } case 147: - exprDollar = exprS[exprpt-0 : exprpt+1] + exprDollar = exprS[exprpt-1 : exprpt+1] { - exprVAL.BinOpModifier = BinOpOptions{} + exprVAL.BinOpModifier = exprDollar[1].BoolModifier } case 148: - exprDollar = exprS[exprpt-1 : exprpt+1] + exprDollar = exprS[exprpt-5 : exprpt+1] { - exprVAL.BinOpModifier = BinOpOptions{ReturnBool: true} + exprVAL.BinOpModifier = exprDollar[1].BoolModifier + exprVAL.BinOpModifier.VectorMatching = &VectorMatching{On: true, Include: exprDollar[4].Labels} } case 149: + exprDollar = exprS[exprpt-4 : exprpt+1] + { + exprVAL.BinOpModifier = exprDollar[1].BoolModifier + exprVAL.BinOpModifier.VectorMatching = &VectorMatching{On: true, Include: []string{}} + } + case 150: + exprDollar = exprS[exprpt-5 : exprpt+1] + { + exprVAL.BinOpModifier = exprDollar[1].BoolModifier + exprVAL.BinOpModifier.VectorMatching = &VectorMatching{On: false, Include: exprDollar[4].Labels} + } + case 151: + exprDollar = exprS[exprpt-4 : exprpt+1] + { + exprVAL.BinOpModifier = exprDollar[1].BoolModifier + exprVAL.BinOpModifier.VectorMatching = &VectorMatching{On: false, Include: []string{}} + } + case 152: + exprDollar = exprS[exprpt-0 : exprpt+1] + { + exprVAL.BoolModifier = &BinOpOptions{} + } + case 153: + exprDollar = exprS[exprpt-1 : exprpt+1] + { + exprVAL.BoolModifier = &BinOpOptions{ReturnBool: true} + } + case 154: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[1].str, false) } - case 150: + case 155: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[2].str, false) } - case 151: + case 156: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[2].str, true) } - case 152: + case 157: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeSum } - case 153: + case 158: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeAvg } - case 154: + case 159: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeCount } - case 155: + case 160: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMax } - case 156: + case 161: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMin } - case 157: + case 162: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStddev } - case 158: + case 163: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStdvar } - case 159: + case 164: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeBottomK } - case 160: + case 165: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeTopK } - case 161: + case 166: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeCount } - case 162: + case 167: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeRate } - case 163: + case 168: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeBytes } - case 164: + case 169: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeBytesRate } - case 165: + case 170: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeAvg } - case 166: + case 171: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeSum } - case 167: + case 172: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeMin } - case 168: + case 173: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeMax } - case 169: + case 174: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeStdvar } - case 170: + case 175: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeStddev } - case 171: + case 176: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeQuantile } - case 172: + case 177: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeFirst } - case 173: + case 178: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeLast } - case 174: + case 179: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeAbsent } - case 175: + case 180: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.OffsetExpr = newOffsetExpr(exprDollar[2].duration) } - case 176: + case 181: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Labels = []string{exprDollar[1].str} } - case 177: + case 182: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Labels = append(exprDollar[1].Labels, exprDollar[3].str) } - case 178: + case 183: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: false, groups: exprDollar[3].Labels} } - case 179: + case 184: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: true, groups: exprDollar[3].Labels} } - case 180: + case 185: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Grouping = &grouping{without: false, groups: nil} } - case 181: + case 186: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Grouping = &grouping{without: true, groups: nil} diff --git a/pkg/logql/lex.go b/pkg/logql/lex.go index aa4f5a0d1eea2..a23dd1cd6f0b7 100644 --- a/pkg/logql/lex.go +++ b/pkg/logql/lex.go @@ -36,6 +36,8 @@ var tokens = map[string]int{ "]": CLOSE_BRACKET, OpLabelReplace: LABEL_REPLACE, OpOffset: OFFSET, + OpOn: ON, + OpIgnoring: IGNORING, // binops OpTypeOr: OR, diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index 57a5412d46f19..9a009fd6db914 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -938,10 +938,10 @@ func TestParse(t *testing.T) { `, exp: mustNewBinOpExpr( OpTypeDiv, - BinOpOptions{}, + &BinOpOptions{}, mustNewBinOpExpr( OpTypeDiv, - BinOpOptions{}, + &BinOpOptions{}, mustNewVectorAggregationExpr(newRangeAggregationExpr( &LogRange{ left: &MatchersExpr{ @@ -1001,10 +1001,10 @@ func TestParse(t *testing.T) { `, exp: mustNewBinOpExpr( OpTypeDiv, - BinOpOptions{}, + &BinOpOptions{}, mustNewBinOpExpr( OpTypePow, - BinOpOptions{}, + &BinOpOptions{}, mustNewVectorAggregationExpr(newRangeAggregationExpr( &LogRange{ left: &MatchersExpr{ @@ -1065,7 +1065,7 @@ func TestParse(t *testing.T) { `, exp: mustNewBinOpExpr( OpTypeAdd, - BinOpOptions{}, + &BinOpOptions{}, mustNewVectorAggregationExpr(newRangeAggregationExpr( &LogRange{ left: &MatchersExpr{ @@ -1084,7 +1084,7 @@ func TestParse(t *testing.T) { ), mustNewBinOpExpr( OpTypeDiv, - BinOpOptions{}, + &BinOpOptions{}, mustNewVectorAggregationExpr(newRangeAggregationExpr( &LogRange{ left: &MatchersExpr{ @@ -1128,7 +1128,7 @@ func TestParse(t *testing.T) { )`, exp: mustNewVectorAggregationExpr( mustNewBinOpExpr(OpTypeDiv, - BinOpOptions{}, + &BinOpOptions{}, newRangeAggregationExpr( &LogRange{ left: newPipelineExpr( @@ -1156,9 +1156,9 @@ func TestParse(t *testing.T) { / count_over_time({namespace="tns"}[5m]) ) * 100`, - exp: mustNewBinOpExpr(OpTypeMul, BinOpOptions{}, mustNewVectorAggregationExpr( + exp: mustNewBinOpExpr(OpTypeMul, &BinOpOptions{}, mustNewVectorAggregationExpr( mustNewBinOpExpr(OpTypeDiv, - BinOpOptions{}, + &BinOpOptions{}, newRangeAggregationExpr( &LogRange{ left: newPipelineExpr( @@ -1187,7 +1187,7 @@ func TestParse(t *testing.T) { in: `sum(count_over_time({foo="bar"}[5m])) by (foo) + 1 / 2`, exp: mustNewBinOpExpr( OpTypeAdd, - BinOpOptions{}, + &BinOpOptions{}, mustNewVectorAggregationExpr( newRangeAggregationExpr( &LogRange{ @@ -1213,9 +1213,9 @@ func TestParse(t *testing.T) { in: `1 + -2 / 1`, exp: mustNewBinOpExpr( OpTypeAdd, - BinOpOptions{}, + &BinOpOptions{}, &LiteralExpr{value: 1}, - mustNewBinOpExpr(OpTypeDiv, BinOpOptions{}, &LiteralExpr{value: -2}, &LiteralExpr{value: 1}), + mustNewBinOpExpr(OpTypeDiv, &BinOpOptions{}, &LiteralExpr{value: -2}, &LiteralExpr{value: 1}), ), }, { @@ -1223,8 +1223,8 @@ func TestParse(t *testing.T) { in: `1 + 1 - -1`, exp: mustNewBinOpExpr( OpTypeSub, - BinOpOptions{}, - mustNewBinOpExpr(OpTypeAdd, BinOpOptions{}, &LiteralExpr{value: 1}, &LiteralExpr{value: 1}), + &BinOpOptions{}, + mustNewBinOpExpr(OpTypeAdd, &BinOpOptions{}, &LiteralExpr{value: 1}, &LiteralExpr{value: 1}), &LiteralExpr{value: -1}, ), }, @@ -2109,7 +2109,7 @@ func TestParse(t *testing.T) { ) by (namespace,instance) ) by (foo,bar) `, - exp: mustNewBinOpExpr(OpTypeAdd, BinOpOptions{ReturnBool: false}, + exp: mustNewBinOpExpr(OpTypeAdd, &BinOpOptions{ReturnBool: false}, mustNewVectorAggregationExpr( newRangeAggregationExpr( newLogRange(&PipelineExpr{ @@ -2176,6 +2176,168 @@ func TestParse(t *testing.T) { ), ), }, + { + in: ` + sum by (foo,bar) ( + quantile_over_time(0.99998,{app="foo"} |= "bar" | json | latency >= 250ms or ( status_code < 500 and status_code > 200) + | line_format "blip{{ .foo }}blop {{.status_code}}" | label_format foo=bar,status_code="buzz{{.bar}}" | unwrap foo [5m] + ) by (namespace,instance) + ) + + ignoring (bar) + avg( + avg_over_time({app="foo"} |= "bar" | json | latency >= 250ms or ( status_code < 500 and status_code > 200) + | line_format "blip{{ .foo }}blop {{.status_code}}" | label_format foo=bar,status_code="buzz{{.bar}}" | unwrap foo [5m] + ) by (namespace,instance) + ) by (foo) + `, + exp: mustNewBinOpExpr(OpTypeAdd, &BinOpOptions{ReturnBool: false, VectorMatching: &VectorMatching{On: false, Include: []string{"bar"}}}, + mustNewVectorAggregationExpr( + newRangeAggregationExpr( + newLogRange(&PipelineExpr{ + left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), + pipeline: MultiStageExpr{ + newLineFilterExpr(labels.MatchEqual, "", "bar"), + newLabelParserExpr(OpParserTypeJSON, ""), + &LabelFilterExpr{ + LabelFilterer: log.NewOrLabelFilter( + log.NewDurationLabelFilter(log.LabelFilterGreaterThanOrEqual, "latency", 250*time.Millisecond), + log.NewAndLabelFilter( + log.NewNumericLabelFilter(log.LabelFilterLesserThan, "status_code", 500.0), + log.NewNumericLabelFilter(log.LabelFilterGreaterThan, "status_code", 200.0), + ), + ), + }, + newLineFmtExpr("blip{{ .foo }}blop {{.status_code}}"), + newLabelFmtExpr([]log.LabelFmt{ + log.NewRenameLabelFmt("foo", "bar"), + log.NewTemplateLabelFmt("status_code", "buzz{{.bar}}"), + }), + }, + }, + 5*time.Minute, + newUnwrapExpr("foo", ""), + nil), + OpRangeTypeQuantile, &grouping{without: false, groups: []string{"namespace", "instance"}}, NewStringLabelFilter("0.99998"), + ), + OpTypeSum, + &grouping{groups: []string{"foo", "bar"}}, + nil, + ), + mustNewVectorAggregationExpr( + newRangeAggregationExpr( + newLogRange(&PipelineExpr{ + left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), + pipeline: MultiStageExpr{ + newLineFilterExpr(labels.MatchEqual, "", "bar"), + newLabelParserExpr(OpParserTypeJSON, ""), + &LabelFilterExpr{ + LabelFilterer: log.NewOrLabelFilter( + log.NewDurationLabelFilter(log.LabelFilterGreaterThanOrEqual, "latency", 250*time.Millisecond), + log.NewAndLabelFilter( + log.NewNumericLabelFilter(log.LabelFilterLesserThan, "status_code", 500.0), + log.NewNumericLabelFilter(log.LabelFilterGreaterThan, "status_code", 200.0), + ), + ), + }, + newLineFmtExpr("blip{{ .foo }}blop {{.status_code}}"), + newLabelFmtExpr([]log.LabelFmt{ + log.NewRenameLabelFmt("foo", "bar"), + log.NewTemplateLabelFmt("status_code", "buzz{{.bar}}"), + }), + }, + }, + 5*time.Minute, + newUnwrapExpr("foo", ""), + nil), + OpRangeTypeAvg, &grouping{without: false, groups: []string{"namespace", "instance"}}, nil, + ), + OpTypeAvg, + &grouping{groups: []string{"foo"}}, + nil, + ), + ), + }, + { + in: ` + sum by (foo,bar) ( + quantile_over_time(0.99998,{app="foo"} |= "bar" | json | latency >= 250ms or ( status_code < 500 and status_code > 200) + | line_format "blip{{ .foo }}blop {{.status_code}}" | label_format foo=bar,status_code="buzz{{.bar}}" | unwrap foo [5m] + ) by (namespace,instance) + ) + + on (foo) + avg( + avg_over_time({app="foo"} |= "bar" | json | latency >= 250ms or ( status_code < 500 and status_code > 200) + | line_format "blip{{ .foo }}blop {{.status_code}}" | label_format foo=bar,status_code="buzz{{.bar}}" | unwrap foo [5m] + ) by (namespace,instance) + ) by (foo) + `, + exp: mustNewBinOpExpr(OpTypeAdd, &BinOpOptions{ReturnBool: false, VectorMatching: &VectorMatching{On: true, Include: []string{"foo"}}}, + mustNewVectorAggregationExpr( + newRangeAggregationExpr( + newLogRange(&PipelineExpr{ + left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), + pipeline: MultiStageExpr{ + newLineFilterExpr(labels.MatchEqual, "", "bar"), + newLabelParserExpr(OpParserTypeJSON, ""), + &LabelFilterExpr{ + LabelFilterer: log.NewOrLabelFilter( + log.NewDurationLabelFilter(log.LabelFilterGreaterThanOrEqual, "latency", 250*time.Millisecond), + log.NewAndLabelFilter( + log.NewNumericLabelFilter(log.LabelFilterLesserThan, "status_code", 500.0), + log.NewNumericLabelFilter(log.LabelFilterGreaterThan, "status_code", 200.0), + ), + ), + }, + newLineFmtExpr("blip{{ .foo }}blop {{.status_code}}"), + newLabelFmtExpr([]log.LabelFmt{ + log.NewRenameLabelFmt("foo", "bar"), + log.NewTemplateLabelFmt("status_code", "buzz{{.bar}}"), + }), + }, + }, + 5*time.Minute, + newUnwrapExpr("foo", ""), + nil), + OpRangeTypeQuantile, &grouping{without: false, groups: []string{"namespace", "instance"}}, NewStringLabelFilter("0.99998"), + ), + OpTypeSum, + &grouping{groups: []string{"foo", "bar"}}, + nil, + ), + mustNewVectorAggregationExpr( + newRangeAggregationExpr( + newLogRange(&PipelineExpr{ + left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), + pipeline: MultiStageExpr{ + newLineFilterExpr(labels.MatchEqual, "", "bar"), + newLabelParserExpr(OpParserTypeJSON, ""), + &LabelFilterExpr{ + LabelFilterer: log.NewOrLabelFilter( + log.NewDurationLabelFilter(log.LabelFilterGreaterThanOrEqual, "latency", 250*time.Millisecond), + log.NewAndLabelFilter( + log.NewNumericLabelFilter(log.LabelFilterLesserThan, "status_code", 500.0), + log.NewNumericLabelFilter(log.LabelFilterGreaterThan, "status_code", 200.0), + ), + ), + }, + newLineFmtExpr("blip{{ .foo }}blop {{.status_code}}"), + newLabelFmtExpr([]log.LabelFmt{ + log.NewRenameLabelFmt("foo", "bar"), + log.NewTemplateLabelFmt("status_code", "buzz{{.bar}}"), + }), + }, + }, + 5*time.Minute, + newUnwrapExpr("foo", ""), + nil), + OpRangeTypeAvg, &grouping{without: false, groups: []string{"namespace", "instance"}}, nil, + ), + OpTypeAvg, + &grouping{groups: []string{"foo"}}, + nil, + ), + ), + }, { in: ` label_replace( @@ -2196,7 +2358,7 @@ func TestParse(t *testing.T) { "(.*)" )`, exp: mustNewLabelReplaceExpr( - mustNewBinOpExpr(OpTypeAdd, BinOpOptions{ReturnBool: false}, + mustNewBinOpExpr(OpTypeAdd, &BinOpOptions{ReturnBool: false}, mustNewVectorAggregationExpr( newRangeAggregationExpr( newLogRange(&PipelineExpr{ @@ -2333,6 +2495,10 @@ func TestParse(t *testing.T) { in: `count_over_time({ foo ="bar" }[12m]) > count_over_time({ foo = "bar" }[12m])`, exp: &BinOpExpr{ op: OpTypeGT, + opts: &BinOpOptions{ + ReturnBool: false, + VectorMatching: nil, + }, SampleExpr: &RangeAggregationExpr{ left: &LogRange{ left: &MatchersExpr{matchers: []*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}}, @@ -2353,6 +2519,10 @@ func TestParse(t *testing.T) { in: `count_over_time({ foo = "bar" }[12m]) > 1`, exp: &BinOpExpr{ op: OpTypeGT, + opts: &BinOpOptions{ + ReturnBool: false, + VectorMatching: nil, + }, SampleExpr: &RangeAggregationExpr{ left: &LogRange{ left: &MatchersExpr{matchers: []*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}}, @@ -2372,6 +2542,10 @@ func TestParse(t *testing.T) { in: `count_over_time({ foo = "bar" }[12m]) or count_over_time({ foo = "bar" }[12m]) > 1`, exp: &BinOpExpr{ op: OpTypeOr, + opts: &BinOpOptions{ + ReturnBool: false, + VectorMatching: nil, + }, SampleExpr: &RangeAggregationExpr{ left: &LogRange{ left: &MatchersExpr{matchers: []*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}}, @@ -2381,6 +2555,10 @@ func TestParse(t *testing.T) { }, RHS: &BinOpExpr{ op: OpTypeGT, + opts: &BinOpOptions{ + ReturnBool: false, + VectorMatching: nil, + }, SampleExpr: &RangeAggregationExpr{ left: &LogRange{ left: &MatchersExpr{matchers: []*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}}, diff --git a/pkg/logql/shardmapper_test.go b/pkg/logql/shardmapper_test.go index 2c1d5c3ea6024..bc0029d9f483d 100644 --- a/pkg/logql/shardmapper_test.go +++ b/pkg/logql/shardmapper_test.go @@ -675,7 +675,11 @@ func TestMapping(t *testing.T) { { in: `1 + sum by (cluster) (rate({foo="bar"}[5m]))`, expr: &BinOpExpr{ - op: OpTypeAdd, + op: OpTypeAdd, + opts: &BinOpOptions{ + ReturnBool: false, + VectorMatching: nil, + }, SampleExpr: &LiteralExpr{value: 1}, RHS: &VectorAggregationExpr{ grouping: &grouping{ @@ -850,6 +854,10 @@ func TestMapping(t *testing.T) { in: `max(sum by (cluster) (rate({foo="bar"}[5m]))) / count(rate({foo="bar"}[5m]))`, expr: &BinOpExpr{ op: OpTypeDiv, + opts: &BinOpOptions{ + ReturnBool: false, + VectorMatching: nil, + }, SampleExpr: &VectorAggregationExpr{ operation: OpTypeMax, grouping: &grouping{}, From 0035ee37ce04230cc5b27ffa3fab480f19c91e90 Mon Sep 17 00:00:00 2001 From: garrettlish Date: Wed, 29 Sep 2021 15:45:35 +0800 Subject: [PATCH 2/6] add on/ignoring new operator shardmapper unit test --- pkg/logql/evaluator.go | 6 +- pkg/logql/shardmapper_test.go | 121 ++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 5 deletions(-) diff --git a/pkg/logql/evaluator.go b/pkg/logql/evaluator.go index 8431e782d62f8..1340c51907a60 100644 --- a/pkg/logql/evaluator.go +++ b/pkg/logql/evaluator.go @@ -586,11 +586,7 @@ func binOpStepEvaluator( } pair := pairs[hash] if pair[i] != nil { - if i == 0 { - failStepEvaluator(lhs, errors.New("multiple matches for labels")) - } else { - failStepEvaluator(rhs, errors.New("multiple matches for labels")) - } + failStepEvaluator(eval, errors.New("multiple matches for labels")) return false, ts, nil } pair[i] = &promql.Sample{ diff --git a/pkg/logql/shardmapper_test.go b/pkg/logql/shardmapper_test.go index bc0029d9f483d..82d2e98c04f09 100644 --- a/pkg/logql/shardmapper_test.go +++ b/pkg/logql/shardmapper_test.go @@ -972,6 +972,127 @@ func TestMapping(t *testing.T) { }, }, }, + { + in: `sum by (cluster) (rate({foo="bar"}[5m])) / ignoring(cluster) count(rate({foo="bar"}[5m]))`, + expr: &BinOpExpr{ + op: OpTypeDiv, + opts: &BinOpOptions{ + ReturnBool: false, + VectorMatching: &VectorMatching{ + On: false, + Include: []string{"cluster"}, + }, + }, + SampleExpr: &VectorAggregationExpr{ + grouping: &grouping{ + groups: []string{"cluster"}, + }, + operation: OpTypeSum, + left: &ConcatSampleExpr{ + DownstreamSampleExpr: DownstreamSampleExpr{ + shard: &astmapper.ShardAnnotation{ + Shard: 0, + Of: 2, + }, + SampleExpr: &VectorAggregationExpr{ + grouping: &grouping{ + groups: []string{"cluster"}, + }, + operation: OpTypeSum, + left: &RangeAggregationExpr{ + operation: OpRangeTypeRate, + left: &LogRange{ + left: &MatchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, + }, + }, + }, + next: &ConcatSampleExpr{ + DownstreamSampleExpr: DownstreamSampleExpr{ + shard: &astmapper.ShardAnnotation{ + Shard: 1, + Of: 2, + }, + SampleExpr: &VectorAggregationExpr{ + grouping: &grouping{ + groups: []string{"cluster"}, + }, + operation: OpTypeSum, + left: &RangeAggregationExpr{ + operation: OpRangeTypeRate, + left: &LogRange{ + left: &MatchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, + }, + }, + }, + next: nil, + }, + }, + }, + RHS: &VectorAggregationExpr{ + operation: OpTypeSum, + grouping: &grouping{}, + left: &ConcatSampleExpr{ + DownstreamSampleExpr: DownstreamSampleExpr{ + shard: &astmapper.ShardAnnotation{ + Shard: 0, + Of: 2, + }, + SampleExpr: &VectorAggregationExpr{ + grouping: &grouping{}, + operation: OpTypeCount, + left: &RangeAggregationExpr{ + operation: OpRangeTypeRate, + left: &LogRange{ + left: &MatchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, + }, + }, + }, + next: &ConcatSampleExpr{ + DownstreamSampleExpr: DownstreamSampleExpr{ + shard: &astmapper.ShardAnnotation{ + Shard: 1, + Of: 2, + }, + SampleExpr: &VectorAggregationExpr{ + grouping: &grouping{}, + operation: OpTypeCount, + left: &RangeAggregationExpr{ + operation: OpRangeTypeRate, + left: &LogRange{ + left: &MatchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, + }, + }, + }, + next: nil, + }, + }, + }, + }, + }, } { t.Run(tc.in, func(t *testing.T) { ast, err := ParseExpr(tc.in) From 73bcb19457f1c523b745a1fe84bf4c9460836830 Mon Sep 17 00:00:00 2001 From: garrettlish Date: Wed, 29 Sep 2021 16:29:46 +0800 Subject: [PATCH 3/6] add doc for on and ignoring keywords --- docs/sources/logql/_index.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/sources/logql/_index.md b/docs/sources/logql/_index.md index b4bf7fd3802d5..005bcf16ac108 100644 --- a/docs/sources/logql/_index.md +++ b/docs/sources/logql/_index.md @@ -143,6 +143,19 @@ More details can be found in the [Golang language documentation](https://golang. `2 * 3 % 2` is evaluated as `(2 * 3) % 2`. +### On and Ignoring keywords + +The `ignoring` keyword allows ignoring certain labels when matching, while the `on` keyword allows reducing the set of considered labels to a provided list: +```logql + ignoring/on() +``` +Example: +```logql +sum by(machine) (count_over_time({app="foo"}[1m])) > bool ignoring(machine) sum(count_over_time({app="bar"}[1m])) + +sum by(app,machine) (count_over_time({app="foo"}[1m])) + on(app) sum by (app) (count_over_time({app="bar"}[1m])) +``` + ## Comments LogQL queries can be commented using the `#` character: From 8f2e0b591d227937e1b4cd189253540a2e2446fc Mon Sep 17 00:00:00 2001 From: garrettlish Date: Thu, 30 Sep 2021 08:21:24 +0800 Subject: [PATCH 4/6] resolve comments * use a scoped error to replace mutate stepEvaluator via type switching * change VectorMatching Include from empty string array to nil * prohibit sharding when we're changing the label groupings such as on or ignoring * add unit test to cover multiple matches for labels error * fix expr.y format issue --- pkg/logql/ast.go | 4 ++++ pkg/logql/engine_test.go | 29 ++++++++++++++++++++++++----- pkg/logql/evaluator.go | 19 ++++--------------- pkg/logql/expr.y | 6 +++--- pkg/logql/expr.y.go | 4 ++-- 5 files changed, 37 insertions(+), 25 deletions(-) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 98f99968de90e..7172ceb336b34 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -998,6 +998,10 @@ func (e *BinOpExpr) String() string { // impl SampleExpr func (e *BinOpExpr) Shardable() bool { + if e.opts != nil && e.opts.VectorMatching != nil { + // prohibit sharding when we're changing the label groupings, such as on or ignoring + return false + } return shardableOps[e.op] && e.SampleExpr.Shardable() && e.RHS.Shardable() } diff --git a/pkg/logql/engine_test.go b/pkg/logql/engine_test.go index 03ae33144f13e..bd631885e4d0b 100644 --- a/pkg/logql/engine_test.go +++ b/pkg/logql/engine_test.go @@ -43,7 +43,7 @@ func TestEngine_LogsInstantQuery(t *testing.T) { data interface{} params interface{} - expected promql_parser.Value + expected interface{} }{ { `{app="foo"}`, time.Unix(30, 0), logproto.FORWARD, 10, @@ -636,7 +636,7 @@ func TestEngine_LogsInstantQuery(t *testing.T) { }, }, { - `sum by (app,machine) (count_over_time({app="foo"}[1m])) + on (app) sum by (app) (count_over_time({app="foo"}[1m]))`, + `sum by (app,machine) (count_over_time({app="foo"}[1m])) + on () sum by (app) (count_over_time({app="foo"}[1m]))`, time.Unix(60, 0), logproto.FORWARD, 0, @@ -669,6 +669,21 @@ func TestEngine_LogsInstantQuery(t *testing.T) { promql.Sample{Point: promql.Point{T: 60 * 1000, V: 0}, Metric: labels.Labels{labels.Label{Name: "app", Value: "foo"}, labels.Label{Name: "machine", Value: "fuzz"}}}, }, }, + { + `sum by (app,machine) (count_over_time({app="foo"}[1m])) > bool ignoring (machine) sum by (app) (count_over_time({app="foo"}[1m]))`, + time.Unix(60, 0), + logproto.FORWARD, + 0, + [][]logproto.Series{ + {newSeries(testSize, identity, `{app="foo",machine="fuzz"}`), newSeries(testSize, identity, `{app="foo",machine="buzz"}`)}, + {newSeries(testSize, identity, `{app="foo"}`)}, + }, + []SelectSampleParams{ + {&logproto.SampleQueryRequest{Start: time.Unix(0, 0), End: time.Unix(60, 0), Selector: `sum by (app,machine) (count_over_time({app="foo"}[1m]))`}}, + {&logproto.SampleQueryRequest{Start: time.Unix(0, 0), End: time.Unix(60, 0), Selector: `sum by (app) (count_over_time({app="foo"}[1m]))`}}, + }, + errors.New("multiple matches for labels"), + }, } { test := test t.Run(fmt.Sprintf("%s %s", test.qs, test.direction), func(t *testing.T) { @@ -683,10 +698,14 @@ func TestEngine_LogsInstantQuery(t *testing.T) { limit: test.limit, }) res, err := q.Exec(user.InjectOrgID(context.Background(), "fake")) - if err != nil { - t.Fatal(err) + if expectedError, ok := test.expected.(error); ok { + assert.Equal(t, expectedError.Error(), err.Error()) + } else { + if err != nil { + t.Fatal(err) + } + assert.Equal(t, test.expected, res.Data) } - assert.Equal(t, test.expected, res.Data) }) } } diff --git a/pkg/logql/evaluator.go b/pkg/logql/evaluator.go index 1340c51907a60..65c6e3cd91b21 100644 --- a/pkg/logql/evaluator.go +++ b/pkg/logql/evaluator.go @@ -586,7 +586,7 @@ func binOpStepEvaluator( } pair := pairs[hash] if pair[i] != nil { - failStepEvaluator(eval, errors.New("multiple matches for labels")) + err = errors.New("multiple matches for labels") return false, ts, nil } pair[i] = &promql.Sample{ @@ -619,6 +619,9 @@ func binOpStepEvaluator( return lastError }, func() error { var errs []error + if err != nil { + errs = append(errs, err) + } for _, ev := range []StepEvaluator{lhs, rhs} { if err := ev.Error(); err != nil { errs = append(errs, err) @@ -1033,17 +1036,3 @@ func absentLabels(expr SampleExpr) labels.Labels { } return m } - -// failStepEvaluator marks the step evaluator as failed -func failStepEvaluator(se StepEvaluator, err error) { - switch s := se.(type) { - case *stepEvaluator: - s.err = func() error { - return err - } - case *rangeVectorEvaluator: - s.err = err - case *absentRangeVectorEvaluator: - s.err = err - } -} diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index ab20c72d605d6..26ce2004374eb 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -34,7 +34,7 @@ import ( duration time.Duration LiteralExpr *LiteralExpr BinOpModifier *BinOpOptions - BoolModifier *BinOpOptions + BoolModifier *BinOpOptions LabelParser *LabelParserExpr LineFilters *LineFilterExpr LineFilter *LineFilterExpr @@ -373,7 +373,7 @@ binOpModifier: | boolModifier ON OPEN_PARENTHESIS CLOSE_PARENTHESIS { $$ = $1 - $$.VectorMatching = &VectorMatching{On: true, Include: []string{}} + $$.VectorMatching = &VectorMatching{On: true, Include: nil} } | boolModifier IGNORING OPEN_PARENTHESIS labels CLOSE_PARENTHESIS { @@ -383,7 +383,7 @@ binOpModifier: | boolModifier IGNORING OPEN_PARENTHESIS CLOSE_PARENTHESIS { $$ = $1 - $$.VectorMatching = &VectorMatching{On: false, Include: []string{}} + $$.VectorMatching = &VectorMatching{On: false, Include: nil} } ; diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index 31ce3950c66b9..129441d62db9d 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -1548,7 +1548,7 @@ exprdefault: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpModifier = exprDollar[1].BoolModifier - exprVAL.BinOpModifier.VectorMatching = &VectorMatching{On: true, Include: []string{}} + exprVAL.BinOpModifier.VectorMatching = &VectorMatching{On: true, Include: nil} } case 150: exprDollar = exprS[exprpt-5 : exprpt+1] @@ -1560,7 +1560,7 @@ exprdefault: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpModifier = exprDollar[1].BoolModifier - exprVAL.BinOpModifier.VectorMatching = &VectorMatching{On: false, Include: []string{}} + exprVAL.BinOpModifier.VectorMatching = &VectorMatching{On: false, Include: nil} } case 152: exprDollar = exprS[exprpt-0 : exprpt+1] From 48314f3e2a2fd49dc6e1b1ace1ec8dacc81e4040 Mon Sep 17 00:00:00 2001 From: garrettlish Date: Fri, 1 Oct 2021 23:18:41 +0800 Subject: [PATCH 5/6] resolve comments * reformat on and ignoring keywords doc * add sharding unit test for on and ignoring keywords --- docs/sources/logql/_index.md | 17 ++++++++++++----- pkg/logql/shardmapper_test.go | 8 ++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/docs/sources/logql/_index.md b/docs/sources/logql/_index.md index 005bcf16ac108..a20bc563edb0e 100644 --- a/docs/sources/logql/_index.md +++ b/docs/sources/logql/_index.md @@ -143,16 +143,23 @@ More details can be found in the [Golang language documentation](https://golang. `2 * 3 % 2` is evaluated as `(2 * 3) % 2`. -### On and Ignoring keywords - -The `ignoring` keyword allows ignoring certain labels when matching, while the `on` keyword allows reducing the set of considered labels to a provided list: +### Keywords on and ignoring +The `ignoring` keyword causes specified labels to be ignored during matching. +The syntax: ```logql - ignoring/on() + ignoring() ``` Example: ```logql sum by(machine) (count_over_time({app="foo"}[1m])) > bool ignoring(machine) sum(count_over_time({app="bar"}[1m])) - +``` +The on keyword reduces the set of considered labels to a specified list. +The syntax: +```logql + on() +``` +Example: +```logql sum by(app,machine) (count_over_time({app="foo"}[1m])) + on(app) sum by (app) (count_over_time({app="bar"}[1m])) ``` diff --git a/pkg/logql/shardmapper_test.go b/pkg/logql/shardmapper_test.go index 82d2e98c04f09..56d2ca41ff168 100644 --- a/pkg/logql/shardmapper_test.go +++ b/pkg/logql/shardmapper_test.go @@ -192,6 +192,14 @@ func TestMappingStrings(t *testing.T) { in: `sum(count_over_time({foo="bar"} | logfmt | label_format bar=baz | bar="buz" [5m]))`, out: `sum(count_over_time({foo="bar"} | logfmt | label_format bar=baz | bar="buz" [5m]))`, }, + { + in: `sum by (cluster) (rate({foo="bar"} [5m])) + ignoring(machine) sum by (cluster,machine) (rate({foo="bar"} [5m]))`, + out: `(sumby(cluster)(downstream++downstream)+ignoring(machine)sumby(cluster,machine)(downstream++downstream))`, + }, + { + in: `sum by (cluster) (sum by (cluster) (rate({foo="bar"} [5m])) + ignoring(machine) sum by (cluster,machine) (rate({foo="bar"} [5m])))`, + out: `sumby(cluster)((sumby(cluster)(downstream++downstream)+ignoring(machine)sumby(cluster,machine)(downstream++downstream)))`, + }, } { t.Run(tc.in, func(t *testing.T) { ast, err := ParseExpr(tc.in) From 8064e3dfef381d16da718d024a9c7b329f376e70 Mon Sep 17 00:00:00 2001 From: garrettlish Date: Sat, 2 Oct 2021 07:59:49 +0800 Subject: [PATCH 6/6] resolve comments: add a sentence for each example to help docs readers to understand the larger picture --- docs/sources/logql/_index.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/sources/logql/_index.md b/docs/sources/logql/_index.md index a20bc563edb0e..34a132e76996d 100644 --- a/docs/sources/logql/_index.md +++ b/docs/sources/logql/_index.md @@ -149,18 +149,18 @@ The syntax: ```logql ignoring() ``` -Example: +This example will return the machines which total count within the last minutes exceed average value for app `foo`. ```logql -sum by(machine) (count_over_time({app="foo"}[1m])) > bool ignoring(machine) sum(count_over_time({app="bar"}[1m])) +max by(machine) (count_over_time({app="foo"}[1m])) > bool ignoring(machine) avg(count_over_time({app="foo"}[1m])) ``` The on keyword reduces the set of considered labels to a specified list. The syntax: ```logql on() ``` -Example: +This example will return every machine total count within the last minutes ratio in app `foo`: ```logql -sum by(app,machine) (count_over_time({app="foo"}[1m])) + on(app) sum by (app) (count_over_time({app="bar"}[1m])) +sum by(machine) (count_over_time({app="foo"}[1m])) / on() sum(count_over_time({app="foo"}[1m])) ``` ## Comments