From d5bb84fe9ff1e490a9573bcc0d5879984267f3ff Mon Sep 17 00:00:00 2001 From: Jiyong Huang Date: Thu, 7 Mar 2024 11:54:00 +0800 Subject: [PATCH] refactor(xsql): unify row and collection - Each node now only needs to handle two kinds of data Row/Collection. Other data may be error, watermark, barrier Signed-off-by: Jiyong Huang --- go.work.sum | 2 + internal/topo/node/join_align_node.go | 26 +- internal/topo/node/lookup_node.go | 8 +- internal/topo/node/lookup_node_test.go | 26 +- internal/topo/node/node.go | 2 +- internal/topo/node/operations.go | 2 +- internal/topo/node/sink_node_test.go | 2 +- internal/topo/node/switch_node.go | 15 +- internal/topo/node/switch_node_test.go | 14 +- internal/topo/node/window_op.go | 6 +- internal/topo/node/window_op_test.go | 8 +- internal/topo/operator/aggregate_operator.go | 8 +- internal/topo/operator/aggregate_test.go | 90 +++--- .../topo/operator/analyticfuncs_operator.go | 10 +- internal/topo/operator/filter_operator.go | 16 +- internal/topo/operator/filter_test.go | 70 ++-- internal/topo/operator/func_operator.go | 23 +- internal/topo/operator/having_test.go | 76 ++--- internal/topo/operator/join_multi_test.go | 134 ++++---- internal/topo/operator/join_operator.go | 26 +- internal/topo/operator/join_test.go | 304 +++++++++--------- internal/topo/operator/math_func_test.go | 40 ++- internal/topo/operator/misc_func_test.go | 4 +- internal/topo/operator/order_operator.go | 8 +- internal/topo/operator/order_test.go | 117 ++++--- internal/topo/operator/project_operator.go | 76 ++--- internal/topo/operator/project_test.go | 294 ++++++++--------- internal/topo/operator/projectset_operator.go | 32 +- internal/topo/operator/script_test.go | 8 +- internal/topo/operator/table_processor.go | 8 +- .../topo/operator/table_processor_test.go | 8 +- internal/topo/operator/windowfunc_operator.go | 8 +- internal/xsql/collection.go | 90 +++--- internal/xsql/collection_test.go | 22 +- internal/xsql/row.go | 66 ++-- internal/xsql/row_test.go | 20 +- internal/xsql/sorter.go | 36 +-- 37 files changed, 844 insertions(+), 861 deletions(-) diff --git a/go.work.sum b/go.work.sum index 6c0aa0eda4..5c16766c70 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1075,6 +1075,7 @@ github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/ github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0= +github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle/v2 v2.1.2 h1:0f7vaaXINONKTsxYDn4otOAiJanX/BMeAtY//BXqzlg= github.com/jackc/puddle/v2 v2.1.2/go.mod h1:2lpufsF5mRHO6SuZkm0fNYxM6SWHfvyFj62KwNzgels= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= @@ -1440,6 +1441,7 @@ golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0 golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= diff --git a/internal/topo/node/join_align_node.go b/internal/topo/node/join_align_node.go index 1976e6a7bd..d479068909 100644 --- a/internal/topo/node/join_align_node.go +++ b/internal/topo/node/join_align_node.go @@ -100,17 +100,19 @@ func (n *JoinAlignNode) Exec(ctx api.StreamContext, errCh chan<- error) { n.alignBatch(ctx, d) } else { // table window log.Debugf("JoinAlignNode receive batch source %s", d) - emitter := d.Content[0].GetEmitter() - // Buffer and update batch inputs - _, ok := n.batch[emitter] - if !ok { - e := fmt.Errorf("run JoinAlignNode error: receive batch input from unknown emitter %[1]T(%[1]v)", d) - n.Broadcast(e) - n.statManager.IncTotalExceptions(e.Error()) - break + if et, ok := d.Content[0].(xsql.EmittedData); ok { + emitter := et.GetEmitter() + // Buffer and update batch inputs + _, ok := n.batch[emitter] + if !ok { + e := fmt.Errorf("run JoinAlignNode error: receive batch input from unknown emitter %[1]T(%[1]v)", d) + n.Broadcast(e) + n.statManager.IncTotalExceptions(e.Error()) + break + } + n.batch[emitter] = convertToTupleSlice(d.Content) + _ = ctx.PutState(BatchKey, n.batch) } - n.batch[emitter] = convertToTupleSlice(d.Content) - _ = ctx.PutState(BatchKey, n.batch) } default: e := fmt.Errorf("run JoinAlignNode error: invalid input type but got %[1]T(%[1]v)", d) @@ -129,7 +131,7 @@ func (n *JoinAlignNode) Exec(ctx api.StreamContext, errCh chan<- error) { }() } -func convertToTupleSlice(content []xsql.TupleRow) []*xsql.Tuple { +func convertToTupleSlice(content []xsql.Row) []*xsql.Tuple { tuples := make([]*xsql.Tuple, len(content)) for i, v := range content { tuples[i] = v.(*xsql.Tuple) @@ -143,7 +145,7 @@ func (n *JoinAlignNode) alignBatch(_ api.StreamContext, input any) { switch t := input.(type) { case *xsql.Tuple: w = &xsql.WindowTuples{ - Content: make([]xsql.TupleRow, 0), + Content: make([]xsql.Row, 0), } w.AddTuple(t) case *xsql.WindowTuples: diff --git a/internal/topo/node/lookup_node.go b/internal/topo/node/lookup_node.go index 95fc44f0d2..bc8462849d 100644 --- a/internal/topo/node/lookup_node.go +++ b/internal/topo/node/lookup_node.go @@ -114,7 +114,7 @@ func (n *LookupNode) Exec(ctx api.StreamContext, errCh chan<- error) { n.statManager.IncTotalExceptions(d.Error()) case *xsql.WatermarkTuple: n.Broadcast(d) - case xsql.TupleRow: + case xsql.Row: log.Debugf("Lookup Node receive tuple input %s", d) n.statManager.ProcessTimeStart() sets := &xsql.JoinTuples{Content: make([]*xsql.JoinTuple, 0)} @@ -134,7 +134,7 @@ func (n *LookupNode) Exec(ctx api.StreamContext, errCh chan<- error) { n.statManager.ProcessTimeStart() sets := &xsql.JoinTuples{Content: make([]*xsql.JoinTuple, 0), WindowRange: item.(*xsql.WindowTuples).GetWindowRange()} err := d.Range(func(i int, r xsql.ReadonlyRow) (bool, error) { - tr, ok := r.(xsql.TupleRow) + tr, ok := r.(xsql.Row) if !ok { return false, fmt.Errorf("Invalid window element, must be a tuple row but got %v", r) } @@ -171,7 +171,7 @@ func (n *LookupNode) Exec(ctx api.StreamContext, errCh chan<- error) { } // lookup will lookup the cache firstly, if expires, read the external source -func (n *LookupNode) lookup(ctx api.StreamContext, d xsql.TupleRow, fv *xsql.FunctionValuer, ns api.LookupSource, tuples *xsql.JoinTuples, c *cache.Cache) error { +func (n *LookupNode) lookup(ctx api.StreamContext, d xsql.Row, fv *xsql.FunctionValuer, ns api.LookupSource, tuples *xsql.JoinTuples, c *cache.Cache) error { ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(d, fv)} cvs := make([]interface{}, len(n.vals)) hasNil := false @@ -230,7 +230,7 @@ func (n *LookupNode) lookup(ctx api.StreamContext, d xsql.TupleRow, fv *xsql.Fun } } -func (n *LookupNode) merge(ctx api.StreamContext, d xsql.TupleRow, r []map[string]interface{}) { +func (n *LookupNode) merge(ctx api.StreamContext, d xsql.Row, r []map[string]interface{}) { n.statManager.ProcessTimeStart() sets := &xsql.JoinTuples{Content: make([]*xsql.JoinTuple, 0)} diff --git a/internal/topo/node/lookup_node_test.go b/internal/topo/node/lookup_node_test.go index b87a2a8f4e..121512238a 100644 --- a/internal/topo/node/lookup_node_test.go +++ b/internal/topo/node/lookup_node_test.go @@ -1,4 +1,4 @@ -// Copyright 2022-2023 EMQ Technologies Co., Ltd. +// Copyright 2022-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -145,7 +145,7 @@ func TestLookup(t *testing.T) { output: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{ Emitter: "demo", Message: map[string]interface{}{ @@ -162,7 +162,7 @@ func TestLookup(t *testing.T) { }, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{ Emitter: "demo", Message: map[string]interface{}{ @@ -184,7 +184,7 @@ func TestLookup(t *testing.T) { }, { input: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "demo", Message: map[string]interface{}{ @@ -205,7 +205,7 @@ func TestLookup(t *testing.T) { output: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{ Emitter: "demo", Message: map[string]interface{}{ @@ -222,7 +222,7 @@ func TestLookup(t *testing.T) { }, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{ Emitter: "demo", Message: map[string]interface{}{ @@ -239,7 +239,7 @@ func TestLookup(t *testing.T) { }, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{ Emitter: "demo", Message: map[string]interface{}{ @@ -256,7 +256,7 @@ func TestLookup(t *testing.T) { }, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{ Emitter: "demo", Message: map[string]interface{}{ @@ -273,7 +273,7 @@ func TestLookup(t *testing.T) { }, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{ Emitter: "demo", Message: map[string]interface{}{ @@ -290,7 +290,7 @@ func TestLookup(t *testing.T) { }, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{ Emitter: "demo", Message: map[string]interface{}{ @@ -406,7 +406,7 @@ func TestCachedLookup(t *testing.T) { outputBefore := &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{ Emitter: "demo", Message: map[string]interface{}{ @@ -424,7 +424,7 @@ func TestCachedLookup(t *testing.T) { }, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{ Emitter: "demo", Message: map[string]interface{}{ @@ -447,7 +447,7 @@ func TestCachedLookup(t *testing.T) { outputAfter := &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{ Emitter: "demo", Message: map[string]interface{}{ diff --git a/internal/topo/node/node.go b/internal/topo/node/node.go index 5acc9dc485..3edad53a93 100644 --- a/internal/topo/node/node.go +++ b/internal/topo/node/node.go @@ -158,7 +158,7 @@ func (o *defaultNode) doBroadcast(val interface{}) { case xsql.Collection: val = vt.Clone() break - case xsql.TupleRow: + case xsql.Row: val = vt.Clone() } } diff --git a/internal/topo/node/operations.go b/internal/topo/node/operations.go index 6daf4239e6..6e0ef382b1 100644 --- a/internal/topo/node/operations.go +++ b/internal/topo/node/operations.go @@ -123,7 +123,7 @@ func (o *UnaryOperator) doOp(ctx api.StreamContext, errCh chan<- error) { o.statManager.IncTotalMessagesProcessed(1) o.statManager.IncTotalExceptions(val.Error()) continue - case []xsql.TupleRow: + case []xsql.Row: o.statManager.ProcessTimeEnd() for _, v := range val { o.Broadcast(v) diff --git a/internal/topo/node/sink_node_test.go b/internal/topo/node/sink_node_test.go index 720b0ad275..f9baa78fc6 100644 --- a/internal/topo/node/sink_node_test.go +++ b/internal/topo/node/sink_node_test.go @@ -514,7 +514,7 @@ func Test_itemToMap(t *testing.T) { { name: "test4", args: args{ - item: xsql.Collection(&xsql.WindowTuples{Content: []xsql.TupleRow{ + item: xsql.Collection(&xsql.WindowTuples{Content: []xsql.Row{ &xsql.Tuple{Emitter: "a", Message: map[string]interface{}{"a": 1, "b": "2"}, Timestamp: conf.GetNowInMilli(), Metadata: nil}, }}), }, diff --git a/internal/topo/node/switch_node.go b/internal/topo/node/switch_node.go index bc3fba7390..16f43d31ca 100644 --- a/internal/topo/node/switch_node.go +++ b/internal/topo/node/switch_node.go @@ -97,13 +97,20 @@ func (n *SwitchNode) Exec(ctx api.StreamContext, errCh chan<- error) { n.statManager.IncTotalExceptions(d.Error()) case *xsql.WatermarkTuple: n.Broadcast(d) - case xsql.TupleRow: + case xsql.Row: ctx.GetLogger().Debugf("SwitchNode receive tuple input %s", d) ve = &xsql.ValuerEval{Valuer: xsql.MultiValuer(d, fv)} - case xsql.SingleCollection: + case xsql.Collection: ctx.GetLogger().Debugf("SwitchNode receive window input %s", d) - afv.SetData(d) - ve = &xsql.ValuerEval{Valuer: xsql.MultiAggregateValuer(d, fv, d, fv, afv, &xsql.WildcardValuer{Data: d})} + if cr, ok := d.(xsql.CollectionRow); ok { + afv.SetData(cr) + ve = &xsql.ValuerEval{Valuer: xsql.MultiAggregateValuer(cr, fv, cr, fv, afv, &xsql.WildcardValuer{Data: cr})} + } else { + e := fmt.Errorf("run switch node error: invalid input type but got %[1]T(%[1]v)", d) + n.Broadcast(e) + n.statManager.IncTotalExceptions(e.Error()) + break + } default: e := fmt.Errorf("run switch node error: invalid input type but got %[1]T(%[1]v)", d) n.Broadcast(e) diff --git a/internal/topo/node/switch_node_test.go b/internal/topo/node/switch_node_test.go index 55737ba6e3..db0a6aff13 100644 --- a/internal/topo/node/switch_node_test.go +++ b/internal/topo/node/switch_node_test.go @@ -1,4 +1,4 @@ -// Copyright 2022-2023 EMQ Technologies Co., Ltd. +// Copyright 2022-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -209,7 +209,7 @@ outterFor: func TestCollection(t *testing.T) { inputs := []*xsql.WindowTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Message: map[string]interface{}{ "f1": "v1", @@ -224,7 +224,7 @@ func TestCollection(t *testing.T) { }, }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Message: map[string]interface{}{ "f1": "v2", @@ -245,7 +245,7 @@ func TestCollection(t *testing.T) { }, }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Message: map[string]interface{}{ "f1": "v1", @@ -270,7 +270,7 @@ func TestCollection(t *testing.T) { outputs := [][]*xsql.WindowTuples{ { // avg(f2) > 50 { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Message: map[string]interface{}{ "f1": "v1", @@ -285,7 +285,7 @@ func TestCollection(t *testing.T) { }, }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Message: map[string]interface{}{ "f1": "v1", @@ -309,7 +309,7 @@ func TestCollection(t *testing.T) { }, { // else { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Message: map[string]interface{}{ "f1": "v2", diff --git a/internal/topo/node/window_op.go b/internal/topo/node/window_op.go index bf5ae3bbad..cd9f0e765f 100644 --- a/internal/topo/node/window_op.go +++ b/internal/topo/node/window_op.go @@ -501,7 +501,7 @@ func (tl *TupleList) count() int { func (tl *TupleList) nextCountWindow() *xsql.WindowTuples { results := &xsql.WindowTuples{ - Content: make([]xsql.TupleRow, 0), + Content: make([]xsql.Row, 0), } var subT []*xsql.Tuple subT = tl.tuples[len(tl.tuples)-tl.size : len(tl.tuples)] @@ -533,7 +533,7 @@ func (o *WindowOperator) isTimeRelatedWindow() bool { return false } -func (o *WindowOperator) handleInputs(inputs []*xsql.Tuple, triggerTime int64, ctx api.StreamContext) ([]*xsql.Tuple, []xsql.TupleRow) { +func (o *WindowOperator) handleInputs(inputs []*xsql.Tuple, triggerTime int64, ctx api.StreamContext) ([]*xsql.Tuple, []xsql.Row) { log := ctx.GetLogger() log.Debugf("window %s triggered at %s(%d)", o.name, time.Unix(triggerTime/1000, triggerTime%1000), triggerTime) var delta int64 @@ -541,7 +541,7 @@ func (o *WindowOperator) handleInputs(inputs []*xsql.Tuple, triggerTime int64, c if o.window.Type == ast.HOPPING_WINDOW || o.window.Type == ast.SLIDING_WINDOW { delta = o.calDelta(triggerTime, log) } - content := make([]xsql.TupleRow, 0) + content := make([]xsql.Row, 0) i := 0 // Sync table for _, tuple := range inputs { diff --git a/internal/topo/node/window_op_test.go b/internal/topo/node/window_op_test.go index 716cd2337e..91be543788 100644 --- a/internal/topo/node/window_op_test.go +++ b/internal/topo/node/window_op_test.go @@ -1,4 +1,4 @@ -// Copyright 2021-2023 EMQ Technologies Co., Ltd. +// Copyright 2021-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -155,7 +155,7 @@ func TestCountWindow(t *testing.T) { expWinCount: 1, winTupleSets: []xsql.WindowTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Message: map[string]interface{}{ "f1": "v1", @@ -216,7 +216,7 @@ func TestCountWindow(t *testing.T) { expWinCount: 1, winTupleSets: []xsql.WindowTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Message: map[string]interface{}{ "f3": "v3", @@ -257,7 +257,7 @@ func TestCountWindow(t *testing.T) { expWinCount: 1, winTupleSets: []xsql.WindowTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Message: map[string]interface{}{ "f4": "v4", diff --git a/internal/topo/operator/aggregate_operator.go b/internal/topo/operator/aggregate_operator.go index 64bdf30558..bb7edca3b8 100644 --- a/internal/topo/operator/aggregate_operator.go +++ b/internal/topo/operator/aggregate_operator.go @@ -1,4 +1,4 @@ -// Copyright 2021-2022 EMQ Technologies Co., Ltd. +// Copyright 2021-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -38,12 +38,12 @@ func (p *AggregateOp) Apply(ctx api.StreamContext, data interface{}, fv *xsql.Fu switch input := data.(type) { case error: return input - case xsql.SingleCollection: + case xsql.Collection: wr := input.GetWindowRange() result := make(map[string]*xsql.GroupedTuples) err := input.Range(func(i int, ir xsql.ReadonlyRow) (bool, error) { var name string - tr := ir.(xsql.TupleRow) + tr := ir.(xsql.Row) ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(tr, &xsql.WindowRangeValuer{WindowRange: wr}, fv)} for _, d := range p.Dimensions { r := ve.Eval(d.Expr) @@ -54,7 +54,7 @@ func (p *AggregateOp) Apply(ctx api.StreamContext, data interface{}, fv *xsql.Fu } } if ts, ok := result[name]; !ok { - result[name] = &xsql.GroupedTuples{Content: []xsql.TupleRow{tr}, WindowRange: wr} + result[name] = &xsql.GroupedTuples{Content: []xsql.Row{tr}, WindowRange: wr} } else { ts.Content = append(ts.Content, tr) } diff --git a/internal/topo/operator/aggregate_test.go b/internal/topo/operator/aggregate_test.go index b64231af08..0ee593b9ed 100644 --- a/internal/topo/operator/aggregate_test.go +++ b/internal/topo/operator/aggregate_test.go @@ -1,4 +1,4 @@ -// Copyright 2021-2022 EMQ Technologies Co., Ltd. +// Copyright 2021-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -36,7 +36,7 @@ func TestAggregatePlan_Apply(t *testing.T) { { sql: "SELECT abc FROM src1 GROUP BY TUMBLINGWINDOW(ss, 10), f1", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -53,7 +53,7 @@ func TestAggregatePlan_Apply(t *testing.T) { result: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -66,7 +66,7 @@ func TestAggregatePlan_Apply(t *testing.T) { WindowRange: xsql.NewWindowRange(1541152486013, 1541152487013), }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}, @@ -80,7 +80,7 @@ func TestAggregatePlan_Apply(t *testing.T) { { sql: "SELECT abc FROM src1 GROUP BY id1, TUMBLINGWINDOW(ss, 10), f1", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -96,7 +96,7 @@ func TestAggregatePlan_Apply(t *testing.T) { result: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -104,7 +104,7 @@ func TestAggregatePlan_Apply(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}, @@ -112,7 +112,7 @@ func TestAggregatePlan_Apply(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}, @@ -125,7 +125,7 @@ func TestAggregatePlan_Apply(t *testing.T) { { sql: "SELECT abc FROM src1 GROUP BY meta(topic), TUMBLINGWINDOW(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -145,7 +145,7 @@ func TestAggregatePlan_Apply(t *testing.T) { result: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -159,7 +159,7 @@ func TestAggregatePlan_Apply(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}, @@ -175,19 +175,19 @@ func TestAggregatePlan_Apply(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, @@ -197,9 +197,9 @@ func TestAggregatePlan_Apply(t *testing.T) { result: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, @@ -208,9 +208,9 @@ func TestAggregatePlan_Apply(t *testing.T) { WindowRange: xsql.NewWindowRange(1541152486013, 1541152487013), }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, @@ -219,9 +219,9 @@ func TestAggregatePlan_Apply(t *testing.T) { WindowRange: xsql.NewWindowRange(1541152486013, 1541152487013), }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, @@ -236,19 +236,19 @@ func TestAggregatePlan_Apply(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, @@ -257,24 +257,24 @@ func TestAggregatePlan_Apply(t *testing.T) { result: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, @@ -289,19 +289,19 @@ func TestAggregatePlan_Apply(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1", "ts": cast.TimeFromUnixMilli(1568854515000)}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2", "ts": cast.TimeFromUnixMilli(1568854573431)}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1", "ts": cast.TimeFromUnixMilli(1568854515000)}}, }, }, @@ -310,24 +310,24 @@ func TestAggregatePlan_Apply(t *testing.T) { result: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1", "ts": cast.TimeFromUnixMilli(1568854515000)}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1", "ts": cast.TimeFromUnixMilli(1568854515000)}}, }, }, }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2", "ts": cast.TimeFromUnixMilli(1568854573431)}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, @@ -340,7 +340,7 @@ func TestAggregatePlan_Apply(t *testing.T) { { sql: "SELECT abc FROM src1 GROUP BY TUMBLINGWINDOW(ss, 10), CASE WHEN id1 > 1 THEN \"others\" ELSE \"one\" END", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -356,7 +356,7 @@ func TestAggregatePlan_Apply(t *testing.T) { result: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -364,7 +364,7 @@ func TestAggregatePlan_Apply(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}, @@ -384,12 +384,12 @@ func TestAggregatePlan_Apply(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "B", Message: xsql.Message{"module": 1, "topic": "moduleB topic", "value": 1}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "C", Message: xsql.Message{"module": 1, "topic": "moduleC topic", "value": 100}}, }, }, @@ -398,14 +398,14 @@ func TestAggregatePlan_Apply(t *testing.T) { result: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "B", Message: xsql.Message{"module": 1, "topic": "moduleB topic", "value": 1}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "C", Message: xsql.Message{"module": 1, "topic": "moduleC topic", "value": 100}}, }, }, @@ -467,7 +467,7 @@ func TestAggregatePlanError(t *testing.T) { { sql: "SELECT abc FROM src1 GROUP BY TUMBLINGWINDOW(ss, 10), f1 * 2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, diff --git a/internal/topo/operator/analyticfuncs_operator.go b/internal/topo/operator/analyticfuncs_operator.go index f770ac990a..bc6bb96bd2 100644 --- a/internal/topo/operator/analyticfuncs_operator.go +++ b/internal/topo/operator/analyticfuncs_operator.go @@ -1,4 +1,4 @@ -// Copyright 2022 EMQ Technologies Co., Ltd. +// Copyright 2022-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -27,7 +27,7 @@ type AnalyticFuncsOp struct { FieldFuncs []*ast.Call } -func (p *AnalyticFuncsOp) evalTupleFunc(calls []*ast.Call, ve *xsql.ValuerEval, input xsql.TupleRow) (xsql.TupleRow, error) { +func (p *AnalyticFuncsOp) evalTupleFunc(calls []*ast.Call, ve *xsql.ValuerEval, input xsql.Row) (xsql.Row, error) { for _, call := range calls { f := call result := ve.Eval(f) @@ -39,7 +39,7 @@ func (p *AnalyticFuncsOp) evalTupleFunc(calls []*ast.Call, ve *xsql.ValuerEval, return input, nil } -func (p *AnalyticFuncsOp) evalCollectionFunc(calls []*ast.Call, fv *xsql.FunctionValuer, input xsql.SingleCollection) (xsql.SingleCollection, error) { +func (p *AnalyticFuncsOp) evalCollectionFunc(calls []*ast.Call, fv *xsql.FunctionValuer, input xsql.Collection) (xsql.Collection, error) { err := input.RangeSet(func(_ int, row xsql.Row) (bool, error) { ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(row, &xsql.WindowRangeValuer{WindowRange: input.GetWindowRange()}, fv, &xsql.WildcardValuer{Data: row})} for _, call := range calls { @@ -64,7 +64,7 @@ func (p *AnalyticFuncsOp) Apply(ctx api.StreamContext, data interface{}, fv *xsq switch input := data.(type) { case error: return input - case xsql.TupleRow: + case xsql.Row: ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(input, fv)} input, err = p.evalTupleFunc(p.FieldFuncs, ve, input) if err != nil { @@ -75,7 +75,7 @@ func (p *AnalyticFuncsOp) Apply(ctx api.StreamContext, data interface{}, fv *xsq return err } data = input - case xsql.SingleCollection: + case xsql.Collection: input, err = p.evalCollectionFunc(p.FieldFuncs, fv, input) if err != nil { return err diff --git a/internal/topo/operator/filter_operator.go b/internal/topo/operator/filter_operator.go index e99d1affca..50df9ed53a 100644 --- a/internal/topo/operator/filter_operator.go +++ b/internal/topo/operator/filter_operator.go @@ -1,4 +1,4 @@ -// Copyright 2021-2023 EMQ Technologies Co., Ltd. +// Copyright 2021-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -27,18 +27,18 @@ type FilterOp struct { StateFuncs []*ast.Call } -// Apply -/* - * input: *xsql.Tuple from preprocessor | xsql.WindowTuples from windowOp | xsql.JoinTuples from joinOp - * output: *xsql.Tuple | xsql.WindowTuples | xsql.JoinTuples - */ +// Apply the filter operator to each message in the stream +// The input data could be a xsql.Row or a xsql.Collection +// For xsql.Row, apply the condition to the row and return the row if the condition is true +// For xsql.Collection, apply the condition to each row and return the rows that meet the condition +// If error happens, return the error func (p *FilterOp) Apply(ctx api.StreamContext, data interface{}, fv *xsql.FunctionValuer, _ *xsql.AggregateFunctionValuer) interface{} { log := ctx.GetLogger() log.Debugf("filter plan receive %v", data) switch input := data.(type) { case error: return input - case xsql.TupleRow: + case xsql.Row: ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(input, fv)} result := ve.Eval(p.Condition) switch r := result.(type) { @@ -56,7 +56,7 @@ func (p *FilterOp) Apply(ctx api.StreamContext, data interface{}, fv *xsql.Funct default: return fmt.Errorf("run Where error: invalid condition that returns non-bool value %[1]T(%[1]v)", r) } - case xsql.SingleCollection: + case xsql.Collection: var sel []int err := input.Range(func(i int, r xsql.ReadonlyRow) (bool, error) { ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(r, fv)} diff --git a/internal/topo/operator/filter_test.go b/internal/topo/operator/filter_test.go index cf3e68f269..1fc4e375c8 100644 --- a/internal/topo/operator/filter_test.go +++ b/internal/topo/operator/filter_test.go @@ -1,4 +1,4 @@ -// Copyright 2021-2023 EMQ Technologies Co., Ltd. +// Copyright 2021-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,6 +21,8 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" + "github.com/lf-edge/ekuiper/internal/conf" "github.com/lf-edge/ekuiper/internal/topo/context" "github.com/lf-edge/ekuiper/internal/xsql" @@ -331,7 +333,7 @@ func TestFilterPlan_Apply(t *testing.T) { { sql: "SELECT abc FROM src1 WHERE f1 = \"v1\" GROUP BY TUMBLINGWINDOW(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -346,7 +348,7 @@ func TestFilterPlan_Apply(t *testing.T) { WindowRange: xsql.NewWindowRange(1541152486013, 1541152487013), }, result: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -361,7 +363,7 @@ func TestFilterPlan_Apply(t *testing.T) { { sql: "SELECT abc FROM src1 WHERE f1 = \"v8\" GROUP BY TUMBLINGWINDOW(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -381,19 +383,19 @@ func TestFilterPlan_Apply(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, @@ -403,13 +405,13 @@ func TestFilterPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, @@ -422,19 +424,19 @@ func TestFilterPlan_Apply(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, @@ -444,13 +446,13 @@ func TestFilterPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, @@ -463,19 +465,19 @@ func TestFilterPlan_Apply(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, @@ -606,17 +608,17 @@ func TestFilterPlan_Apply(t *testing.T) { contextLogger := conf.Log.WithField("rule", "TestFilerPlan_Apply") ctx := context.WithValue(context.Background(), context.LoggerKey, contextLogger) for i, tt := range tests { - stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse() - if err != nil { - t.Errorf("statement %d parse error %s", i, err) - break - } - fv, afv := xsql.NewFunctionValuersForOp(ctx) - pp := &FilterOp{Condition: stmt.Condition} - result := pp.Apply(ctx, tt.data, fv, afv) - if !reflect.DeepEqual(tt.result, result) { - t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, result) - } + t.Run(tt.sql, func(t *testing.T) { + stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse() + if err != nil { + t.Errorf("statement %d parse error %s", i, err) + return + } + fv, afv := xsql.NewFunctionValuersForOp(ctx) + pp := &FilterOp{Condition: stmt.Condition} + result := pp.Apply(ctx, tt.data, fv, afv) + assert.Equal(t, tt.result, result) + }) } } @@ -645,7 +647,7 @@ func TestFilterPlanError(t *testing.T) { { sql: "SELECT abc FROM src1 WHERE f1 = \"v1\" GROUP BY TUMBLINGWINDOW(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -668,7 +670,7 @@ func TestFilterPlanError(t *testing.T) { }, }, result: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -688,7 +690,7 @@ func TestFilterPlanError(t *testing.T) { { sql: "SELECT abc FROM src1 WHERE f1 = \"v8\" GROUP BY TUMBLINGWINDOW(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -708,19 +710,19 @@ func TestFilterPlanError(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": 50}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, diff --git a/internal/topo/operator/func_operator.go b/internal/topo/operator/func_operator.go index 72a827cd19..a733434a5b 100644 --- a/internal/topo/operator/func_operator.go +++ b/internal/topo/operator/func_operator.go @@ -1,4 +1,4 @@ -// Copyright 2022 EMQ Technologies Co., Ltd. +// Copyright 2022-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -33,14 +33,14 @@ func (p *FuncOp) Apply(ctx api.StreamContext, data interface{}, fv *xsql.Functio switch input := data.(type) { case error: return input - case xsql.TupleRow: + case xsql.Row: ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(input, fv)} result := ve.Eval(p.CallExpr) if e, ok := result.(error); ok { return e } input.Set(p.Name, result) - case xsql.SingleCollection: + case xsql.Collection: var err error if p.IsAgg { input.SetIsAgg(true) @@ -68,23 +68,6 @@ func (p *FuncOp) Apply(ctx api.StreamContext, data interface{}, fv *xsql.Functio if err != nil { return err } - case xsql.GroupedCollection: // The order is important, because single collection usually is also a groupedCollection - if !p.IsAgg { - return fmt.Errorf("FuncOp: GroupedCollection is not supported for non-aggregate function") - } - err := input.GroupRange(func(_ int, aggRow xsql.CollectionRow) (bool, error) { - afv.SetData(aggRow) - ve := &xsql.ValuerEval{Valuer: xsql.MultiAggregateValuer(aggRow, fv, aggRow, fv, afv, &xsql.WildcardValuer{Data: aggRow})} - result := ve.Eval(p.CallExpr) - if e, ok := result.(error); ok { - return false, e - } - aggRow.Set(p.Name, result) - return true, nil - }) - if err != nil { - return err - } default: return fmt.Errorf("run func error: invalid input %[1]T(%[1]v)", input) } diff --git a/internal/topo/operator/having_test.go b/internal/topo/operator/having_test.go index adb103345d..929bbf92aa 100644 --- a/internal/topo/operator/having_test.go +++ b/internal/topo/operator/having_test.go @@ -1,4 +1,4 @@ -// Copyright 2021-2022 EMQ Technologies Co., Ltd. +// Copyright 2021-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -35,7 +35,7 @@ func TestHavingPlan_Apply(t *testing.T) { { sql: `SELECT id1 FROM src1 HAVING avg(id1) > 1`, data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -50,7 +50,7 @@ func TestHavingPlan_Apply(t *testing.T) { WindowRange: xsql.NewWindowRange(1541152486013, 1541152487013), }, result: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -68,7 +68,7 @@ func TestHavingPlan_Apply(t *testing.T) { { sql: `SELECT id1 FROM src1 HAVING sum(id1) > 1`, data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -81,7 +81,7 @@ func TestHavingPlan_Apply(t *testing.T) { { sql: `SELECT id1 FROM src1 HAVING sum(id1) = 1`, data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -89,7 +89,7 @@ func TestHavingPlan_Apply(t *testing.T) { }, }, result: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -101,7 +101,7 @@ func TestHavingPlan_Apply(t *testing.T) { { sql: `SELECT id1 FROM src1 HAVING max(id1) > 10`, data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -113,7 +113,7 @@ func TestHavingPlan_Apply(t *testing.T) { { sql: `SELECT id1 FROM src1 HAVING max(id1) = 1`, data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -121,7 +121,7 @@ func TestHavingPlan_Apply(t *testing.T) { }, }, result: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -134,7 +134,7 @@ func TestHavingPlan_Apply(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -147,7 +147,7 @@ func TestHavingPlan_Apply(t *testing.T) { WindowRange: xsql.NewWindowRange(1541152486013, 1541152487013), }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}, @@ -160,7 +160,7 @@ func TestHavingPlan_Apply(t *testing.T) { result: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}, @@ -176,15 +176,15 @@ func TestHavingPlan_Apply(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 122.33}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 5, "a": 177.51}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 5, "color": "w2"}}, }, @@ -193,15 +193,15 @@ func TestHavingPlan_Apply(t *testing.T) { WindowRange: xsql.NewWindowRange(1541152486013, 1541152487013), }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 2, "a": 89.03}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 2, "color": "w1"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 4, "a": 14.6}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 4, "color": "w1"}}, }, @@ -214,15 +214,15 @@ func TestHavingPlan_Apply(t *testing.T) { result: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 122.33}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 5, "a": 177.51}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 5, "color": "w2"}}, }, @@ -262,7 +262,7 @@ func TestHavingPlanAlias_Apply(t *testing.T) { { sql: `SELECT avg(id1) as a FROM src1 HAVING a > 1`, data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1", "a": 8 / 3}, @@ -276,7 +276,7 @@ func TestHavingPlanAlias_Apply(t *testing.T) { }, }, result: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1", "a": 8 / 3}, @@ -293,7 +293,7 @@ func TestHavingPlanAlias_Apply(t *testing.T) { { sql: `SELECT sum(id1) as s FROM src1 HAVING s > 1`, data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1", "s": 1}, @@ -307,7 +307,7 @@ func TestHavingPlanAlias_Apply(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1", "c": 2}, @@ -319,7 +319,7 @@ func TestHavingPlanAlias_Apply(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2", "c": 1}, @@ -331,7 +331,7 @@ func TestHavingPlanAlias_Apply(t *testing.T) { result: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1", "c": 2}, @@ -350,15 +350,15 @@ func TestHavingPlanAlias_Apply(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 122.33, "c": 2}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 5, "a": 177.51}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 5, "color": "w2"}}, }, @@ -366,9 +366,9 @@ func TestHavingPlanAlias_Apply(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 2, "a": 89.03, "c": 1}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 2, "color": "w1"}}, }, @@ -380,15 +380,15 @@ func TestHavingPlanAlias_Apply(t *testing.T) { result: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 122.33, "c": 2}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 5, "a": 177.51}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 5, "color": "w2"}}, }, @@ -427,7 +427,7 @@ func TestHavingPlanError(t *testing.T) { { sql: `SELECT id1 FROM src1 HAVING avg(id1) > "str"`, data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -450,7 +450,7 @@ func TestHavingPlanError(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": 3}, @@ -462,7 +462,7 @@ func TestHavingPlanError(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}, diff --git a/internal/topo/operator/join_multi_test.go b/internal/topo/operator/join_multi_test.go index 3640859d40..5f44565afe 100644 --- a/internal/topo/operator/join_multi_test.go +++ b/internal/topo/operator/join_multi_test.go @@ -1,4 +1,4 @@ -// Copyright 2021-2022 EMQ Technologies Co., Ltd. +// Copyright 2021-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -35,7 +35,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2 left join src3 on src2.id2 = src3.id3", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -64,14 +64,14 @@ func TestMultiJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id3": 1, "f3": "x1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}}, }, }, @@ -82,7 +82,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2 inner join src3 on src2.id2 = src3.id3", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -112,7 +112,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id3": 1, "f3": "x1"}}, @@ -125,7 +125,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2 inner join src3 on src1.id1 = src3.id3", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -154,7 +154,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 5, "f1": "v5"}}, &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id3": 5, "f3": "x5"}}, }, @@ -165,7 +165,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2 full join src3 on src1.id1 = src3.id3", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -192,20 +192,20 @@ func TestMultiJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 5, "f1": "v5"}}, &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id3": 5, "f3": "x5"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id3": 2, "f3": "x1"}}, }, }, @@ -216,7 +216,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2 right join src3 on src2.id2 = src3.id3", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -244,7 +244,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id3": 1, "f3": "x1"}}, &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, @@ -252,7 +252,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id3": 5, "f3": "x5"}}, }, }, @@ -263,7 +263,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2 right join src3 on src2.id2 = src3.id3", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -292,7 +292,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id3": 1, "f3": "x1"}}, &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, @@ -300,7 +300,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id3": 1, "f3": "x1"}}, &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w3"}}, @@ -308,7 +308,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id3": 1, "f3": "x1"}}, &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v3"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, @@ -316,7 +316,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id3": 1, "f3": "x1"}}, &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v3"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w3"}}, @@ -324,7 +324,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id3": 5, "f3": "x5"}}, }, }, @@ -335,7 +335,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2 cross join src3", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -364,28 +364,28 @@ func TestMultiJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id3": 2, "f3": "x1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id3": 5, "f3": "x5"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 5, "f1": "v5"}}, &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id3": 2, "f3": "x1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 5, "f1": "v5"}}, &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id3": 5, "f3": "x5"}}, }, @@ -396,7 +396,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 inner join src2 on src1.id = src2.id inner join src3 on src1.id = src3.id", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}, @@ -433,14 +433,14 @@ func TestMultiJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "f2": "w1"}}, &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 1, "f3": "x1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "f2": "w1"}}, &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 1, "f3": "x3"}}, @@ -453,7 +453,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { // 9 sql: "SELECT id1 FROM src1 inner join src2 on src1.id = src2.id right join src3 on src1.id = src3.id", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}, @@ -492,21 +492,21 @@ func TestMultiJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 1, "f3": "x1"}}, &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 1, "f3": "x3"}}, &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 5, "f3": "x5"}}, }, }, @@ -517,7 +517,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { // 10 sql: "SELECT id1 FROM src1 inner join src2 on src1.id * 10 = src2.id right join src3 on src1.id = src3.id", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}, @@ -555,17 +555,17 @@ func TestMultiJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 1, "f3": "x1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 1, "f3": "x3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 5, "f3": "x5"}}, }, }, @@ -576,7 +576,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { // 11 sql: "SELECT id1 FROM src1 full join src2 on src1.id = src2.id inner join src3 on src1.id = src3.id", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}, @@ -618,14 +618,14 @@ func TestMultiJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "f2": "w1"}}, &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 1, "f3": "x1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "f2": "w1"}}, &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 1, "f3": "x3"}}, @@ -638,7 +638,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { // 12 sql: "SELECT id1 FROM src1 full join src2 on src1.id = src2.id right join src3 on src1.id = src3.id", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}, @@ -680,21 +680,21 @@ func TestMultiJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 1, "f3": "x1"}}, &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 1, "f3": "x3"}}, &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 5, "f3": "x5"}}, }, }, @@ -705,7 +705,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { // 13 sql: "SELECT id1 FROM src1 full join src2 on src1.id = src2.id full join src3 on src1.id = src3.id", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}, @@ -725,17 +725,17 @@ func TestMultiJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 5, "f3": "x5"}}, }, }, @@ -746,7 +746,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { // 14 sql: "SELECT id1 FROM src1 right join src2 on src1.id = src2.id right join src3 on src1.id = src3.id", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}, @@ -788,21 +788,21 @@ func TestMultiJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 1, "f3": "x1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "f2": "w1"}}, &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 1, "f3": "x3"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "f2": "w1"}}, &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 5, "f3": "x5"}}, }, }, @@ -813,7 +813,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { // 15 sql: "SELECT id1 FROM src1 right join src2 on src1.id = src2.id right join src3 on src1.id = src3.id", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}, @@ -847,7 +847,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { // 16 sql: "SELECT id1 FROM src1 right join src2 on src1.id = src2.id right join src3 on src1.id = src3.id right join src4 on src4.id = src3.id ", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}, @@ -896,7 +896,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src4", Message: xsql.Message{"id": 1, "f4": "x4"}}, &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 1, "f3": "x1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "f2": "w1"}}, @@ -904,7 +904,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src4", Message: xsql.Message{"id": 2, "f4": "x4"}}, }, }, @@ -915,7 +915,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { // 17 sql: "SELECT id1 FROM src1 right join src2 on src1.id = src2.id right join src3 on src1.id = src3.id cross join src4", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}, @@ -958,7 +958,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { // 18 sql: "SELECT id1 FROM src1 cross join src2 left join src3 on src1.id = src3.id", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src2", Message: xsql.Message{"id": 1, "f2": "w1"}, @@ -988,7 +988,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { // 19 sql: "SELECT id1 FROM src1 full join src2 on src1.id = src2.id full join src3 on src1.id = src3.id", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}, @@ -1008,7 +1008,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "f2": "w1"}}, &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 1, "f3": "x1"}}, @@ -1020,7 +1020,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { // 20 sql: "SELECT id1 FROM src1 full join src2 on src1.id = src2.id full join src3 on src1.id = src3.id", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src2", Message: xsql.Message{"id": 1, "f2": "w1"}, @@ -1035,12 +1035,12 @@ func TestMultiJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 1, "f3": "x1"}}, }, }, @@ -1050,7 +1050,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { // 21 sql: "SELECT id1 FROM src1 full join src2 on src1.id = src2.id full join src3 on src1.id = src3.id", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}, @@ -1065,7 +1065,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "f2": "w1"}}, }, @@ -1077,7 +1077,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { { // 22 sql: "SELECT id1 FROM src1 full join src2 on src1.id = src2.id full join src3 on src1.id = src3.id", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}, @@ -1092,7 +1092,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src3", Message: xsql.Message{"id": 1, "f3": "x1"}}, }, diff --git a/internal/topo/operator/join_operator.go b/internal/topo/operator/join_operator.go index 9684600d38..c5d8a0ac34 100644 --- a/internal/topo/operator/join_operator.go +++ b/internal/topo/operator/join_operator.go @@ -1,4 +1,4 @@ -// Copyright 2021-2022 EMQ Technologies Co., Ltd. +// Copyright 2021-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -28,16 +28,18 @@ type JoinOp struct { Joins ast.Joins } -// Apply -// input: MergedCollection, the Row must be a Tuple -// output: Collection +// Apply JoinOp to join two streams. If running in continuous query, the inner join will always return empty result because there is only one stream data. func (jp *JoinOp) Apply(ctx api.StreamContext, data interface{}, fv *xsql.FunctionValuer, _ *xsql.AggregateFunctionValuer) interface{} { log := ctx.GetLogger() - var input xsql.MergedCollection + var input xsql.Collection switch v := data.(type) { case error: return v - case xsql.MergedCollection: + case xsql.Row: + input = &xsql.WindowTuples{ + Content: []xsql.Row{v}, + } + case xsql.Collection: input = v log.Debugf("join plan receive %v", data) default: @@ -108,7 +110,7 @@ func (jp *JoinOp) getStreamNames(join *ast.Join) ([]string, error) { return srcs, nil } -func (jp *JoinOp) evalSet(input xsql.MergedCollection, join ast.Join, fv *xsql.FunctionValuer) (*xsql.JoinTuples, error) { +func (jp *JoinOp) evalSet(input xsql.Collection, join ast.Join, fv *xsql.FunctionValuer) (*xsql.JoinTuples, error) { var leftStream, rightStream string if join.JoinType != ast.CROSS_JOIN { @@ -132,7 +134,7 @@ func (jp *JoinOp) evalSet(input xsql.MergedCollection, join ast.Join, fv *xsql.F } } - var lefts, rights []xsql.TupleRow + var lefts, rights []xsql.Row lefts = input.GetBySrc(leftStream) rights = input.GetBySrc(rightStream) @@ -215,14 +217,14 @@ func evalOn(join ast.Join, ve *xsql.ValuerEval, left interface{}, right xsql.Row return result } -func (jp *JoinOp) evalSetWithRightJoin(input xsql.MergedCollection, join ast.Join, excludeJoint bool, fv *xsql.FunctionValuer) (*xsql.JoinTuples, error) { +func (jp *JoinOp) evalSetWithRightJoin(input xsql.Collection, join ast.Join, excludeJoint bool, fv *xsql.FunctionValuer) (*xsql.JoinTuples, error) { streams, err := jp.getStreamNames(&join) if err != nil { return nil, err } leftStream := streams[0] rightStream := streams[1] - var lefts, rights []xsql.TupleRow + var lefts, rights []xsql.Row lefts = input.GetBySrc(leftStream) rights = input.GetBySrc(rightStream) @@ -267,7 +269,7 @@ func (jp *JoinOp) evalSetWithRightJoin(input xsql.MergedCollection, join ast.Joi return sets, nil } -func (jp *JoinOp) evalJoinSets(set *xsql.JoinTuples, input xsql.MergedCollection, join ast.Join, fv *xsql.FunctionValuer) (interface{}, error) { +func (jp *JoinOp) evalJoinSets(set *xsql.JoinTuples, input xsql.Collection, join ast.Join, fv *xsql.FunctionValuer) (interface{}, error) { var rightStream string if join.Alias == "" { rightStream = join.Name @@ -338,7 +340,7 @@ func (jp *JoinOp) evalJoinSets(set *xsql.JoinTuples, input xsql.MergedCollection return newSets, nil } -func (jp *JoinOp) evalRightJoinSets(set *xsql.JoinTuples, input xsql.MergedCollection, join ast.Join, excludeJoint bool, fv *xsql.FunctionValuer) (*xsql.JoinTuples, error) { +func (jp *JoinOp) evalRightJoinSets(set *xsql.JoinTuples, input xsql.Collection, join ast.Join, excludeJoint bool, fv *xsql.FunctionValuer) (*xsql.JoinTuples, error) { var rightStream string if join.Alias == "" { rightStream = join.Name diff --git a/internal/topo/operator/join_test.go b/internal/topo/operator/join_test.go index 9fe1a73909..2bef79186a 100644 --- a/internal/topo/operator/join_test.go +++ b/internal/topo/operator/join_test.go @@ -1,4 +1,4 @@ -// Copyright 2021-2022 EMQ Technologies Co., Ltd. +// Copyright 2021-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -47,7 +47,7 @@ func TestLeftJoinPlan_Apply(t *testing.T) { { // 0 sql: "SELECT id1 FROM src1 left join src2 on id1 = id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -74,19 +74,19 @@ func TestLeftJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}}, }, }, @@ -97,7 +97,7 @@ func TestLeftJoinPlan_Apply(t *testing.T) { { // 1 sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -124,18 +124,18 @@ func TestLeftJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}}, }, }, @@ -145,7 +145,7 @@ func TestLeftJoinPlan_Apply(t *testing.T) { { // 2 sql: "SELECT id1 FROM src1 left join src2 on src1.ts = src2.ts", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1", "ts": cast.TimeFromUnixMilli(1568854515000)}, @@ -172,19 +172,19 @@ func TestLeftJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1", "ts": cast.TimeFromUnixMilli(1568854515000)}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1", "ts": cast.TimeFromUnixMilli(1568854515000)}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2", "ts": cast.TimeFromUnixMilli(1568854525000)}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2", "ts": cast.TimeFromUnixMilli(1568854525000)}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3", "ts": cast.TimeFromUnixMilli(1568854535000)}}, }, }, @@ -194,7 +194,7 @@ func TestLeftJoinPlan_Apply(t *testing.T) { { // 3 sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -224,7 +224,7 @@ func TestLeftJoinPlan_Apply(t *testing.T) { { // 4 sql: "SELECT id1 FROM src1 As s1 left join src2 as s2 on s1.id1 = s2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "s1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -250,19 +250,19 @@ func TestLeftJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "s1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "s2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "s1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "s2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "s1", Message: xsql.Message{"id1": 3, "f1": "v3"}}, }, }, @@ -273,7 +273,7 @@ func TestLeftJoinPlan_Apply(t *testing.T) { { // 5 sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -291,13 +291,13 @@ func TestLeftJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w2"}}, }, }, @@ -308,7 +308,7 @@ func TestLeftJoinPlan_Apply(t *testing.T) { { // 6 sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -324,17 +324,17 @@ func TestLeftJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}}, }, }, @@ -345,7 +345,7 @@ func TestLeftJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -361,17 +361,17 @@ func TestLeftJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}}, }, }, @@ -382,7 +382,7 @@ func TestLeftJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}, @@ -397,7 +397,7 @@ func TestLeftJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}, @@ -413,7 +413,7 @@ func TestLeftJoinPlan_Apply(t *testing.T) { { // 10 select id1 FROM src1 left join src2 on null = null sql: "SELECT id1 FROM src1 left join src2 on src1.id2 = src2.id1", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f2": "w1"}, @@ -431,13 +431,13 @@ func TestLeftJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f2": "w1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f2": "w1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, @@ -449,7 +449,7 @@ func TestLeftJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 left join src2 on src1.id1*2 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -476,19 +476,19 @@ func TestLeftJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}}, }, }, @@ -499,7 +499,7 @@ func TestLeftJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2*2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -526,18 +526,18 @@ func TestLeftJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}}, }, }, @@ -548,7 +548,7 @@ func TestLeftJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 left join src2 on src1.f1->cid = src2.f2->cid", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": str2Map(`{"cid" : 1, "name" : "tom1"}`)}, @@ -575,19 +575,19 @@ func TestLeftJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": str2Map(`{"cid" : 1, "name" : "tom1"}`)}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": str2Map(`{"cid" : 1, "name" : "tom2"}`)}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": str2Map(`{"cid" : 2, "name" : "mike1"}`)}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": str2Map(`{"cid" : 2, "name" : "mike2"}`)}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": str2Map(`{"cid" : 3, "name" : "alice1"}`)}}, }, }, @@ -598,7 +598,7 @@ func TestLeftJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1, mqtt(src1.topic) AS a, mqtt(src2.topic) as b FROM src1 left join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -615,7 +615,7 @@ func TestLeftJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, Metadata: xsql.Metadata{"topic": "devices/type1/device001"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}, Metadata: xsql.Metadata{"topic": "devices/type2/device001"}}, }, @@ -626,7 +626,7 @@ func TestLeftJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -659,43 +659,43 @@ func TestLeftJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 3, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 3, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v4"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 3, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v4"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 3, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 4, "f1": "v5"}}, }, }, @@ -705,7 +705,7 @@ func TestLeftJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 full join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -732,25 +732,25 @@ func TestLeftJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}}, }, }, @@ -791,7 +791,7 @@ func TestInnerJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -818,13 +818,13 @@ func TestInnerJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, @@ -835,7 +835,7 @@ func TestInnerJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -862,7 +862,7 @@ func TestInnerJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, @@ -873,7 +873,7 @@ func TestInnerJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 As s1 inner join src2 as s2 on s1.id1 = s2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "s1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -899,13 +899,13 @@ func TestInnerJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "s1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "s2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "s1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "s2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, @@ -917,7 +917,7 @@ func TestInnerJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -935,13 +935,13 @@ func TestInnerJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w2"}}, }, @@ -953,7 +953,7 @@ func TestInnerJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -975,25 +975,25 @@ func TestInnerJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w2"}}, }, @@ -1005,7 +1005,7 @@ func TestInnerJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -1024,7 +1024,7 @@ func TestInnerJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -1043,7 +1043,7 @@ func TestInnerJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}, @@ -1059,7 +1059,7 @@ func TestInnerJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}, @@ -1075,7 +1075,7 @@ func TestInnerJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 inner join src2 on src1.id1*2 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -1102,13 +1102,13 @@ func TestInnerJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, @@ -1120,7 +1120,7 @@ func TestInnerJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2*2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -1147,7 +1147,7 @@ func TestInnerJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, @@ -1159,7 +1159,7 @@ func TestInnerJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 inner join src2 on src1.f1->cid = src2.f2->cid", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": str2Map(`{"cid" : 1, "name" : "tom1"}`)}, @@ -1186,13 +1186,13 @@ func TestInnerJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": str2Map(`{"cid" : 1, "name" : "tom1"}`)}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": str2Map(`{"cid" : 1, "name" : "tom2"}`)}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": str2Map(`{"cid" : 2, "name" : "mike1"}`)}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": str2Map(`{"cid" : 2, "name" : "mike2"}`)}}, }, @@ -1203,7 +1203,7 @@ func TestInnerJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 As s1 inner join src2 as s2 on s1.id1 = s2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "s1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -1229,19 +1229,19 @@ func TestInnerJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "s1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "s2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "s1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "s2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "s1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "s2", Message: xsql.Message{"id2": 2, "f2": "w3"}}, }, @@ -1283,7 +1283,7 @@ func TestRightJoinPlan_Apply(t *testing.T) { { // 0 sql: "SELECT id1 FROM src1 right join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -1310,21 +1310,21 @@ func TestRightJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, @@ -1334,7 +1334,7 @@ func TestRightJoinPlan_Apply(t *testing.T) { { // 1 sql: "SELECT id1 FROM src1 right join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -1361,26 +1361,26 @@ func TestRightJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, @@ -1390,7 +1390,7 @@ func TestRightJoinPlan_Apply(t *testing.T) { { // 2 sql: "SELECT id1 FROM src1 right join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -1408,13 +1408,13 @@ func TestRightJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w2"}}, &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, }, @@ -1425,7 +1425,7 @@ func TestRightJoinPlan_Apply(t *testing.T) { { // 3 sql: "SELECT id1 FROM src1 right join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -1443,7 +1443,7 @@ func TestRightJoinPlan_Apply(t *testing.T) { { // 4 sql: "SELECT id1 FROM src1 right join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}, @@ -1459,19 +1459,19 @@ func TestRightJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, @@ -1511,7 +1511,7 @@ func TestFullJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 full join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -1541,33 +1541,33 @@ func TestFullJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w4"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, @@ -1578,7 +1578,7 @@ func TestFullJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 full join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -1605,27 +1605,27 @@ func TestFullJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, @@ -1636,7 +1636,7 @@ func TestFullJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 full join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -1652,19 +1652,19 @@ func TestFullJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}}, }, }, @@ -1675,7 +1675,7 @@ func TestFullJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 full join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -1702,32 +1702,32 @@ func TestFullJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 5, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 6, "f2": "w3"}}, }, }, @@ -1738,7 +1738,7 @@ func TestFullJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 full join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}, @@ -1754,19 +1754,19 @@ func TestFullJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, @@ -1806,7 +1806,7 @@ func TestCrossJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 cross join src2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -1833,57 +1833,57 @@ func TestCrossJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, @@ -1895,7 +1895,7 @@ func TestCrossJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 cross join src2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -1913,13 +1913,13 @@ func TestCrossJoinPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w2"}}, }, }, @@ -1929,7 +1929,7 @@ func TestCrossJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 cross join src2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}, @@ -1944,7 +1944,7 @@ func TestCrossJoinPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src2 cross join src1", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}, @@ -1993,7 +1993,7 @@ func TestCrossJoinPlanError(t *testing.T) { }, { sql: "SELECT id1 FROM src1 full join src2 on src1.id1 = src2.id2", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, diff --git a/internal/topo/operator/math_func_test.go b/internal/topo/operator/math_func_test.go index 8c1fd75b39..c0143924fb 100644 --- a/internal/topo/operator/math_func_test.go +++ b/internal/topo/operator/math_func_test.go @@ -1,4 +1,4 @@ -// Copyright 2022-2023 EMQ Technologies Co., Ltd. +// Copyright 2022-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,10 +16,11 @@ package operator import ( "fmt" - "reflect" "strings" "testing" + "github.com/stretchr/testify/assert" + "github.com/lf-edge/ekuiper/internal/conf" "github.com/lf-edge/ekuiper/internal/topo/context" "github.com/lf-edge/ekuiper/internal/xsql" @@ -483,24 +484,21 @@ func TestMathAndConversionFunc_Apply1(t *testing.T) { contextLogger := conf.Log.WithField("rule", "TestMathAndConversionFunc_Apply1") ctx := context.WithValue(context.Background(), context.LoggerKey, contextLogger) for i, tt := range tests { - stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse() - if err != nil && tt.result == nil { - continue - } else if err != nil && tt.result != nil { - t.Errorf("%d: found error %q", i, err) - continue - } - pp := &ProjectOp{SendMeta: true, IsAggregate: xsql.WithAggFields(stmt)} - parseStmt(pp, stmt.Fields) - fv, afv := xsql.NewFunctionValuersForOp(nil) - opResult := pp.Apply(ctx, tt.data, fv, afv) - result, err := parseResult(opResult, pp.IsAggregate) - if err != nil { - t.Errorf("parse result error： %s", err) - continue - } - if !reflect.DeepEqual(tt.result, result) { - t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, result) - } + t.Run(tt.sql, func(t *testing.T) { + stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse() + if err != nil && tt.result == nil { + return + } else if err != nil && tt.result != nil { + t.Errorf("%d: found error %q", i, err) + return + } + pp := &ProjectOp{SendMeta: true, IsAggregate: xsql.WithAggFields(stmt)} + parseStmt(pp, stmt.Fields) + fv, afv := xsql.NewFunctionValuersForOp(nil) + opResult := pp.Apply(ctx, tt.data, fv, afv) + result, err := parseResult(opResult, pp.IsAggregate) + assert.NoError(t, err) + assert.Equal(t, tt.result, result) + }) } } diff --git a/internal/topo/operator/misc_func_test.go b/internal/topo/operator/misc_func_test.go index 0850fce877..9fd9392bd2 100644 --- a/internal/topo/operator/misc_func_test.go +++ b/internal/topo/operator/misc_func_test.go @@ -1,4 +1,4 @@ -// Copyright 2022-2023 EMQ Technologies Co., Ltd. +// Copyright 2022-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -309,7 +309,7 @@ func TestMqttFunc_Apply2(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": "1", "f1": "v1"}, Metadata: xsql.Metadata{"topic": "devices/type1/device001"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": "1", "f2": "w1"}, Metadata: xsql.Metadata{"topic": "devices/type2/device001"}}, }, diff --git a/internal/topo/operator/order_operator.go b/internal/topo/operator/order_operator.go index e8fea52686..3e17c86424 100644 --- a/internal/topo/operator/order_operator.go +++ b/internal/topo/operator/order_operator.go @@ -1,4 +1,4 @@ -// Copyright 2021-2022 EMQ Technologies Co., Ltd. +// Copyright 2021-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -26,10 +26,6 @@ type OrderOp struct { SortFields ast.SortFields } -/** - * input: *xsql.Tuple from preprocessor | xsql.WindowTuples from windowOp | xsql.JoinTuples from joinOp - * output: *xsql.Tuple | xsql.WindowTuples | xsql.JoinTuples - */ func (p *OrderOp) Apply(ctx api.StreamContext, data interface{}, fv *xsql.FunctionValuer, afv *xsql.AggregateFunctionValuer) interface{} { log := ctx.GetLogger() log.Debugf("order plan receive %v", data) @@ -37,7 +33,7 @@ func (p *OrderOp) Apply(ctx api.StreamContext, data interface{}, fv *xsql.Functi switch input := data.(type) { case error: return input - case xsql.TupleRow: + case xsql.Row: return input case xsql.SortingData: if err := sorter.Sort(input); err != nil { diff --git a/internal/topo/operator/order_test.go b/internal/topo/operator/order_test.go index 0f5b799109..28a98bc83b 100644 --- a/internal/topo/operator/order_test.go +++ b/internal/topo/operator/order_test.go @@ -1,4 +1,4 @@ -// Copyright 2021-2023 EMQ Technologies Co., Ltd. +// Copyright 2021-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,10 +17,11 @@ package operator import ( "errors" "fmt" - "reflect" "strings" "testing" + "github.com/stretchr/testify/assert" + "github.com/lf-edge/ekuiper/internal/conf" "github.com/lf-edge/ekuiper/internal/topo/context" "github.com/lf-edge/ekuiper/internal/xsql" @@ -69,7 +70,7 @@ func TestOrderPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 WHERE f1 = \"v1\" GROUP BY TUMBLINGWINDOW(ss, 10) ORDER BY id1 DESC", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -84,7 +85,7 @@ func TestOrderPlan_Apply(t *testing.T) { }, result: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}, @@ -102,7 +103,7 @@ func TestOrderPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 WHERE f1 = \"v1\" GROUP BY TUMBLINGWINDOW(ss, 10) ORDER BY id1 DESC", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -117,7 +118,7 @@ func TestOrderPlan_Apply(t *testing.T) { WindowRange: xsql.NewWindowRange(1541152486013, 1541152487013), }, result: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}, @@ -135,7 +136,7 @@ func TestOrderPlan_Apply(t *testing.T) { { sql: "SELECT id1 FROM src1 WHERE f1 = \"v1\" GROUP BY TUMBLINGWINDOW(ss, 10) ORDER BY id1 DESC", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -153,7 +154,7 @@ func TestOrderPlan_Apply(t *testing.T) { { sql: "SELECT * FROM src1 WHERE f1 = \"v1\" GROUP BY TUMBLINGWINDOW(ss, 10) ORDER BY f1, id1 DESC", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -167,7 +168,7 @@ func TestOrderPlan_Apply(t *testing.T) { }, }, result: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}, @@ -184,7 +185,7 @@ func TestOrderPlan_Apply(t *testing.T) { { sql: "SELECT * FROM src1 GROUP BY TUMBLINGWINDOW(ss, 10) ORDER BY ts DESC", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1", "ts": cast.TimeFromUnixMilli(1568854515000)}, @@ -198,7 +199,7 @@ func TestOrderPlan_Apply(t *testing.T) { }, }, result: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1", "ts": cast.TimeFromUnixMilli(1568854535000)}, @@ -218,19 +219,19 @@ func TestOrderPlan_Apply(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, @@ -239,18 +240,18 @@ func TestOrderPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, @@ -263,19 +264,19 @@ func TestOrderPlan_Apply(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, @@ -284,19 +285,19 @@ func TestOrderPlan_Apply(t *testing.T) { result: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, @@ -309,7 +310,7 @@ func TestOrderPlan_Apply(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "tbl", Message: xsql.Message{ @@ -324,7 +325,7 @@ func TestOrderPlan_Apply(t *testing.T) { result: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "tbl", Message: xsql.Message{ @@ -342,7 +343,7 @@ func TestOrderPlan_Apply(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -354,7 +355,7 @@ func TestOrderPlan_Apply(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}, @@ -366,7 +367,7 @@ func TestOrderPlan_Apply(t *testing.T) { result: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}, @@ -374,7 +375,7 @@ func TestOrderPlan_Apply(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -393,7 +394,7 @@ func TestOrderPlan_Apply(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1", "c": 2}, @@ -405,7 +406,7 @@ func TestOrderPlan_Apply(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2", "c": 1}, @@ -417,7 +418,7 @@ func TestOrderPlan_Apply(t *testing.T) { result: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2", "c": 1}, @@ -425,7 +426,7 @@ func TestOrderPlan_Apply(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1", "c": 2}, @@ -444,9 +445,9 @@ func TestOrderPlan_Apply(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, @@ -455,9 +456,9 @@ func TestOrderPlan_Apply(t *testing.T) { WindowRange: xsql.NewWindowRange(1541152486013, 1541152487013), }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, @@ -466,9 +467,9 @@ func TestOrderPlan_Apply(t *testing.T) { WindowRange: xsql.NewWindowRange(1541152486013, 1541152487013), }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, @@ -480,9 +481,9 @@ func TestOrderPlan_Apply(t *testing.T) { result: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, @@ -491,9 +492,9 @@ func TestOrderPlan_Apply(t *testing.T) { WindowRange: xsql.NewWindowRange(1541152486013, 1541152487013), }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, @@ -502,9 +503,9 @@ func TestOrderPlan_Apply(t *testing.T) { WindowRange: xsql.NewWindowRange(1541152486013, 1541152487013), }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, @@ -517,7 +518,7 @@ func TestOrderPlan_Apply(t *testing.T) { { sql: "SELECT a FROM demo GROUP BY a, TUMBLINGWINDOW(ss, 10) ORDER BY a ASC", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "demo", Message: xsql.Message{"a": 4}, @@ -571,7 +572,7 @@ func TestOrderPlan_Apply(t *testing.T) { }, }, result: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "demo", Message: xsql.Message{"a": 1}, @@ -630,18 +631,16 @@ func TestOrderPlan_Apply(t *testing.T) { fmt.Printf("The test bucket size is %d.\n\n", len(tests)) contextLogger := conf.Log.WithField("rule", "TestOrderPlan_Apply") ctx := context.WithValue(context.Background(), context.LoggerKey, contextLogger) - for i, tt := range tests { - stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse() - if err != nil { - t.Errorf("statement parse error %s", err) - break - } + for _, tt := range tests { + t.Run(tt.sql, func(t *testing.T) { + stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse() + assert.NoError(t, err) + + pp := &OrderOp{SortFields: stmt.SortFields} + fv, afv := xsql.NewFunctionValuersForOp(nil) + result := pp.Apply(ctx, tt.data, fv, afv) + assert.Equal(t, tt.result, result) + }) - pp := &OrderOp{SortFields: stmt.SortFields} - fv, afv := xsql.NewFunctionValuersForOp(nil) - result := pp.Apply(ctx, tt.data, fv, afv) - if !reflect.DeepEqual(tt.result, result) { - t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, result) - } } } diff --git a/internal/topo/operator/project_operator.go b/internal/topo/operator/project_operator.go index f77a4cbc23..f65705babf 100644 --- a/internal/topo/operator/project_operator.go +++ b/internal/topo/operator/project_operator.go @@ -1,4 +1,4 @@ -// Copyright 2022-2023 EMQ Technologies Co., Ltd. +// Copyright 2022-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -34,7 +34,7 @@ type ProjectOp struct { WildcardEmitters map[string]bool AliasFields ast.Fields ExprFields ast.Fields - IsAggregate bool + IsAggregate bool // Whether the project is used in an aggregate context. This is set by planner by analyzing the SQL query EnableLimit bool LimitCount int @@ -53,49 +53,44 @@ func (pp *ProjectOp) Apply(ctx api.StreamContext, data interface{}, fv *xsql.Fun log := ctx.GetLogger() log.Debugf("project plan receive %v", data) if pp.LimitCount == 0 && pp.EnableLimit { - return []xsql.TupleRow{} + return []xsql.Row{} } switch input := data.(type) { case error: return input - case *xsql.Tuple: - ve := pp.getVE(input, input, nil, fv, afv) + case xsql.Row: + ve := pp.getRowVE(input, nil, fv, afv) if err := pp.project(input, ve); err != nil { return fmt.Errorf("run Select error: %s", err) } else { - if pp.SendMeta && input.Metadata != nil { - input.Set(message.MetaKey, input.Metadata) + if pp.SendMeta { + if md, ok := input.(xsql.MetaData); ok { + metadata := md.MetaData() + if metadata != nil { + input.Set(message.MetaKey, md.MetaData()) + } + } } } - case xsql.SingleCollection: + case xsql.Collection: var err error if pp.IsAggregate { input.SetIsAgg(true) - err = input.GroupRange(func(_ int, aggRow xsql.CollectionRow) (bool, error) { + err = input.GroupRange(func(i int, aggRow xsql.CollectionRow) (bool, error) { + if pp.EnableLimit && pp.LimitCount > 0 && i >= pp.LimitCount { + return false, nil + } ve := pp.getVE(aggRow, aggRow, input.GetWindowRange(), fv, afv) if err := pp.project(aggRow, ve); err != nil { return false, fmt.Errorf("run Select error: %s", err) } return true, nil }) - if pp.EnableLimit && pp.LimitCount > 0 && input.Len() > pp.LimitCount { - var sel []int - sel = make([]int, pp.LimitCount, pp.LimitCount) - for i := 0; i < pp.LimitCount; i++ { - sel[i] = i - } - input = input.Filter(sel).(xsql.SingleCollection) - } } else { - if pp.EnableLimit && pp.LimitCount > 0 && input.Len() > pp.LimitCount { - var sel []int - sel = make([]int, pp.LimitCount, pp.LimitCount) - for i := 0; i < pp.LimitCount; i++ { - sel[i] = i + err = input.RangeSet(func(i int, row xsql.Row) (bool, error) { + if pp.EnableLimit && pp.LimitCount > 0 && i >= pp.LimitCount { + return false, nil } - input = input.Filter(sel).(xsql.SingleCollection) - } - err = input.RangeSet(func(_ int, row xsql.Row) (bool, error) { aggData, ok := input.(xsql.AggregateData) if !ok { return false, fmt.Errorf("unexpected type, cannot find aggregate data") @@ -110,32 +105,13 @@ func (pp *ProjectOp) Apply(ctx api.StreamContext, data interface{}, fv *xsql.Fun if err != nil { return err } - case xsql.GroupedCollection: // The order is important, because single collection usually is also a groupedCollection - if pp.EnableLimit && pp.LimitCount > 0 && input.Len() > pp.LimitCount { - var sel []int - sel = make([]int, pp.LimitCount, pp.LimitCount) - for i := 0; i < pp.LimitCount; i++ { - sel[i] = i - } - input = input.Filter(sel).(xsql.GroupedCollection) - } - err := input.GroupRange(func(_ int, aggRow xsql.CollectionRow) (bool, error) { - ve := pp.getVE(aggRow, aggRow, input.GetWindowRange(), fv, afv) - if err := pp.project(aggRow, ve); err != nil { - return false, fmt.Errorf("run Select error: %s", err) - } - return true, nil - }) - if err != nil { - return err - } default: return fmt.Errorf("run Select error: invalid input %[1]T(%[1]v)", input) } return data } -func (pp *ProjectOp) getVE(tuple xsql.Row, agg xsql.AggregateData, wr *xsql.WindowRange, fv *xsql.FunctionValuer, afv *xsql.AggregateFunctionValuer) *xsql.ValuerEval { +func (pp *ProjectOp) getVE(tuple xsql.RawRow, agg xsql.AggregateData, wr *xsql.WindowRange, fv *xsql.FunctionValuer, afv *xsql.AggregateFunctionValuer) *xsql.ValuerEval { afv.SetData(agg) if pp.IsAggregate { return &xsql.ValuerEval{Valuer: xsql.MultiAggregateValuer(agg, fv, tuple, fv, afv, &xsql.WildcardValuer{Data: tuple})} @@ -147,7 +123,15 @@ func (pp *ProjectOp) getVE(tuple xsql.Row, agg xsql.AggregateData, wr *xsql.Wind } } -func (pp *ProjectOp) project(row xsql.Row, ve *xsql.ValuerEval) error { +func (pp *ProjectOp) getRowVE(tuple xsql.Row, wr *xsql.WindowRange, fv *xsql.FunctionValuer, afv *xsql.AggregateFunctionValuer) *xsql.ValuerEval { + if ag, ok := tuple.(xsql.AggregateData); ok { + return pp.getVE(tuple, ag, wr, fv, afv) + } else { + return pp.getVE(tuple, nil, wr, fv, afv) + } +} + +func (pp *ProjectOp) project(row xsql.RawRow, ve *xsql.ValuerEval) error { // Calculate all fields then pick the needed ones // To make sure all calculations are run with the same context (e.g. alias values) // Do not set value during calculations diff --git a/internal/topo/operator/project_test.go b/internal/topo/operator/project_test.go index d8c4f308c2..ff2080cd64 100644 --- a/internal/topo/operator/project_test.go +++ b/internal/topo/operator/project_test.go @@ -1,4 +1,4 @@ -// Copyright 2022-2023 EMQ Technologies Co., Ltd. +// Copyright 2022-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -60,7 +60,7 @@ func parseStmt(p *ProjectOp, fields ast.Fields) { func parseResult(opResult interface{}, aggregate bool) (result []map[string]interface{}, err error) { switch rt := opResult.(type) { - case xsql.TupleRow: + case xsql.Row: result = append(result, rt.ToMap()) case xsql.Collection: result = rt.ToMaps() @@ -733,7 +733,7 @@ func TestProjectPlan_MultiInput(t *testing.T) { { sql: "SELECT id1 FROM src1 WHERE f1 = \"v1\" GROUP BY TUMBLINGWINDOW(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -758,7 +758,7 @@ func TestProjectPlan_MultiInput(t *testing.T) { { sql: "SELECT id1 FROM src1 WHERE f1 = \"v1\" GROUP BY TUMBLINGWINDOW(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -782,7 +782,7 @@ func TestProjectPlan_MultiInput(t *testing.T) { { sql: "SELECT * FROM src1 WHERE f1 = \"v1\" GROUP BY TUMBLINGWINDOW(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -810,7 +810,7 @@ func TestProjectPlan_MultiInput(t *testing.T) { { sql: "SELECT * FROM src1 WHERE f1 = \"v1\" GROUP BY TUMBLINGWINDOW(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -838,7 +838,7 @@ func TestProjectPlan_MultiInput(t *testing.T) { { sql: "SELECT src1.* FROM src1 WHERE f1 = \"v1\" GROUP BY TUMBLINGWINDOW(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -868,19 +868,19 @@ func TestProjectPlan_MultiInput(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, @@ -900,19 +900,19 @@ func TestProjectPlan_MultiInput(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id2": 3, "f1": "v1"}}, }, }, @@ -930,7 +930,7 @@ func TestProjectPlan_MultiInput(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "tbl", Message: xsql.Message{ @@ -952,7 +952,7 @@ func TestProjectPlan_MultiInput(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "tbl", Message: xsql.Message{ @@ -971,7 +971,7 @@ func TestProjectPlan_MultiInput(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -983,7 +983,7 @@ func TestProjectPlan_MultiInput(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}, @@ -1004,7 +1004,7 @@ func TestProjectPlan_MultiInput(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -1016,7 +1016,7 @@ func TestProjectPlan_MultiInput(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id2": 2, "f1": "v2"}, @@ -1035,9 +1035,9 @@ func TestProjectPlan_MultiInput(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, @@ -1045,9 +1045,9 @@ func TestProjectPlan_MultiInput(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, @@ -1055,9 +1055,9 @@ func TestProjectPlan_MultiInput(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, @@ -1077,19 +1077,19 @@ func TestProjectPlan_MultiInput(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, @@ -1114,19 +1114,19 @@ func TestProjectPlan_MultiInput(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 2, "f2": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 4, "f2": "w3"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id": 3, "f1": "v1"}}, }, }, @@ -1151,7 +1151,7 @@ func TestProjectPlan_MultiInput(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, @@ -1163,7 +1163,7 @@ func TestProjectPlan_MultiInput(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}, @@ -1186,9 +1186,9 @@ func TestProjectPlan_MultiInput(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, @@ -1196,9 +1196,9 @@ func TestProjectPlan_MultiInput(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, @@ -1206,9 +1206,9 @@ func TestProjectPlan_MultiInput(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, @@ -1235,9 +1235,9 @@ func TestProjectPlan_MultiInput(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}}, }, @@ -1245,9 +1245,9 @@ func TestProjectPlan_MultiInput(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}}, }, @@ -1255,9 +1255,9 @@ func TestProjectPlan_MultiInput(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v1"}}, }, }, @@ -1282,7 +1282,7 @@ func TestProjectPlan_MultiInput(t *testing.T) { { sql: `SELECT * EXCEPT(a, b) from test WHERE f1 = "v1" GROUP BY TUMBLINGWINDOW(ss, 10)`, data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1", "a": map[string]interface{}{"b": "test"}, "b": "b", "c": "c"}, @@ -1307,7 +1307,7 @@ func TestProjectPlan_MultiInput(t *testing.T) { { sql: `SELECT * REPLACE(a->b as a) from test WHERE f1 = "v1" GROUP BY TUMBLINGWINDOW(ss, 10)`, data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1", "a": map[string]interface{}{"b": "test"}, "b": "b", "c": "c"}, @@ -1332,7 +1332,7 @@ func TestProjectPlan_MultiInput(t *testing.T) { { sql: `SELECT * EXCEPT(c) REPLACE("test" as b, a->b as a) from test WHERE f1 = "v1" GROUP BY TUMBLINGWINDOW(ss, 10)`, data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1", "a": map[string]interface{}{"b": "test"}, "b": "test"}, @@ -1399,7 +1399,7 @@ func TestProjectPlan_Funcs(t *testing.T) { { sql: "SELECT round(a) as r FROM test GROUP BY TumblingWindow(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"a": 53.1}, @@ -1425,7 +1425,7 @@ func TestProjectPlan_Funcs(t *testing.T) { { sql: "SELECT round(a) as r FROM test GROUP BY TumblingWindow(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"a": 53.1}, @@ -1453,19 +1453,19 @@ func TestProjectPlan_Funcs(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 65.55}}, &xsql.Tuple{Emitter: "test1", Message: xsql.Message{"id": 1, "b": 12}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 2, "a": 73.499}}, &xsql.Tuple{Emitter: "test1", Message: xsql.Message{"id": 2, "b": 34}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 3, "a": 88.88}}, &xsql.Tuple{Emitter: "test1", Message: xsql.Message{"id": 3, "b": 6}}, }, @@ -1486,19 +1486,19 @@ func TestProjectPlan_Funcs(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 65.55}}, &xsql.Tuple{Emitter: "test1", Message: xsql.Message{"id": 1, "b": 12}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 2, "a": 73.499}}, &xsql.Tuple{Emitter: "test1", Message: xsql.Message{"id": 2, "b": 34}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 3, "a": 88.88}}, &xsql.Tuple{Emitter: "test1", Message: xsql.Message{"id": 3, "b": 6}}, }, @@ -1532,19 +1532,19 @@ func TestProjectPlan_Funcs(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 65.55}, Metadata: xsql.Metadata{"device": "devicea"}}, &xsql.Tuple{Emitter: "test1", Message: xsql.Message{"id": 1, "b": 12}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 2, "a": 73.499}, Metadata: xsql.Metadata{"device": "deviceb"}}, &xsql.Tuple{Emitter: "test1", Message: xsql.Message{"id": 2, "b": 34}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 3, "a": 88.88}, Metadata: xsql.Metadata{"device": "devicec"}}, &xsql.Tuple{Emitter: "test1", Message: xsql.Message{"id": 3, "b": 6}}, }, @@ -1595,15 +1595,15 @@ func TestProjectPlan_AggFuncs(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 122.33, "c": 2, "r": 122}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 5, "a": 177.51}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 5, "color": "w2"}}, }, @@ -1612,15 +1612,15 @@ func TestProjectPlan_AggFuncs(t *testing.T) { WindowRange: xsql.NewWindowRange(1541152486013, 1541152487013), }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 2, "a": 89.03, "c": 2, "r": 89}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 2, "color": "w1"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 4, "a": 14.6}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 4, "color": "w1"}}, }, @@ -1648,15 +1648,15 @@ func TestProjectPlan_AggFuncs(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 122.33, "c": 2, "r": 122}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 5}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 5, "color": "w2"}}, }, @@ -1664,15 +1664,15 @@ func TestProjectPlan_AggFuncs(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 2, "a": 89.03, "c": 2, "r": 89}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 2, "color": "w1"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 4, "a": 14.6}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 4, "color": "w1"}}, }, @@ -1701,27 +1701,27 @@ func TestProjectPlan_AggFuncs(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 122.33}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 68.54}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 4, "a": 98.31}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 4, "color": "w2"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 5, "a": 177.54}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 5, "color": "w2"}}, }, @@ -1729,15 +1729,15 @@ func TestProjectPlan_AggFuncs(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 2, "a": 89.03}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 2, "color": "w1"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 4, "a": 14.6}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 4, "color": "w1"}}, }, @@ -1758,21 +1758,21 @@ func TestProjectPlan_AggFuncs(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 122.33}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 68.55}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 5, "a": 177.51}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 5, "color": "w2"}}, }, @@ -1780,15 +1780,15 @@ func TestProjectPlan_AggFuncs(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 2, "a": 89.03}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 2, "color": "w1"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 4, "a": 14.6}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 4, "color": "w1"}}, }, @@ -1809,19 +1809,19 @@ func TestProjectPlan_AggFuncs(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 122.33}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 68.55}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 5, "a": 177.51}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 5, "color": "w2"}}, }, @@ -1842,19 +1842,19 @@ func TestProjectPlan_AggFuncs(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 68.55}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 5, "a": 177.51}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 5, "color": "w2"}}, }, @@ -1875,7 +1875,7 @@ func TestProjectPlan_AggFuncs(t *testing.T) { { sql: "SELECT sum(a), window_start() as ws, window_end() FROM test GROUP BY TumblingWindow(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"a": 53}, @@ -1899,7 +1899,7 @@ func TestProjectPlan_AggFuncs(t *testing.T) { { sql: "SELECT sum(a) as s FROM test GROUP BY TumblingWindow(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"a": 53, "s": 123203}, @@ -1921,7 +1921,7 @@ func TestProjectPlan_AggFuncs(t *testing.T) { { sql: "SELECT sum(a) FROM test GROUP BY TumblingWindow(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"a": 53}, @@ -1942,7 +1942,7 @@ func TestProjectPlan_AggFuncs(t *testing.T) { { sql: "SELECT count(*) as all, count(a) as c, avg(a) as a, sum(a) as s, min(a) as min, max(a) as max FROM test GROUP BY TumblingWindow(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"a": 53}, @@ -1970,15 +1970,15 @@ func TestProjectPlan_AggFuncs(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 122.33}}, &xsql.Tuple{Emitter: "test1", Message: xsql.Message{"id": 1, "color": "w2"}, Metadata: xsql.Metadata{"device": "devicea"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 5, "a": 177.51}}, &xsql.Tuple{Emitter: "test1", Message: xsql.Message{"id": 5, "color": "w2"}, Metadata: xsql.Metadata{"device": "deviceb"}}, }, @@ -1986,15 +1986,15 @@ func TestProjectPlan_AggFuncs(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 2, "a": 89.03}}, &xsql.Tuple{Emitter: "test1", Message: xsql.Message{"id": 2, "color": "w1"}, Metadata: xsql.Metadata{"device": "devicec"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 4, "a": 14.6}}, &xsql.Tuple{Emitter: "test1", Message: xsql.Message{"id": 4, "color": "w1"}, Metadata: xsql.Metadata{"device": "deviced"}}, }, @@ -2017,15 +2017,15 @@ func TestProjectPlan_AggFuncs(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 122.33, "c": 2, "d": "devicea"}}, &xsql.Tuple{Emitter: "test1", Message: xsql.Message{"id": 1, "color": "w2"}, Metadata: xsql.Metadata{"device": "devicea"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 5, "a": 177.51}}, &xsql.Tuple{Emitter: "test1", Message: xsql.Message{"id": 5, "color": "w2"}, Metadata: xsql.Metadata{"device": "deviceb"}}, }, @@ -2033,15 +2033,15 @@ func TestProjectPlan_AggFuncs(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 2, "a": 89.03, "c": 2, "d": "devicec"}}, &xsql.Tuple{Emitter: "test1", Message: xsql.Message{"id": 2, "color": "w1"}, Metadata: xsql.Metadata{"device": "devicec"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 4, "a": 14.6}}, &xsql.Tuple{Emitter: "test1", Message: xsql.Message{"id": 4, "color": "w1"}, Metadata: xsql.Metadata{"device": "deviced"}}, }, @@ -2064,15 +2064,15 @@ func TestProjectPlan_AggFuncs(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 122.33, "c": 2, "r": 122}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 5, "a": 177.51}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 5, "color": "w2"}}, }, @@ -2081,15 +2081,15 @@ func TestProjectPlan_AggFuncs(t *testing.T) { WindowRange: xsql.NewWindowRange(1541152486013, 1541152487013), }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 2, "a": 89.03, "c": 2, "r": 89}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 2, "color": "w1"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 4, "a": 14.6}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 4, "color": "w1"}}, }, @@ -2119,15 +2119,15 @@ func TestProjectPlan_AggFuncs(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 122.33, "c": 2, "r": 122}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 5, "a": 177.51}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 5, "color": "w2"}}, }, @@ -2135,15 +2135,15 @@ func TestProjectPlan_AggFuncs(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 2, "a": 89.03, "c": 2, "r": 89}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 2, "color": "w1"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 4, "a": 14.6}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 4, "color": "w1"}}, }, @@ -2160,7 +2160,7 @@ func TestProjectPlan_AggFuncs(t *testing.T) { { sql: "SELECT collect(*)[1] as c1 FROM test GROUP BY TumblingWindow(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"a": 53, "s": 123203}, @@ -2185,7 +2185,7 @@ func TestProjectPlan_AggFuncs(t *testing.T) { { sql: "SELECT collect(*)[1]->a as c1 FROM test GROUP BY TumblingWindow(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"a": 53, "s": 123203}, @@ -2207,7 +2207,7 @@ func TestProjectPlan_AggFuncs(t *testing.T) { { sql: "SELECT collect(*)[1]->sl[0] as c1 FROM test GROUP BY TumblingWindow(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"a": 53, "sl": []string{"hello", "world"}}, @@ -2231,15 +2231,15 @@ func TestProjectPlan_AggFuncs(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 122.33, "c": 2, "r": 122}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 5, "a": 177.51}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 5, "color": "w2"}}, }, @@ -2247,15 +2247,15 @@ func TestProjectPlan_AggFuncs(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 2, "a": 89.03, "c": 2, "r": 89}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 2, "color": "w1"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 4, "a": 14.6}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 4, "color": "w1"}}, }, @@ -2282,7 +2282,7 @@ func TestProjectPlan_AggFuncs(t *testing.T) { { sql: "SELECT deduplicate(a, false)->a as c1 FROM test GROUP BY TumblingWindow(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"a": 53, "s": 123203}, @@ -2304,7 +2304,7 @@ func TestProjectPlan_AggFuncs(t *testing.T) { { sql: "SELECT deduplicate(a, false) as c1 FROM test GROUP BY TumblingWindow(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"a": 53, "s": 123203}, @@ -2324,7 +2324,7 @@ func TestProjectPlan_AggFuncs(t *testing.T) { { sql: "SELECT deduplicate(a, false) as c1 FROM test GROUP BY TumblingWindow(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"a": 53, "s": 123203}, @@ -2346,14 +2346,14 @@ func TestProjectPlan_AggFuncs(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "B", Message: xsql.Message{"module": 1, "topic": "moduleB topic", "value": 1}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "C", Message: xsql.Message{"module": 1, "topic": "moduleC topic", "value": 100}}, }, }, @@ -2373,19 +2373,19 @@ func TestProjectPlan_AggFuncs(t *testing.T) { data: &xsql.JoinTuples{ Content: []*xsql.JoinTuple{ { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": "a", "b": "b"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": "a", "b": "b"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, { - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 5, "a": "a", "b": "b"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 5, "color": "w2"}}, }, @@ -2472,27 +2472,27 @@ func TestProjectPlanError(t *testing.T) { data: &xsql.GroupedTuplesSet{ Groups: []*xsql.GroupedTuples{ { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 122.33}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 1, "a": 68.54}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 1, "color": "w2"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 4, "a": "dde"}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 4, "color": "w2"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 5, "a": 177.54}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 5, "color": "w2"}}, }, @@ -2500,15 +2500,15 @@ func TestProjectPlanError(t *testing.T) { }, }, { - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 2, "a": 89.03}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 2, "color": "w1"}}, }, }, &xsql.JoinTuple{ - Tuples: []xsql.TupleRow{ + Tuples: []xsql.Row{ &xsql.Tuple{Emitter: "test", Message: xsql.Message{"id": 4, "a": 14.6}}, &xsql.Tuple{Emitter: "src2", Message: xsql.Message{"id": 4, "color": "w1"}}, }, @@ -2523,7 +2523,7 @@ func TestProjectPlanError(t *testing.T) { { sql: "SELECT sum(a) as sum FROM test GROUP BY TumblingWindow(ss, 10)", data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "src1", Message: xsql.Message{"a": 53}, diff --git a/internal/topo/operator/projectset_operator.go b/internal/topo/operator/projectset_operator.go index 5ea193b27c..e1643adeef 100644 --- a/internal/topo/operator/projectset_operator.go +++ b/internal/topo/operator/projectset_operator.go @@ -1,4 +1,4 @@ -// Copyright 2023 EMQ Technologies Co., Ltd. +// Copyright 2023-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -35,12 +35,12 @@ type ProjectSetOperator struct { // [{"a":[1,2],"b":3},{"a":[1,2],"b":4}] = > [{"a":"1","b":3},{"a":"2","b":3},{"a":"1","b":4},{"a":"2","b":4}] func (ps *ProjectSetOperator) Apply(_ api.StreamContext, data interface{}, _ *xsql.FunctionValuer, _ *xsql.AggregateFunctionValuer) interface{} { if ps.LimitCount == 0 && ps.EnableLimit { - return []xsql.TupleRow{} + return []xsql.Row{} } switch input := data.(type) { case error: return input - case xsql.TupleRow: + case xsql.Row: results, err := ps.handleSRFRow(input) if err != nil { return err @@ -97,7 +97,7 @@ func (ps *ProjectSetOperator) handleSRFRowForCollection(data xsql.Collection) er } collection.Groups = newGroups case *xsql.WindowTuples: - newContent := make([]xsql.TupleRow, 0) + newContent := make([]xsql.Row, 0) for _, c := range collection.Content { rs, err := ps.handleSRFRow(c) if err != nil { @@ -112,7 +112,7 @@ func (ps *ProjectSetOperator) handleSRFRowForCollection(data xsql.Collection) er return nil } -func (ps *ProjectSetOperator) handleSRFRow(row xsql.CloneAbleRow) (*resultWrapper, error) { +func (ps *ProjectSetOperator) handleSRFRow(row xsql.Row) (*resultWrapper, error) { // for now we only support 1 srf function in the field srfName := "" for k := range ps.SrfMapping { @@ -129,17 +129,17 @@ func (ps *ProjectSetOperator) handleSRFRow(row xsql.CloneAbleRow) (*resultWrappe } res := newResultWrapper(len(aValues), row) for i, v := range aValues { - newTupleRow := row.Clone() + newRow := row.Clone().(xsql.Row) // clear original column value - newTupleRow.Del(srfName) + newRow.Del(srfName) if mv, ok := v.(map[string]interface{}); ok { for k, v := range mv { - newTupleRow.Set(k, v) + newRow.Set(k, v) } } else { - newTupleRow.Set(srfName, v) + newRow.Set(srfName, v) } - res.appendTuple(i, newTupleRow) + res.appendTuple(i, newRow) } return res, nil } @@ -147,29 +147,29 @@ func (ps *ProjectSetOperator) handleSRFRow(row xsql.CloneAbleRow) (*resultWrappe type resultWrapper struct { joinTuples []*xsql.JoinTuple groupTuples []*xsql.GroupedTuples - rows []xsql.TupleRow + rows []xsql.Row } -func newResultWrapper(len int, row xsql.CloneAbleRow) *resultWrapper { +func newResultWrapper(len int, row xsql.Row) *resultWrapper { r := &resultWrapper{} switch row.(type) { case *xsql.JoinTuple: r.joinTuples = make([]*xsql.JoinTuple, len) case *xsql.GroupedTuples: r.groupTuples = make([]*xsql.GroupedTuples, len) - case xsql.TupleRow: - r.rows = make([]xsql.TupleRow, len) + case xsql.Row: + r.rows = make([]xsql.Row, len) } return r } -func (r *resultWrapper) appendTuple(index int, newRow xsql.CloneAbleRow) { +func (r *resultWrapper) appendTuple(index int, newRow xsql.Row) { switch row := newRow.(type) { case *xsql.JoinTuple: r.joinTuples[index] = row case *xsql.GroupedTuples: r.groupTuples[index] = row - case xsql.TupleRow: + case xsql.Row: r.rows[index] = row } } diff --git a/internal/topo/operator/script_test.go b/internal/topo/operator/script_test.go index 7ac8fb8aa7..1eb310a688 100644 --- a/internal/topo/operator/script_test.go +++ b/internal/topo/operator/script_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 EMQ Technologies Co., Ltd. +// Copyright 2023-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -56,7 +56,7 @@ func TestScriptOp(t *testing.T) { return msgs }`, data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "tbl", Message: xsql.Message{ @@ -78,7 +78,7 @@ func TestScriptOp(t *testing.T) { }, }, result: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "", Message: xsql.Message{ @@ -110,7 +110,7 @@ func TestScriptOp(t *testing.T) { }`, isAgg: true, data: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Emitter: "tbl", Message: xsql.Message{ diff --git a/internal/topo/operator/table_processor.go b/internal/topo/operator/table_processor.go index 3555cbfe8d..bb57ce95f4 100644 --- a/internal/topo/operator/table_processor.go +++ b/internal/topo/operator/table_processor.go @@ -1,4 +1,4 @@ -// Copyright 2021-2023 EMQ Technologies Co., Ltd. +// Copyright 2021-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -66,7 +66,7 @@ func (p *TableProcessor) Apply(ctx api.StreamContext, data interface{}, _ *xsql. logger.Debugf("preprocessor receive %v", tuple) if p.batchEmitted { p.output = &xsql.WindowTuples{ - Content: make([]xsql.TupleRow, 0), + Content: make([]xsql.Row, 0), } p.batchEmitted = false } @@ -77,12 +77,12 @@ func (p *TableProcessor) Apply(ctx api.StreamContext, data interface{}, _ *xsql. return fmt.Errorf("error in preprocessor: %s", err) } } - var newTuples []xsql.TupleRow + var newTuples []xsql.Row _ = p.output.Range(func(i int, r xsql.ReadonlyRow) (bool, error) { if p.retainSize > 0 && p.output.Len() == p.retainSize && i == 0 { return true, nil } - newTuples = append(newTuples, r.(xsql.TupleRow)) + newTuples = append(newTuples, r.(xsql.Row)) return true, nil }) newTuples = append(newTuples, tuple) diff --git a/internal/topo/operator/table_processor_test.go b/internal/topo/operator/table_processor_test.go index 36a6b9b8cb..dc9dc989d1 100644 --- a/internal/topo/operator/table_processor_test.go +++ b/internal/topo/operator/table_processor_test.go @@ -1,4 +1,4 @@ -// Copyright 2021-2022 EMQ Technologies Co., Ltd. +// Copyright 2021-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -48,7 +48,7 @@ func TestTableProcessor_Apply(t *testing.T) { }, data: []byte(`[{"a": [{"b" : "hello1"}, {"b" : "hello2"}]},{"a": [{"b" : "hello2"}, {"b" : "hello3"}]},{"a": [{"b" : "hello3"}, {"b" : "hello4"}]}]`), result: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Message: xsql.Message{ "a": []interface{}{ @@ -85,7 +85,7 @@ func TestTableProcessor_Apply(t *testing.T) { }, data: []byte(`[{"a": {"b" : "hello", "c": {"d": 35.2}}},{"a": {"b" : "world", "c": {"d": 65.2}}}]`), result: &xsql.WindowTuples{ - Content: []xsql.TupleRow{ + Content: []xsql.Row{ &xsql.Tuple{ Message: xsql.Message{ "a": map[string]interface{}{ @@ -121,7 +121,7 @@ func TestTableProcessor_Apply(t *testing.T) { pp := &TableProcessor{isBatchInput: true, emitterName: "demo", checkSchema: true} pp.streamFields = tt.stmt.StreamFields.ToJsonSchema() pp.output = &xsql.WindowTuples{ - Content: make([]xsql.TupleRow, 0), + Content: make([]xsql.Row, 0), } var dm []map[string]interface{} diff --git a/internal/topo/operator/windowfunc_operator.go b/internal/topo/operator/windowfunc_operator.go index 545bd108ab..6e643e1801 100644 --- a/internal/topo/operator/windowfunc_operator.go +++ b/internal/topo/operator/windowfunc_operator.go @@ -1,4 +1,4 @@ -// Copyright 2023 EMQ Technologies Co., Ltd. +// Copyright 2023-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -27,7 +27,7 @@ type WindowFuncOperator struct { } type windowFuncHandle interface { - handleTuple(input xsql.TupleRow) + handleTuple(input xsql.Row) handleCollection(input xsql.Collection) } @@ -35,7 +35,7 @@ type rowNumberFuncHandle struct { name string } -func (rh *rowNumberFuncHandle) handleTuple(input xsql.TupleRow) { +func (rh *rowNumberFuncHandle) handleTuple(input xsql.Row) { input.Set(rh.name, 1) } @@ -66,7 +66,7 @@ func (wf *WindowFuncOperator) Apply(_ api.StreamContext, data interface{}, _ *xs return err } switch input := data.(type) { - case xsql.TupleRow: + case xsql.Row: wh.handleTuple(input) case xsql.Collection: wh.handleCollection(input) diff --git a/internal/xsql/collection.go b/internal/xsql/collection.go index ed7d032cea..bb44527a9c 100644 --- a/internal/xsql/collection.go +++ b/internal/xsql/collection.go @@ -1,4 +1,4 @@ -// Copyright 2022-2023 EMQ Technologies Co., Ltd. +// Copyright 2022-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -40,33 +40,22 @@ type Collection interface { GroupRange(func(i int, aggRow CollectionRow) (bool, error)) error // Range through each row. For grouped collection, each row is an aggregation of groups Range(func(i int, r ReadonlyRow) (bool, error)) error - // RangeSet range through each row by cloneing the row + // RangeSet range through each row by cloning the row RangeSet(func(i int, r Row) (bool, error)) error Filter(indexes []int) Collection GetWindowRange() *WindowRange - Clone() Collection // ToMaps returns the data as a map ToMaps() []map[string]interface{} -} - -type SingleCollection interface { - Collection - CollectionRow + // SetIsAgg Set by project, indicate if the collection is used in an aggregate context which will affect ToMaps output SetIsAgg(isAgg bool) // ToAggMaps returns the aggregated data as a map ToAggMaps() []map[string]interface{} // ToRowMaps returns all the data in the collection ToRowMaps() []map[string]interface{} -} - -type GroupedCollection interface { - Collection -} - -// MergedCollection is a collection of rows that are from different sources -type MergedCollection interface { - Collection - GetBySrc(emitter string) []TupleRow + // GetBySrc returns the rows by the given emitter + GetBySrc(emitter string) []Row + // Clone the collection + Clone() Collection } /* @@ -74,9 +63,9 @@ type MergedCollection interface { */ type WindowTuples struct { - Content []TupleRow // immutable + Content []Row // immutable *WindowRange - contentBySrc map[string][]TupleRow // volatile, temporary cache] + contentBySrc map[string][]Row // volatile, temporary cache] AffiliateRow cachedMap map[string]interface{} @@ -84,13 +73,10 @@ type WindowTuples struct { } var ( - _ MergedCollection = &WindowTuples{} - _ SingleCollection = &WindowTuples{} + _ Collection = &WindowTuples{} + _ CollectionRow = &WindowTuples{} ) -// Window Tuples is also an aggregate row -var _ CollectionRow = &WindowTuples{} - type JoinTuples struct { Content []*JoinTuple *WindowRange @@ -101,8 +87,8 @@ type JoinTuples struct { } var ( - _ SingleCollection = &JoinTuples{} - _ CollectionRow = &JoinTuples{} + _ Collection = &JoinTuples{} + _ CollectionRow = &JoinTuples{} ) type GroupedTuplesSet struct { @@ -110,7 +96,7 @@ type GroupedTuplesSet struct { *WindowRange } -var _ GroupedCollection = &GroupedTuplesSet{} +var _ Collection = &GroupedTuplesSet{} /* * Collection implementations @@ -129,15 +115,17 @@ func (w *WindowTuples) Swap(i, j int) { w.Content[i], w.Content[j] = w.Content[j], w.Content[i] } -func (w *WindowTuples) GetBySrc(emitter string) []TupleRow { +func (w *WindowTuples) GetBySrc(emitter string) []Row { if w.contentBySrc == nil { - w.contentBySrc = make(map[string][]TupleRow) + w.contentBySrc = make(map[string][]Row) for _, t := range w.Content { - e := t.GetEmitter() - if _, hasEmitter := w.contentBySrc[e]; !hasEmitter { - w.contentBySrc[e] = make([]TupleRow, 0) + if et, ok := t.(EmittedData); ok { + e := et.GetEmitter() + if _, hasEmitter := w.contentBySrc[e]; !hasEmitter { + w.contentBySrc[e] = make([]Row, 0) + } + w.contentBySrc[e] = append(w.contentBySrc[e], t) } - w.contentBySrc[e] = append(w.contentBySrc[e], t) } } return w.contentBySrc[emitter] @@ -170,7 +158,7 @@ func (w *WindowTuples) RangeSet(f func(i int, r Row) (bool, error)) error { if !b { break } - w.Content[i] = rc.(TupleRow) + w.Content[i] = rc.(Row) } return nil } @@ -196,7 +184,7 @@ func (w *WindowTuples) AggregateEval(expr ast.Expr, v CallValuer) []interface{} // Filter the tuples by the given predicate func (w *WindowTuples) Filter(indexes []int) Collection { w.cachedMap = nil - newC := make([]TupleRow, 0, len(indexes)) + newC := make([]Row, 0, len(indexes)) for _, i := range indexes { newC = append(newC, w.Content[i]) } @@ -245,9 +233,9 @@ func (w *WindowTuples) ToMap() map[string]interface{} { } func (w *WindowTuples) Clone() Collection { - ts := make([]TupleRow, len(w.Content)) + ts := make([]Row, len(w.Content)) for i, t := range w.Content { - ts[i] = t.Clone().(TupleRow) + ts[i] = t.Clone() } c := &WindowTuples{ Content: ts, @@ -283,7 +271,7 @@ func (w *WindowTuples) Pick(allWildcard bool, cols [][]string, wildcardEmitters for i, t := range w.Content { tc := t.Clone() tc.Pick(allWildcard, cols, wildcardEmitters, except) - w.Content[i] = tc.(TupleRow) + w.Content[i] = tc } } @@ -429,6 +417,11 @@ func (s *JoinTuples) SetIsAgg(_ bool) { s.isAgg = true } +// GetBySrc to be implemented to support join after join +func (s *JoinTuples) GetBySrc(_ string) []Row { + return nil +} + func (s *GroupedTuplesSet) Len() int { return len(s.Groups) } func (s *GroupedTuplesSet) Swap(i, j int) { s.Groups[i], s.Groups[j] = s.Groups[j], s.Groups[i] } func (s *GroupedTuplesSet) Index(i int) Row { return s.Groups[i] } @@ -499,7 +492,7 @@ func (s *GroupedTuplesSet) Clone() Collection { } } -func (s *GroupedTuplesSet) ToMaps() []map[string]interface{} { +func (s *GroupedTuplesSet) ToMaps() []map[string]any { r := make([]map[string]interface{}, len(s.Groups)) for i, t := range s.Groups { r[i] = t.ToMap() @@ -507,6 +500,23 @@ func (s *GroupedTuplesSet) ToMaps() []map[string]interface{} { return r } +func (s *GroupedTuplesSet) SetIsAgg(_ bool) { + // do nothing +} + +func (s *GroupedTuplesSet) ToAggMaps() []map[string]any { + return s.ToMaps() +} + +func (s *GroupedTuplesSet) ToRowMaps() []map[string]any { + return s.ToMaps() +} + +// GetBySrc to be implemented to support join after join +func (s *GroupedTuplesSet) GetBySrc(_ string) []Row { + return nil +} + /* * WindowRange definitions. It should be immutable */ diff --git a/internal/xsql/collection_test.go b/internal/xsql/collection_test.go index fd6cc8eabe..6ede216463 100644 --- a/internal/xsql/collection_test.go +++ b/internal/xsql/collection_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 EMQ Technologies Co., Ltd. +// Copyright 2022-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -33,13 +33,13 @@ func TestCollectionAgg(t *testing.T) { result [][][]map[string]interface{} }{ { - collO: &WindowTuples{Content: []TupleRow{}}, + collO: &WindowTuples{Content: []Row{}}, set: [][]map[string]interface{}{}, interMaps: [][]map[string]interface{}{}, result: [][][]map[string]interface{}{}, }, { - collO: &WindowTuples{Content: []TupleRow{ + collO: &WindowTuples{Content: []Row{ &Tuple{Emitter: "a", Message: map[string]interface{}{"a": 1, "b": "2"}, Timestamp: conf.GetNowInMilli(), Metadata: nil}, &Tuple{Emitter: "a", Message: map[string]interface{}{"a": 2, "b": "4"}, Timestamp: conf.GetNowInMilli(), Metadata: nil}, &Tuple{Emitter: "a", Message: map[string]interface{}{"a": 3, "b": "6"}, Timestamp: conf.GetNowInMilli(), Metadata: nil}, @@ -88,12 +88,12 @@ func TestCollectionAgg(t *testing.T) { { collO: &JoinTuples{Content: []*JoinTuple{ { - Tuples: []TupleRow{ + Tuples: []Row{ &Tuple{Emitter: "src1", Message: Message{"a": 1, "b": "v1"}}, &Tuple{Emitter: "src2", Message: Message{"a": 2, "c": "w2"}}, }, }, { - Tuples: []TupleRow{ + Tuples: []Row{ &Tuple{Emitter: "src1", Message: Message{"a": 3, "b": "v2"}}, &Tuple{Emitter: "src2", Message: Message{"a": 4, "c": "w1"}}, }, @@ -141,12 +141,12 @@ func TestCollectionAgg(t *testing.T) { { collO: &GroupedTuplesSet{Groups: []*GroupedTuples{ { - Content: []TupleRow{ + Content: []Row{ &Tuple{Emitter: "src1", Message: Message{"a": 1, "b": "v1"}}, &Tuple{Emitter: "src1", Message: Message{"a": 2, "b": "w2"}}, }, }, { - Content: []TupleRow{ + Content: []Row{ &Tuple{Emitter: "src1", Message: Message{"a": 3, "b": "v2"}}, &Tuple{Emitter: "src1", Message: Message{"a": 4, "b": "w1"}}, }, @@ -206,7 +206,7 @@ func TestCollectionAgg(t *testing.T) { for si, set := range tt.set { wg.Add(1) go func(si int, set []map[string]interface{}) { - nr := tt.collO.Clone() + nr := tt.collO.Clone().(Collection) nr.RangeSet(func(_ int, row Row) (bool, error) { for k, v := range set[0] { if strings.HasPrefix(k, "@") { @@ -223,10 +223,8 @@ func TestCollectionAgg(t *testing.T) { for j := 1; j < len(set); j++ { wg2.Add(1) go func(j int) { - nnr := nr.Clone() - if nnsr, ok := nnr.(SingleCollection); ok { - nnsr.SetIsAgg(true) - } + nnr := nr.Clone().(Collection) + nnr.SetIsAgg(true) nnr.GroupRange(func(_ int, aggRow CollectionRow) (bool, error) { for k, v := range set[j] { if strings.HasPrefix(k, "@") { diff --git a/internal/xsql/row.go b/internal/xsql/row.go index e0124e1d71..29a0006529 100644 --- a/internal/xsql/row.go +++ b/internal/xsql/row.go @@ -45,7 +45,8 @@ type ReadonlyRow interface { Wildcarder } -type Row interface { +// RawRow is the basic data type for logical row. It could be a row or a collection row. +type RawRow interface { ReadonlyRow // Del Only for some ops like functionOp * and Alias Del(col string) @@ -58,15 +59,17 @@ type Row interface { Pick(allWildcard bool, cols [][]string, wildcardEmitters map[string]bool, except []string) } -type CloneAbleRow interface { - Row - // Clone when broadcast to make sure each row are dealt single threaded - Clone() CloneAbleRow +type Row interface { + RawRow + Clone() Row +} + +type MetaData interface { + MetaData() Metadata } -// TupleRow is a mutable row. Function with * could modify the row. -type TupleRow interface { - CloneAbleRow +// EmittedData is data that is produced by a specific source +type EmittedData interface { // GetEmitter returns the emitter of the row GetEmitter() string } @@ -74,7 +77,7 @@ type TupleRow interface { // CollectionRow is the aggregation row of a non-grouped collection. Thinks of it as a single group. // The row data is immutable type CollectionRow interface { - Row + RawRow AggregateData // Clone when broadcast to make sure each row are dealt single threaded // Clone() CollectionRow @@ -245,7 +248,10 @@ type Tuple struct { cachedMap map[string]interface{} // clone of the row and cached for performance } -var _ TupleRow = &Tuple{} +var ( + _ Row = &Tuple{} + _ MetaData = &Tuple{} +) type WatermarkTuple struct { Timestamp int64 @@ -261,7 +267,7 @@ func (t *WatermarkTuple) IsWatermark() bool { // JoinTuple is a row produced by a join operation type JoinTuple struct { - Tuples []TupleRow // The content is immutable, but the slice may be add or removed + Tuples []Row // The content is immutable, but the slice may be added or removed AffiliateRow lock sync.Mutex cachedMap map[string]interface{} // clone of the row and cached for performance of toMap @@ -271,11 +277,11 @@ func (jt *JoinTuple) AggregateEval(expr ast.Expr, v CallValuer) []interface{} { return []interface{}{Eval(expr, MultiValuer(jt, v, &WildcardValuer{jt}))} } -var _ TupleRow = &JoinTuple{} +var _ Row = &JoinTuple{} // GroupedTuples is a collection of tuples grouped by a key type GroupedTuples struct { - Content []TupleRow + Content []Row *WindowRange AffiliateRow lock sync.Mutex @@ -362,7 +368,7 @@ func (t *Tuple) All(string) (map[string]interface{}, bool) { return t.Message, true } -func (t *Tuple) Clone() CloneAbleRow { +func (t *Tuple) Clone() Row { return &Tuple{ Emitter: t.Emitter, Timestamp: t.Timestamp, @@ -398,6 +404,10 @@ func (t *Tuple) Meta(key, table string) (interface{}, bool) { return t.Metadata.Value(key, table) } +func (t *Tuple) MetaData() Metadata { + return t.Metadata +} + func (t *Tuple) GetEmitter() string { return t.Emitter } @@ -457,11 +467,11 @@ func (t *Tuple) Pick(allWildcard bool, cols [][]string, wildcardEmitters map[str // JoinTuple implementation -func (jt *JoinTuple) AddTuple(tuple TupleRow) { +func (jt *JoinTuple) AddTuple(tuple Row) { jt.Tuples = append(jt.Tuples, tuple) } -func (jt *JoinTuple) AddTuples(tuples []TupleRow) { +func (jt *JoinTuple) AddTuples(tuples []Row) { jt.Tuples = append(jt.Tuples, tuples...) } @@ -483,7 +493,7 @@ func (jt *JoinTuple) doGetValue(key, table string, isVal bool) (interface{}, boo } else { // TODO should use hash here for _, tuple := range tuples { - if tuple.GetEmitter() == table { + if et, ok := tuple.(EmittedData); ok && et.GetEmitter() == table { return getTupleValue(tuple, key, isVal) } } @@ -518,7 +528,7 @@ func (jt *JoinTuple) Meta(key, table string) (interface{}, bool) { func (jt *JoinTuple) All(stream string) (map[string]interface{}, bool) { if stream != "" { for _, t := range jt.Tuples { - if t.GetEmitter() == stream { + if et, ok := t.(EmittedData); ok && et.GetEmitter() == stream { return t.All("") } } @@ -534,10 +544,10 @@ func (jt *JoinTuple) All(stream string) (map[string]interface{}, bool) { return result, true } -func (jt *JoinTuple) Clone() CloneAbleRow { - ts := make([]TupleRow, len(jt.Tuples)) +func (jt *JoinTuple) Clone() Row { + ts := make([]Row, len(jt.Tuples)) for i, t := range jt.Tuples { - ts[i] = t.Clone().(TupleRow) + ts[i] = t.Clone().(Row) } c := &JoinTuple{ Tuples: ts, @@ -567,10 +577,12 @@ func (jt *JoinTuple) Pick(allWildcard bool, cols [][]string, wildcardEmitters ma if !allWildcard { if len(cols) > 0 { for i, tuple := range jt.Tuples { - if _, ok := wildcardEmitters[tuple.GetEmitter()]; ok { - continue + if et, ok := tuple.(EmittedData); ok { + if _, ok := wildcardEmitters[et.GetEmitter()]; ok { + continue + } } - nt := tuple.Clone().(TupleRow) + nt := tuple.Clone().(Row) nt.Pick(allWildcard, cols, wildcardEmitters, except) jt.Tuples[i] = nt } @@ -621,8 +633,8 @@ func (s *GroupedTuples) ToMap() map[string]interface{} { return s.cachedMap } -func (s *GroupedTuples) Clone() CloneAbleRow { - ts := make([]TupleRow, len(s.Content)) +func (s *GroupedTuples) Clone() Row { + ts := make([]Row, len(s.Content)) for i, t := range s.Content { ts[i] = t } @@ -636,7 +648,7 @@ func (s *GroupedTuples) Clone() CloneAbleRow { func (s *GroupedTuples) Pick(allWildcard bool, cols [][]string, wildcardEmitters map[string]bool, except []string) { cols = s.AffiliateRow.Pick(cols) - sc := s.Content[0].Clone().(TupleRow) + sc := s.Content[0].Clone() sc.Pick(allWildcard, cols, wildcardEmitters, except) s.Content[0] = sc } diff --git a/internal/xsql/row_test.go b/internal/xsql/row_test.go index 4370cde94f..b4839094bc 100644 --- a/internal/xsql/row_test.go +++ b/internal/xsql/row_test.go @@ -1,4 +1,4 @@ -// Copyright 2022-2023 EMQ Technologies Co., Ltd. +// Copyright 2022-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -44,7 +44,7 @@ func TestCollectionRow(t *testing.T) { wildcard: []string{""}, result: []interface{}{4, "b1", 3, map[string]interface{}{"a": 1, "b": "2"}}, }, { - rowC: &JoinTuple{Tuples: []TupleRow{ + rowC: &JoinTuple{Tuples: []Row{ &Tuple{Emitter: "src1", Message: Message{"a": 1, "b": "v1"}}, &Tuple{Emitter: "src2", Message: Message{"a": 2, "c": "w2"}}, }}, @@ -52,7 +52,7 @@ func TestCollectionRow(t *testing.T) { wildcard: []string{"", "src1"}, result: []interface{}{1, 2, "v1", "w2", map[string]interface{}{"a": 2, "b": "v1", "c": "w2"}, map[string]interface{}{"a": 1, "b": "v1"}}, }, { - rowC: &JoinTuple{Tuples: []TupleRow{ + rowC: &JoinTuple{Tuples: []Row{ &Tuple{Emitter: "src1", Message: Message{"a": 1, "b": "v1"}}, &Tuple{Emitter: "src2", Message: Message{"a": 2, "c": "w2"}}, }, AffiliateRow: AffiliateRow{CalCols: map[string]interface{}{"a": 4, "d": 3}, AliasMap: map[string]interface{}{"d": 4}}}, @@ -60,22 +60,22 @@ func TestCollectionRow(t *testing.T) { wildcard: []string{"", "src1"}, result: []interface{}{4, 2, "v1", "w2", 4, map[string]interface{}{"a": 2, "b": "v1", "c": "w2"}, map[string]interface{}{"a": 1, "b": "v1"}}, }, { - rowC: &GroupedTuples{Content: []TupleRow{&Tuple{Emitter: "src1", Message: Message{"a": 1, "b": "v1"}}, &Tuple{Emitter: "src1", Message: Message{"a": 2, "b": "v2"}}}}, + rowC: &GroupedTuples{Content: []Row{&Tuple{Emitter: "src1", Message: Message{"a": 1, "b": "v1"}}, &Tuple{Emitter: "src1", Message: Message{"a": 2, "b": "v2"}}}}, value: []string{"a", "b"}, wildcard: []string{""}, result: []interface{}{1, "v1", map[string]interface{}{"a": 1, "b": "v1"}}, }, { - rowC: &GroupedTuples{Content: []TupleRow{&Tuple{Emitter: "src1", Message: Message{"a": 1, "b": "v1"}}, &Tuple{Emitter: "src1", Message: Message{"a": 2, "b": "v2"}}}, AffiliateRow: AffiliateRow{CalCols: map[string]interface{}{"a": 4, "d": 3}, AliasMap: map[string]interface{}{"d": 4}}}, + rowC: &GroupedTuples{Content: []Row{&Tuple{Emitter: "src1", Message: Message{"a": 1, "b": "v1"}}, &Tuple{Emitter: "src1", Message: Message{"a": 2, "b": "v2"}}}, AffiliateRow: AffiliateRow{CalCols: map[string]interface{}{"a": 4, "d": 3}, AliasMap: map[string]interface{}{"d": 4}}}, value: []string{"a", "b", "d"}, wildcard: []string{""}, result: []interface{}{4, "v1", 4, map[string]interface{}{"a": 1, "b": "v1"}}, }, { - rowC: &WindowTuples{Content: []TupleRow{&Tuple{Emitter: "src1", Message: Message{"a": 1, "b": "v1"}}, &Tuple{Emitter: "src1", Message: Message{"a": 2, "b": "v2"}}}, AffiliateRow: AffiliateRow{CalCols: map[string]interface{}{"a": 4, "d": 3}, AliasMap: map[string]interface{}{"d": 4}}}, + rowC: &WindowTuples{Content: []Row{&Tuple{Emitter: "src1", Message: Message{"a": 1, "b": "v1"}}, &Tuple{Emitter: "src1", Message: Message{"a": 2, "b": "v2"}}}, AffiliateRow: AffiliateRow{CalCols: map[string]interface{}{"a": 4, "d": 3}, AliasMap: map[string]interface{}{"d": 4}}}, value: []string{"a", "b", "d"}, wildcard: []string{""}, result: []interface{}{4, "v1", 4, map[string]interface{}{"a": 1, "b": "v1"}}, }, { - rowC: &JoinTuples{Content: []*JoinTuple{{Tuples: []TupleRow{ + rowC: &JoinTuples{Content: []*JoinTuple{{Tuples: []Row{ &Tuple{Emitter: "src1", Message: Message{"a": 1, "b": "v1"}, AffiliateRow: AffiliateRow{CalCols: map[string]interface{}{"b": "v2", "$$lag_a": 1}}}, &Tuple{Emitter: "src2", Message: Message{"a": 2, "c": "w2"}}, }}}, AffiliateRow: AffiliateRow{CalCols: map[string]interface{}{"a": 4, "d": 3}, AliasMap: map[string]interface{}{"d": 4}}}, @@ -117,10 +117,10 @@ func TestCollectionRow(t *testing.T) { } } -func TestTupleRow(t *testing.T) { +func TestRow(t *testing.T) { // broadcast(clone) -> set -> broadcast -> set -> compare tests := []struct { - rowO TupleRow + rowO Row // The multiple values to set or alias; The first value is set in the first broadcast. the next values are set in the second broadcast. set [][]map[string]interface{} result [][]map[string]interface{} @@ -153,7 +153,7 @@ func TestTupleRow(t *testing.T) { }, }, }, { - rowO: &JoinTuple{Tuples: []TupleRow{ + rowO: &JoinTuple{Tuples: []Row{ &Tuple{Emitter: "src1", Message: Message{"a": 1, "b": "v1"}}, &Tuple{Emitter: "src2", Message: Message{"a": 2, "c": "w2"}}, }}, diff --git a/internal/xsql/sorter.go b/internal/xsql/sorter.go index abcb5d854b..4c6ea58130 100644 --- a/internal/xsql/sorter.go +++ b/internal/xsql/sorter.go @@ -1,4 +1,4 @@ -// Copyright 2022-2023 EMQ Technologies Co., Ltd. +// Copyright 2022-2024 EMQ Technologies Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -76,7 +76,7 @@ func (ms *MultiSorter) Swap(i, j int) { ms.SortingData.Swap(i, j) } -// Sort sorts the argument slice according to the less functions passed to OrderedBy. +// Sort sorts the argument slice according to the fewer functions passed to OrderedBy. func (ms *MultiSorter) Sort(data SortingData) error { ms.SortingData = data types := make([]string, len(ms.fields)) @@ -84,31 +84,17 @@ func (ms *MultiSorter) Sort(data SortingData) error { switch input := data.(type) { case error: return input - case SingleCollection: + case Collection: err := input.RangeSet(func(i int, row Row) (bool, error) { - ms.values[i] = make(map[string]interface{}) - vep := &ValuerEval{Valuer: MultiValuer(ms.valuer, row, ms.valuer, &WildcardValuer{Data: row})} - for j, field := range ms.fields { - vp := vep.Eval(field.FieldExpr) - if types[j] == "" && vp != nil { - types[j] = fmt.Sprintf("%T", vp) - } - if err := validate(types[j], vp); err != nil { - return false, err - } else { - ms.values[i][field.Uname] = vp - } + var vep *ValuerEval + if aggRow, ok := row.(AggregateData); ok { + ms.aggValuer.SetData(aggRow) + vep = &ValuerEval{Valuer: MultiAggregateValuer(aggRow, ms.valuer, row, ms.aggValuer, &WildcardValuer{Data: row})} + } else { + vep = &ValuerEval{Valuer: MultiValuer(ms.valuer, row, ms.valuer, &WildcardValuer{Data: row})} } - return true, nil - }) - if err != nil { - return err - } - case GroupedCollection: - err := input.GroupRange(func(i int, aggRow CollectionRow) (bool, error) { ms.values[i] = make(map[string]interface{}) - ms.aggValuer.SetData(aggRow) - vep := &ValuerEval{Valuer: MultiAggregateValuer(aggRow, ms.valuer, aggRow, ms.aggValuer, &WildcardValuer{Data: aggRow})} + for j, field := range ms.fields { vp := vep.Eval(field.FieldExpr) if types[j] == "" && vp != nil { @@ -125,6 +111,8 @@ func (ms *MultiSorter) Sort(data SortingData) error { if err != nil { return err } + default: + return nil } sort.Sort(ms) return nil